From 3799242bce3e517feb4e426046f8fb3610bcd597 Mon Sep 17 00:00:00 2001 From: "Enot (ded) Skelly" Date: Fri, 24 Jul 2026 13:40:57 -0700 Subject: [PATCH 001/141] docs: update for v1.6.0 --- README.md | 28 ++++++++++++++++++++++++---- config.yaml.example | 9 +++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 54251ba6..53e1fd7b 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,13 @@ deployment `resolvedPath` will show `"confidence": "none"` for all hops and arrives and populates `node_short_ids`. This is expected behaviour — resolution improves automatically as the mesh is observed over time. +Similarly, GRP_TXT packets whose channel key isn't yet known at ingest time are +stored as hash-only, undecrypted rows. Adding the channel's key to `config.yaml` +doesn't retroactively decrypt that history immediately — it's picked up +automatically on the next restart, when Beacon scans for undecrypted packets +matching a now-known channel and decrypts them. Watch the startup log for +`backfilled N previously-undecrypted channel message(s)`. + --- ## Configuration @@ -129,11 +136,15 @@ improves automatically as the mesh is observed over time. ```yaml # Optional IATA overrides — auto-created on first packet arrival, # only needed if you want to customise display name or coordinates. +# borderFile points to a GeoJSON Feature (Polygon or MultiPolygon) for the +# region border map feature; relative paths resolve against this config +# file's own directory. Validated at boot — invalid geometry fails startup. iatas: YVR: name: Vancouver International lat: 49.1967 lng: -123.1815 + borderFile: borders/yvr.geojson # optional # Super-regions grouping multiple IATAs. regions: @@ -187,6 +198,12 @@ presence: websocket: max_connections_per_ip: 5 # default: 5 +# Node staleness, deletion, and clock-drift thresholds. +nodes: + stale_threshold: 24h # mark a node "stale" in the API after this long unseen (default: 24h) + delete_after: 720h # delete a node entirely after this long unseen (default: 30 days, same default as packets.retention) + clock_drift_threshold: 5m # |device clock - server clock| above which clockOutOfSync=true for a repeater/room server (default: 5m) + # Redis caching layer (optional). # Caches read-heavy, slow-changing responses to reduce PostgreSQL load. # Connection details (address, password, database) are set via environment @@ -212,8 +229,8 @@ ingest: ``` IATAs are auto-created on first packet arrival. The config file adds display -names and coordinates. Regions and channel keys must be defined here — they are -not auto-created. +names, coordinates, and optional region borders. Regions and channel keys +must be defined here — they are not auto-created. --- @@ -366,9 +383,10 @@ Not yet implemented — see the Authentication section above. | `GET` | `/channels/{id}/messages` | List messages for a channel (optional: `?since=&iata=&limit=50`) | | `GET` | `/iatas` | List all known IATA codes | | `GET` | `/iatas/{iata}` | Get a single IATA code | +| `GET` | `/iatas/{iata}/border` | Get an IATA's GeoJSON region border, if configured (204 if not) | | `GET` | `/messages` | List all messages (optional: `?channelId=&channelHash=&iata=&since=&limit=50`) | | `GET` | `/messages/backfill` | Backfill messages after a given message ID | -| `GET` | `/nodes` | List nodes | +| `GET` | `/nodes` | List nodes (optional: `?pubkeyPrefix=&neighbors&iatas=&pubkey=`) | | `GET` | `/nodes/{nodeId}` | Get node detail | | `GET` | `/nodes/{nodeId}/neighbors` | List neighboring nodes observed in the mesh | | `GET` | `/nodes/{nodeId}/observations` | List observations for a node | @@ -376,7 +394,7 @@ Not yet implemented — see the Authentication section above. | `GET` | `/observers/{observerId}` | Get observer detail including broker last-seen timestamps | | `GET` | `/observers/{observerId}/adverts` | Adverts heard by observer | | `GET` | `/observers/{observerId}/telemetry` | Observer telemetry history (optional: `?range=24h&interval=1h\|6h\|24h`) | -| `GET` | `/packets` | List packets with filters | +| `GET` | `/packets` | List packets (optional: `?payloadTypes=&routeTypes=&scopes=` accept plural, comma-separated values alongside the singular params) | | `GET` | `/packets/backfill` | Backfill packets after a given observation ID | | `GET` | `/packets/{packetHash}` | Get packet with all observations | | `GET` | `/regions` | List all regions (summary) | @@ -390,8 +408,10 @@ Not yet implemented — see the Authentication section above. | `GET` | `/stats/overview` | Network overview stats | | `GET` | `/stats/payload-breakdown` | Observation counts by payload type (last 24h by default) | | `GET` | `/stats/scopes` | Configured region scopes and breakdown of packets, nodes, observers | +| `GET` | `/stats/top-advertisers` | Top N nodes by distinct ADVERT packet count (last 24h by default, from materialized view) | | `GET` | `/stats/top-nodes` | Top N nodes by observation count (from materialized view) | | `GET` | `/stats/top-observers` | Top N observers by observation count (last 24h by default) | +| `GET` | `/stats/top-talkers` | Top N companion names by decrypted channel message count (last 24h by default, from materialized view) | | `GET` | `/traces` | List trace tags with filters (optional: ?type=TRACE\|PING) | | `GET` | `/traces/{tag}` | Get full trace detail with resolved routes | diff --git a/config.yaml.example b/config.yaml.example index d99f4de1..ce975415 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -6,6 +6,9 @@ iatas: name: Vancouver International lat: 49.1967 lng: -123.1815 + # borderFile: borders/yvr.geojson # optional GeoJSON Feature (Polygon/MultiPolygon) + # # for the region border map; path is relative to + # # this file's directory; validated at boot. YYJ: name: Victoria International lat: 48.6469 @@ -75,6 +78,12 @@ packets: websocket: max_connections_per_ip: 5 # default: 5 +# Node staleness, deletion, and clock-drift thresholds. +#nodes: +# stale_threshold: 24h # default: 24h +# delete_after: 720h # default: 30 days, same default as packets.retention +# clock_drift_threshold: 5m # default: 5m + # CORS configuration. # Controls which origins, methods and headers are allowed for cross-origin requests. # Defaults to allowing all origins with GET/HEAD/OPTIONS if omitted — appropriate From d8371f04d09c4a1f6f6bc269472b29df288cf9de Mon Sep 17 00:00:00 2001 From: "Enot (ded) Skelly" Date: Fri, 24 Jul 2026 13:49:40 -0700 Subject: [PATCH 002/141] feat: split top adverts into flood/direct closes: #90 --- .../021_mv_top_advertisers_route_type.sql | 32 +++++++++++++ db/queries/queries.sql | 2 + db/sqlc/models.go | 16 ++++--- db/sqlc/queries.sql.go | 18 +++++--- db/stats.go | 16 ++++--- db/stats_test.go | 45 +++++++++++++++++++ docs/docs.go | 7 +++ docs/swagger.json | 7 +++ docs/swagger.yaml | 9 ++++ internal/api/stats.go | 8 +++- 10 files changed, 139 insertions(+), 21 deletions(-) create mode 100644 db/migrations/021_mv_top_advertisers_route_type.sql diff --git a/db/migrations/021_mv_top_advertisers_route_type.sql b/db/migrations/021_mv_top_advertisers_route_type.sql new file mode 100644 index 00000000..a2ac09b6 --- /dev/null +++ b/db/migrations/021_mv_top_advertisers_route_type.sql @@ -0,0 +1,32 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Splits mv_top_advertisers_by_iata's advert_count into flood- and direct-routed counts, +-- alongside the existing combined total, so /stats/top-advertisers can report how a node is +-- actually being heard rather than only a single blended number. +-- +-- Flood = route_type 0 (transport_flood) or 1 (flood) -- broadcast, no prior path needed. +-- Direct = route_type 2 (direct) or 3 (transport_direct) -- routed along a known path. + +DROP MATERIALIZED VIEW mv_top_advertisers_by_iata; + +CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at)::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN packet_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at); + +CREATE UNIQUE INDEX idx_mv_top_advertisers + ON mv_top_advertisers_by_iata(iata, node_id, bucket); diff --git a/db/queries/queries.sql b/db/queries/queries.sql index b2253b3f..a206cd68 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -902,6 +902,8 @@ SELECT name, node_type, COALESCE(SUM(advert_count), 0)::bigint AS advert_count, + COALESCE(SUM(flood_advert_count), 0)::bigint AS flood_advert_count, + COALESCE(SUM(direct_advert_count), 0)::bigint AS direct_advert_count, MAX(last_heard)::timestamptz AS last_heard, COALESCE(MAX(iata), '')::bpchar AS iata FROM mv_top_advertisers_by_iata diff --git a/db/sqlc/models.go b/db/sqlc/models.go index a91f5451..af80eeb4 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -90,13 +90,15 @@ type MvRadioPreset struct { } type MvTopAdvertisersByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - AdvertCount int64 `json:"advert_count"` - LastHeard interface{} `json:"last_heard"` + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + NodeType int16 `json:"node_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` + LastHeard interface{} `json:"last_heard"` } type MvTopNodesByIatum struct { diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 4156701d..7d64413d 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1277,6 +1277,8 @@ SELECT name, node_type, COALESCE(SUM(advert_count), 0)::bigint AS advert_count, + COALESCE(SUM(flood_advert_count), 0)::bigint AS flood_advert_count, + COALESCE(SUM(direct_advert_count), 0)::bigint AS direct_advert_count, MAX(last_heard)::timestamptz AS last_heard, COALESCE(MAX(iata), '')::bpchar AS iata FROM mv_top_advertisers_by_iata @@ -1294,12 +1296,14 @@ type GetStatsTopAdvertisersParams struct { } type GetStatsTopAdvertisersRow struct { - ID uuid.UUID `json:"id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - AdvertCount int64 `json:"advert_count"` - LastHeard pgtype.Timestamptz `json:"last_heard"` - Iata string `json:"iata"` + ID uuid.UUID `json:"id"` + Name *string `json:"name"` + NodeType int16 `json:"node_type"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` + LastHeard pgtype.Timestamptz `json:"last_heard"` + Iata string `json:"iata"` } // Top N advertisers in the window, summed from the hourly buckets. @@ -1317,6 +1321,8 @@ func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdv &i.Name, &i.NodeType, &i.AdvertCount, + &i.FloodAdvertCount, + &i.DirectAdvertCount, &i.LastHeard, &i.Iata, ); err != nil { diff --git a/db/stats.go b/db/stats.go index 78c79848..75f93113 100644 --- a/db/stats.go +++ b/db/stats.go @@ -146,13 +146,15 @@ func (s *Store) GetStatsTopAdvertisers(ctx context.Context, iatas []string, sinc items := make([]api.TopAdvertiser, 0, len(rows)) for _, v := range rows { items = append(items, api.TopAdvertiser{ - NodeID: v.ID, - NodeName: v.Name, - NodeType: v.NodeType, - NodeTypeName: api.NodeTypeName(v.NodeType), - IATA: v.Iata, - AdvertCount: v.AdvertCount, - LastHeard: v.LastHeard.Time.UnixMilli(), + NodeID: v.ID, + NodeName: v.Name, + NodeType: v.NodeType, + NodeTypeName: api.NodeTypeName(v.NodeType), + IATA: v.Iata, + AdvertCount: v.AdvertCount, + FloodAdvertCount: v.FloodAdvertCount, + DirectAdvertCount: v.DirectAdvertCount, + LastHeard: v.LastHeard.Time.UnixMilli(), }) } return items, nil diff --git a/db/stats_test.go b/db/stats_test.go index fec74d11..a7c56fba 100644 --- a/db/stats_test.go +++ b/db/stats_test.go @@ -114,6 +114,51 @@ func TestGetStatsTopObservers_IATATypeAssertion(t *testing.T) { } } +func TestGetStatsTopAdvertisers_FloodDirectSplit(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000003") + name := "test-node" + heardAt := pgtype.Timestamptz{Time: time.Now(), Valid: true} + + mock.EXPECT(). + GetStatsTopAdvertisers(gomock.Any(), gomock.Any()). + Return([]sqlc.GetStatsTopAdvertisersRow{ + { + ID: nodeID, + Name: &name, + NodeType: 2, // repeater + AdvertCount: 10, + FloodAdvertCount: 7, + DirectAdvertCount: 3, + LastHeard: heardAt, + Iata: "YVR", + }, + }, nil) + + store := &Store{q: mock} + items, err := store.GetStatsTopAdvertisers(context.Background(), []string{"YVR"}, time.Now().Add(-time.Hour), 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + if items[0].AdvertCount != 10 { + t.Errorf("expected AdvertCount 10, got %d", items[0].AdvertCount) + } + if items[0].FloodAdvertCount != 7 { + t.Errorf("expected FloodAdvertCount 7, got %d", items[0].FloodAdvertCount) + } + if items[0].DirectAdvertCount != 3 { + t.Errorf("expected DirectAdvertCount 3, got %d", items[0].DirectAdvertCount) + } + if items[0].FloodAdvertCount+items[0].DirectAdvertCount != items[0].AdvertCount { + t.Error("expected FloodAdvertCount + DirectAdvertCount to equal AdvertCount") + } +} + func TestGetStatsNodeTypes_Mapping(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/docs/docs.go b/docs/docs.go index 347a251b..9a6fdef3 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -3559,6 +3559,13 @@ const docTemplate = `{ "advertCount": { "type": "integer" }, + "directAdvertCount": { + "type": "integer" + }, + "floodAdvertCount": { + "description": "FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed:\nflood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path;\ndirect = route type 2 (direct) or 3 (transport_direct), routed along a known path.\nFloodAdvertCount + DirectAdvertCount == AdvertCount.", + "type": "integer" + }, "iata": { "type": "string" }, diff --git a/docs/swagger.json b/docs/swagger.json index aa876733..325b8ff2 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -3557,6 +3557,13 @@ "advertCount": { "type": "integer" }, + "directAdvertCount": { + "type": "integer" + }, + "floodAdvertCount": { + "description": "FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed:\nflood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path;\ndirect = route type 2 (direct) or 3 (transport_direct), routed along a known path.\nFloodAdvertCount + DirectAdvertCount == AdvertCount.", + "type": "integer" + }, "iata": { "type": "string" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index d0914d78..456ee23c 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -934,6 +934,15 @@ definitions: properties: advertCount: type: integer + directAdvertCount: + type: integer + floodAdvertCount: + description: |- + FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed: + flood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path; + direct = route type 2 (direct) or 3 (transport_direct), routed along a known path. + FloodAdvertCount + DirectAdvertCount == AdvertCount. + type: integer iata: type: string lastHeard: diff --git a/internal/api/stats.go b/internal/api/stats.go index 714c95d4..7f51a21e 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -76,7 +76,13 @@ type TopAdvertiser struct { NodeTypeName string `json:"nodeTypeName"` IATA string `json:"iata"` AdvertCount int64 `json:"advertCount"` - LastHeard int64 `json:"lastHeard"` // epoch ms + // FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed: + // flood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path; + // direct = route type 2 (direct) or 3 (transport_direct), routed along a known path. + // FloodAdvertCount + DirectAdvertCount == AdvertCount. + FloodAdvertCount int64 `json:"floodAdvertCount"` + DirectAdvertCount int64 `json:"directAdvertCount"` + LastHeard int64 `json:"lastHeard"` // epoch ms } // TopTalker is a companion name ranked by decrypted channel message count within the From 336b734a422c6fb6562aafe023c019a40168714c Mon Sep 17 00:00:00 2001 From: "Enot (ded) Skelly" Date: Fri, 24 Jul 2026 14:01:12 -0700 Subject: [PATCH 003/141] feat: add stats api for new clock drift tracking closes: #91 --- db/queries/queries.sql | 21 ++++++ db/sqlc/mock/querier.go | 15 ++++ db/sqlc/querier.go | 4 ++ db/sqlc/queries.sql.go | 64 +++++++++++++++++ db/stats.go | 35 +++++++++ db/stats_test.go | 39 ++++++++++ docs/docs.go | 86 +++++++++++++++++++++++ docs/swagger.json | 86 +++++++++++++++++++++++ docs/swagger.yaml | 58 +++++++++++++++ internal/api/handlers/stats.go | 43 ++++++++++++ internal/api/handlers/stats_test.go | 46 ++++++++++++ internal/api/handlers/stub_reader_test.go | 8 +++ internal/api/reader.go | 6 ++ internal/api/stats.go | 15 ++++ internal/cache/cache_test.go | 4 ++ internal/cache/reader.go | 15 ++++ 16 files changed, 545 insertions(+) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index a206cd68..4fb5410d 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -913,6 +913,27 @@ GROUP BY node_id, name, node_type ORDER BY advert_count DESC LIMIT $3; +-- name: GetStatsClockDrift :many +-- Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds +-- the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's +-- latest measured drift, not an aggregate over a period. +SELECT + n.id, + n.name, + n.node_type, + n.device_clock_drift_seconds, + n.last_advert_at, + json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) FILTER (WHERE ni.iata IS NOT NULL) AS iatas +FROM nodes n +LEFT JOIN node_iatas ni ON ni.node_id = n.id +WHERE n.node_type IN (2, 3) + AND n.device_clock_drift_seconds IS NOT NULL + AND ABS(n.device_clock_drift_seconds) > $1::int + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) +GROUP BY n.id, n.name, n.node_type, n.device_clock_drift_seconds, n.last_advert_at +ORDER BY ABS(n.device_clock_drift_seconds) DESC +LIMIT $3; + -- name: GetStatsTopTalkers :many -- Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. SELECT diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 91f7e0fb..d6caa7d9 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -548,6 +548,21 @@ func (mr *MockQuerierMockRecorder) GetScopesByIATAs(ctx, dollar_1 any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopesByIATAs", reflect.TypeOf((*MockQuerier)(nil).GetScopesByIATAs), ctx, dollar_1) } +// GetStatsClockDrift mocks base method. +func (m *MockQuerier) GetStatsClockDrift(ctx context.Context, arg db.GetStatsClockDriftParams) ([]db.GetStatsClockDriftRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStatsClockDrift", ctx, arg) + ret0, _ := ret[0].([]db.GetStatsClockDriftRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetStatsClockDrift indicates an expected call of GetStatsClockDrift. +func (mr *MockQuerierMockRecorder) GetStatsClockDrift(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsClockDrift", reflect.TypeOf((*MockQuerier)(nil).GetStatsClockDrift), ctx, arg) +} + // GetStatsNodeTypes mocks base method. func (m *MockQuerier) GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]db.GetStatsNodeTypesRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 03a6a840..a2f8ded5 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -66,6 +66,10 @@ type Querier interface { // before COUNT(DISTINCT) (~10s). GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) + // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds + // the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's + // latest measured drift, not an aggregate over a period. + GetStatsClockDrift(ctx context.Context, arg GetStatsClockDriftParams) ([]GetStatsClockDriftRow, error) // Returns node counts grouped by type, optionally filtered by IATA. GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]GetStatsNodeTypesRow, error) // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 7d64413d..0b536a45 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1157,6 +1157,70 @@ func (q *Queries) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]Ge return items, nil } +const getStatsClockDrift = `-- name: GetStatsClockDrift :many +SELECT + n.id, + n.name, + n.node_type, + n.device_clock_drift_seconds, + n.last_advert_at, + json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) FILTER (WHERE ni.iata IS NOT NULL) AS iatas +FROM nodes n +LEFT JOIN node_iatas ni ON ni.node_id = n.id +WHERE n.node_type IN (2, 3) + AND n.device_clock_drift_seconds IS NOT NULL + AND ABS(n.device_clock_drift_seconds) > $1::int + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) +GROUP BY n.id, n.name, n.node_type, n.device_clock_drift_seconds, n.last_advert_at +ORDER BY ABS(n.device_clock_drift_seconds) DESC +LIMIT $3 +` + +type GetStatsClockDriftParams struct { + Column1 int32 `json:"column_1"` + Column2 []string `json:"column_2"` + Limit int32 `json:"limit"` +} + +type GetStatsClockDriftRow struct { + ID uuid.UUID `json:"id"` + Name *string `json:"name"` + NodeType int16 `json:"node_type"` + DeviceClockDriftSeconds *int32 `json:"device_clock_drift_seconds"` + LastAdvertAt pgtype.Timestamptz `json:"last_advert_at"` + Iatas []byte `json:"iatas"` +} + +// Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds +// the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's +// latest measured drift, not an aggregate over a period. +func (q *Queries) GetStatsClockDrift(ctx context.Context, arg GetStatsClockDriftParams) ([]GetStatsClockDriftRow, error) { + rows, err := q.db.Query(ctx, getStatsClockDrift, arg.Column1, arg.Column2, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetStatsClockDriftRow{} + for rows.Next() { + var i GetStatsClockDriftRow + if err := rows.Scan( + &i.ID, + &i.Name, + &i.NodeType, + &i.DeviceClockDriftSeconds, + &i.LastAdvertAt, + &i.Iatas, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getStatsNodeTypes = `-- name: GetStatsNodeTypes :many SELECT n.node_type, diff --git a/db/stats.go b/db/stats.go index 75f93113..ffbcabb6 100644 --- a/db/stats.go +++ b/db/stats.go @@ -5,6 +5,8 @@ package db import ( "context" + "encoding/json" + "log" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -160,6 +162,39 @@ func (s *Store) GetStatsTopAdvertisers(ctx context.Context, iatas []string, sinc return items, nil } +// GetStatsClockDrift returns repeaters/room servers whose current advert-derived clock +// drift exceeds the Store's configured threshold, worst first. +func (s *Store) GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { + thresholdSeconds := int32(s.clockDriftThreshold / time.Second) + rows, err := s.q.GetStatsClockDrift(ctx, sqlc.GetStatsClockDriftParams{ + Column1: thresholdSeconds, + Column2: iatas, + Limit: limit, + }) + if err != nil { + return nil, err + } + items := make([]api.ClockDriftEntry, 0, len(rows)) + for _, v := range rows { + entry := api.ClockDriftEntry{ + NodeID: v.ID, + NodeName: v.Name, + NodeType: v.NodeType, + NodeTypeName: api.NodeTypeName(v.NodeType), + ClockDriftSeconds: int(*v.DeviceClockDriftSeconds), + ClockCheckedAt: v.LastAdvertAt.Time.UnixMilli(), + } + if len(v.Iatas) > 0 { + if err := json.Unmarshal(v.Iatas, &entry.IATAs); err != nil { + log.Printf("store: failed to unmarshal clock drift node iatas: %v", err) + entry.IATAs = []api.NodeIATA{} + } + } + items = append(items, entry) + } + return items, nil +} + func (s *Store) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { if since.IsZero() { since = time.Now().Add(-24 * time.Hour) diff --git a/db/stats_test.go b/db/stats_test.go index a7c56fba..70518705 100644 --- a/db/stats_test.go +++ b/db/stats_test.go @@ -159,6 +159,45 @@ func TestGetStatsTopAdvertisers_FloodDirectSplit(t *testing.T) { } } +func TestGetStatsClockDrift_Mapping(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000004") + name := "drifty-repeater" + drift := int32(-600) + checkedAt := pgtype.Timestamptz{Time: time.Now(), Valid: true} + iatasJSON := []byte(`[{"iata":"YVR","lastHeard":1700000000000}]`) + + mock.EXPECT(). + GetStatsClockDrift(gomock.Any(), gomock.Any()). + Return([]sqlc.GetStatsClockDriftRow{ + { + ID: nodeID, + Name: &name, + NodeType: 2, // repeater + DeviceClockDriftSeconds: &drift, + LastAdvertAt: checkedAt, + Iatas: iatasJSON, + }, + }, nil) + + store := &Store{q: mock, clockDriftThreshold: 5 * time.Minute} + items, err := store.GetStatsClockDrift(context.Background(), []string{"YVR"}, 10) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + if items[0].ClockDriftSeconds != -600 { + t.Errorf("expected ClockDriftSeconds -600, got %d", items[0].ClockDriftSeconds) + } + if len(items[0].IATAs) != 1 || items[0].IATAs[0].IATA != "YVR" { + t.Errorf("expected 1 IATA entry for YVR, got %v", items[0].IATAs) + } +} + func TestGetStatsNodeTypes_Mapping(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/docs/docs.go b/docs/docs.go index 9a6fdef3..bde75e92 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1583,6 +1583,60 @@ const docTemplate = `{ } } }, + "/stats/clock-drift": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Repeaters/room servers whose clock has drifted beyond the configured threshold, worst first", + "parameters": [ + { + "type": "string", + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Max results (default 10)", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/node-types": { "get": { "produces": [ @@ -2367,6 +2421,38 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry": { + "type": "object", + "properties": { + "clockCheckedAt": { + "description": "epoch ms", + "type": "integer" + }, + "clockDriftSeconds": { + "description": "signed; +ve = device ahead of server", + "type": "integer" + }, + "iatas": { + "description": "IATAs this node has been heard in", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeIATA" + } + }, + "nodeId": { + "type": "string" + }, + "nodeName": { + "type": "string" + }, + "nodeType": { + "type": "integer" + }, + "nodeTypeName": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 325b8ff2..67f6bd23 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1581,6 +1581,60 @@ } } }, + "/stats/clock-drift": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Repeaters/room servers whose clock has drifted beyond the configured threshold, worst first", + "parameters": [ + { + "type": "string", + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs", + "name": "region", + "in": "query" + }, + { + "type": "integer", + "description": "Max results (default 10)", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/node-types": { "get": { "produces": [ @@ -2365,6 +2419,38 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry": { + "type": "object", + "properties": { + "clockCheckedAt": { + "description": "epoch ms", + "type": "integer" + }, + "clockDriftSeconds": { + "description": "signed; +ve = device ahead of server", + "type": "integer" + }, + "iatas": { + "description": "IATAs this node has been heard in", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeIATA" + } + }, + "nodeId": { + "type": "string" + }, + "nodeName": { + "type": "string" + }, + "nodeType": { + "type": "integer" + }, + "nodeTypeName": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 456ee23c..74fff865 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -101,6 +101,28 @@ definitions: description: display name from config or nil type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry: + properties: + clockCheckedAt: + description: epoch ms + type: integer + clockDriftSeconds: + description: signed; +ve = device ahead of server + type: integer + iatas: + description: IATAs this node has been heard in + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.NodeIATA' + type: array + nodeId: + type: string + nodeName: + type: string + nodeType: + type: integer + nodeTypeName: + type: string + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop: properties: fromIata: @@ -2130,6 +2152,42 @@ paths: summary: Get scope detail by name tags: - Scopes + /stats/clock-drift: + get: + parameters: + - description: Comma-separated IATA codes + in: query + name: iatas + type: string + - description: Filter by region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Filter by region slug, expands to member IATAs + in: query + name: region + type: string + - description: Max results (default 10) + in: query + name: limit + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ClockDriftEntry' + type: array + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Repeaters/room servers whose clock has drifted beyond the configured + threshold, worst first + tags: + - Stats /stats/node-types: get: parameters: diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index d9bc8177..474c9246 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -34,6 +34,7 @@ func StatsRouter(reader api.Reader) http.Handler { r.Get("/top-nodes", getStatsTopNodes(reader)) r.Get("/top-observers", getStatsTopObservers(reader)) r.Get("/top-advertisers", getStatsTopAdvertisers(reader)) + r.Get("/clock-drift", getStatsClockDrift(reader)) r.Get("/top-talkers", getStatsTopTalkers(reader)) r.Get("/radio-presets", getStatsRadioPresets(reader)) r.Get("/scopes", getStatsScopes(reader)) @@ -303,6 +304,48 @@ func getStatsTopAdvertisers(reader api.Reader) http.HandlerFunc { } } +// getStatsClockDrift godoc +// +// @Summary Repeaters/room servers whose clock has drifted beyond the configured threshold, worst first +// @Tags Stats +// @Produce json +// @Param iatas query string false "Comma-separated IATA codes" +// @Param regionId query int false "Filter by region ID, expands to member IATAs" +// @Param region query string false "Filter by region slug, expands to member IATAs" +// @Param limit query int false "Max results (default 10)" +// @Success 200 {array} api.ClockDriftEntry +// @Failure 500 {object} handlers.APIError +// @Router /stats/clock-drift [get] +func getStatsClockDrift(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + iatas := parseIATAs(r) + if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + iatas = append(iatas, regionIATAs...) + } + var limit int32 = 10 + if p := r.URL.Query().Get("limit"); p != "" { + l, err := strconv.ParseInt(p, 10, 32) + if err != nil { + respondError(w, http.StatusBadRequest, "limit must be an integer") + return + } + limit = int32(l) + } + entries, err := reader.GetStatsClockDrift(r.Context(), iatas, limit) + if err != nil { + log.Printf("api: GetStatsClockDrift failed: %v", err) + respondError(w, http.StatusInternalServerError, "internal server error") + return + } + respond(w, http.StatusOK, entries) + } +} + // getStatsTopTalkers godoc // // @Summary Top N companion names by decrypted channel message count (last 24h by default) diff --git a/internal/api/handlers/stats_test.go b/internal/api/handlers/stats_test.go index d4e572ea..fe08c8bf 100644 --- a/internal/api/handlers/stats_test.go +++ b/internal/api/handlers/stats_test.go @@ -181,6 +181,52 @@ func TestGetStatsTopAdvertisers_OK(t *testing.T) { } } +func TestGetStatsClockDrift_OK(t *testing.T) { + r := chi.NewRouter() + r.Get("/stats/clock-drift", getStatsClockDrift(stubReader{ + getStatsClockDrift: func(_ context.Context, _ []string, _ int32) ([]api.ClockDriftEntry, error) { + return []api.ClockDriftEntry{{ClockDriftSeconds: -600}}, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/stats/clock-drift", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected 200, got %d", w.Code) + } +} + +func TestGetStatsClockDrift_MultiIATA_PassedThrough(t *testing.T) { + var gotIATAs []string + r := chi.NewRouter() + r.Get("/stats/clock-drift", getStatsClockDrift(stubReader{ + getStatsClockDrift: func(_ context.Context, iatas []string, _ int32) ([]api.ClockDriftEntry, error) { + gotIATAs = iatas + return nil, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/stats/clock-drift?iatas=YVR,YYJ", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("expected 200, got %d", w.Code) + } + if len(gotIATAs) != 2 || gotIATAs[0] != "YVR" || gotIATAs[1] != "YYJ" { + t.Errorf("expected [YVR YYJ], got %v", gotIATAs) + } +} + +func TestGetStatsClockDrift_InvalidLimit(t *testing.T) { + r := chi.NewRouter() + r.Get("/stats/clock-drift", getStatsClockDrift(stubReader{})) + req := httptest.NewRequest(http.MethodGet, "/stats/clock-drift?limit=notanint", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Errorf("expected 400, got %d", w.Code) + } +} + func TestGetStatsTopTalkers_InvalidSince(t *testing.T) { r := chi.NewRouter() r.Get("/stats/top-talkers", getStatsTopTalkers(stubReader{})) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index ba720b30..f53cb0a0 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -47,6 +47,7 @@ type stubReader struct { getStatsTopNodes func(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) getStatsTopObservers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) getStatsTopAdvertisers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) + getStatsClockDrift func(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) getStatsTopTalkers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) getScopeStats func(ctx context.Context) ([]api.ScopeStats, error) getStatsNodeTypes func(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) @@ -280,6 +281,13 @@ func (s stubReader) GetStatsTopAdvertisers(ctx context.Context, iatas []string, return nil, nil } +func (s stubReader) GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { + if s.getStatsClockDrift != nil { + return s.getStatsClockDrift(ctx, iatas, limit) + } + return nil, nil +} + func (s stubReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { if s.getStatsTopTalkers != nil { return s.getStatsTopTalkers(ctx, iatas, since, limit) diff --git a/internal/api/reader.go b/internal/api/reader.go index 8d8cc2fe..964b04b2 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -169,6 +169,12 @@ type Reader interface { // since defines the start of the window; pass zero time for default (last 24h). GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]TopAdvertiser, error) + // GetStatsClockDrift returns repeaters/room servers whose most recent advert-derived + // clock drift exceeds the configured threshold (nodes.clock_drift_threshold), worst + // drift first. Pass nil for iatas to return across all IATAs. Unlike the other stats + // endpoints this isn't time-windowed -- it reflects each node's current drift state. + GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]ClockDriftEntry, error) + // GetStatsTopTalkers returns the top N companion names by decrypted channel message count. // Pass nil for iatas to return stats across all IATAs. // since defines the start of the window; pass zero time for default (last 24h). diff --git a/internal/api/stats.go b/internal/api/stats.go index 7f51a21e..09e954a9 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -100,3 +100,18 @@ type NodeTypeCount struct { NodeTypeName string `json:"nodeTypeName"` Count int64 `json:"count"` } + +// ClockDriftEntry is a repeater or room server whose most recent advert-derived clock drift +// exceeds the configured threshold (nodes.clock_drift_threshold, default 5m) -- see +// GetStatsClockDrift. Ordered worst-drift-first. ClockDriftSeconds/ClockCheckedAt mirror the +// same-named fields on Node; unlike Node this list only ever contains out-of-sync nodes, so +// there's no ClockOutOfSync bool here -- being in the list already means true. +type ClockDriftEntry struct { + NodeID uuid.UUID `json:"nodeId"` + NodeName *string `json:"nodeName,omitempty"` + NodeType int16 `json:"nodeType"` + NodeTypeName string `json:"nodeTypeName"` + ClockDriftSeconds int `json:"clockDriftSeconds"` // signed; +ve = device ahead of server + ClockCheckedAt int64 `json:"clockCheckedAt"` // epoch ms + IATAs []NodeIATA `json:"iatas,omitempty"` // IATAs this node has been heard in +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 42463c94..4531808e 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -81,6 +81,10 @@ func (s *stubReader) GetStatsTopAdvertisers(_ context.Context, _ []string, _ tim return nil, nil } +func (s *stubReader) GetStatsClockDrift(_ context.Context, _ []string, _ int32) ([]api.ClockDriftEntry, error) { + return nil, nil +} + func (s *stubReader) GetStatsTopTalkers(_ context.Context, _ []string, _ time.Time, _ int32) ([]api.TopTalker, error) { return nil, nil } diff --git a/internal/cache/reader.go b/internal/cache/reader.go index afd45888..6b234ccd 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -32,6 +32,7 @@ const ( keyStatsTopNodesPrefix = "beacon:stats:top-nodes:" keyStatsTopObsPrefix = "beacon:stats:top-observers:" keyStatsTopAdvPrefix = "beacon:stats:top-advertisers:" + keyStatsClockDriftPrefix = "beacon:stats:clock-drift:" keyStatsTopTalkersPrefix = "beacon:stats:top-talkers:" keyStatsNodeTypes = "beacon:stats:node-types:" keyRadioPresetsPrefix = "beacon:radio-presets:" @@ -258,6 +259,20 @@ func (cr *CachedReader) GetStatsTopAdvertisers(ctx context.Context, iatas []stri }) } +// GetStatsClockDrift implements [api.Reader]. +func (cr *CachedReader) GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { + segment := "all" + if len(iatas) > 0 { + sorted := append([]string(nil), iatas...) + sort.Strings(sorted) + segment = strings.Join(sorted, ",") + } + key := fmt.Sprintf("%s%s:%d", keyStatsClockDriftPrefix, segment, limit) + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.ClockDriftEntry, error) { + return cr.inner.GetStatsClockDrift(ctx, iatas, limit) + }) +} + // GetStatsTopTalkers implements [api.Reader]. func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { segment := "all" From ae0669ccbedee746d8614417d3cae56e84e0e594 Mon Sep 17 00:00:00 2001 From: "Enot (ded) Skelly" Date: Mon, 27 Jul 2026 09:41:11 -0700 Subject: [PATCH 004/141] enrich packet observations --- db/packets.go | 37 ++++++++++++++ db/packets_test.go | 92 +++++++++++++++++++++++++++++++++ db/queries/queries.sql | 20 ++++++-- db/sqlc/queries.sql.go | 110 ++++++++++++++++++++++++++-------------- docs/docs.go | 24 +++++++++ docs/swagger.json | 24 +++++++++ docs/swagger.yaml | 22 ++++++++ internal/api/packets.go | 12 +++++ 8 files changed, 300 insertions(+), 41 deletions(-) diff --git a/db/packets.go b/db/packets.go index 68bd34cc..27766256 100644 --- a/db/packets.go +++ b/db/packets.go @@ -58,6 +58,32 @@ func (s *Store) SetPacketDecrypted(ctx context.Context, hash []byte) error { return s.q.SetPacketDecrypted(ctx, hash) } +// buildLatestObserverPath fills in PacketLatestObserver's optional path fields from the +// nullable path_length_byte/hash_size/hop_count/path_bytes columns joined in alongside the +// latest (or matching) observation. hashSize/hopCount nil means no observation joined at all +// (only possible via ListPackets/ListPacketsByIATAs' LEFT JOIN LATERAL -- ListPacketsAfterID's +// inner join always has them, callers there can pass &v.Field directly). +func buildLatestObserverPath(pathLengthByte, hashSize, hopCount *int16, pathBytes []byte) (*api.PacketPathLength, *string) { + if hashSize == nil || hopCount == nil { + return nil, nil + } + raw := "" + if pathLengthByte != nil { + raw = fmt.Sprintf("%02x", *pathLengthByte) + } + pathLength := &api.PacketPathLength{ + Raw: raw, + HashSize: *hashSize, + HopCount: *hopCount, + } + var pathBytesHex *string + if pathBytes != nil { + s := hex.EncodeToString(pathBytes) + pathBytesHex = &s + } + return pathLength, pathBytesHex +} + func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int16, iatas []string, scopes []string, since, until time.Time, cursor int64, limit int32) (api.Page[api.PacketSummary], error) { if len(iatas) > 0 { return s.listPacketsByIATAs(ctx, payloadTypes, routeTypes, iatas, scopes, since, until, cursor, limit) @@ -109,6 +135,9 @@ func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int1 DisplayName: v.LatestObserverName, IATA: v.LatestObserverIata, } + item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( + &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, + ) } items = append(items, item) } @@ -177,6 +206,9 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes DisplayName: v.LatestObserverName, IATA: v.LatestObserverIata, } + item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( + &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, + ) } items = append(items, item) } @@ -224,6 +256,11 @@ func (s *Store) ListPacketsAfterID(ctx context.Context, afterObservationID int64 DisplayName: v.LatestObserverName, IATA: v.LatestObserverIata, } + // Inner join here (unlike ListPackets/listPacketsByIATAs' LEFT JOIN LATERAL), so + // these are never nil when an observer was joined at all. + item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( + &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, + ) } items = append(items, item) } diff --git a/db/packets_test.go b/db/packets_test.go index ba228c59..9a4a9cba 100644 --- a/db/packets_test.go +++ b/db/packets_test.go @@ -157,6 +157,59 @@ func TestListPackets_LatestObserverSet(t *testing.T) { } } +func TestListPackets_LatestObserverPathFields(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + heardAt := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + pathLengthByte := int16(0x42) + hashSize := int16(1) + hopCount := int16(2) + pathBytes := []byte{0xa1, 0xb2} + + mock.EXPECT(). + ListPackets(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsRow{ + { + PacketHash: []byte{0xde, 0xad}, + FirstHeardAt: heardAt, + LastHeardAt: heardAt, + LatestObserverID: observerID, + LatestObserverPathLengthByte: pathLengthByte, + LatestObserverHashSize: hashSize, + LatestObserverHopCount: hopCount, + LatestObserverPathBytes: pathBytes, + }, + }, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 10) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + obs := page.Items[0].LatestObserver + if obs == nil { + t.Fatal("expected LatestObserver to be set") + } + if obs.PathLength == nil { + t.Fatal("expected PathLength to be set") + } + if obs.PathLength.HashSize != 1 || obs.PathLength.HopCount != 2 { + t.Errorf("expected hashSize=1 hopCount=2, got hashSize=%d hopCount=%d", obs.PathLength.HashSize, obs.PathLength.HopCount) + } + if obs.PathLength.Raw != "42" { + t.Errorf("expected raw 42, got %s", obs.PathLength.Raw) + } + if obs.PathBytes == nil || *obs.PathBytes != "a1b2" { + t.Errorf("expected pathBytes a1b2, got %v", obs.PathBytes) + } + // Resolution stays a detail-view-only feature on this list endpoint -- deliberately unset. + if obs.ResolvedPath != nil || obs.ResolvedSource != nil || obs.ResolvedDestination != nil { + t.Error("expected no resolved path/source/destination on the list endpoint") + } +} + func TestInsertObservation_Success(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) @@ -352,6 +405,45 @@ func TestListPacketsAfterID_PassesIATAsAsArray(t *testing.T) { } } +func TestListPacketsAfterID_LatestObserverPathFields(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + heardAt := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT(). + ListPacketsAfterID(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsAfterIDRow{ + { + PacketHash: []byte{0xde, 0xad}, + FirstHeardAt: heardAt, + LastHeardAt: heardAt, + LatestObserverID: observerID, + LatestObserverPathLengthByte: 0x42, + LatestObserverHashSize: 1, + LatestObserverHopCount: 2, + LatestObserverPathBytes: []byte{0xa1, 0xb2}, + }, + }, nil) + + store := &Store{q: mock} + items, err := store.ListPacketsAfterID(context.Background(), 0, -1, -1, nil, "", 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + obs := items[0].LatestObserver + if obs == nil || obs.PathLength == nil { + t.Fatal("expected LatestObserver and PathLength to be set") + } + if obs.PathLength.HashSize != 1 || obs.PathLength.HopCount != 2 { + t.Errorf("expected hashSize=1 hopCount=2, got hashSize=%d hopCount=%d", obs.PathLength.HashSize, obs.PathLength.HopCount) + } + if obs.PathBytes == nil || *obs.PathBytes != "a1b2" { + t.Errorf("expected pathBytes a1b2, got %v", obs.PathBytes) + } +} + func TestListNodeObservations_Pagination(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 4fb5410d..6d716ec2 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -393,10 +393,14 @@ SELECT (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes FROM packets p LEFT JOIN LATERAL ( - SELECT observer_id, iata + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -436,7 +440,11 @@ SELECT (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes FROM ( SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at FROM unnest(@iatas::bpchar[]) AS req(iata) @@ -467,7 +475,7 @@ FROM ( ) sh JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( - SELECT observer_id, iata + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -490,6 +498,10 @@ SELECT po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, ts.name AS scope_name FROM packets p JOIN packet_observations po ON po.packet_hash = p.packet_hash diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 0b536a45..02783b46 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2650,10 +2650,14 @@ SELECT (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes FROM packets p LEFT JOIN LATERAL ( - SELECT observer_id, iata + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -2683,17 +2687,21 @@ type ListPacketsParams struct { } type ListPacketsRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` } // Returns packets with the latest observation rolled in for display. @@ -2728,6 +2736,10 @@ func (q *Queries) ListPackets(ctx context.Context, arg ListPacketsParams) ([]Lis &i.LatestObserverID, &i.LatestObserverName, &i.LatestObserverIata, + &i.LatestObserverPathLengthByte, + &i.LatestObserverHashSize, + &i.LatestObserverHopCount, + &i.LatestObserverPathBytes, ); err != nil { return nil, err } @@ -2750,6 +2762,10 @@ SELECT po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes, ts.name AS scope_name FROM packets p JOIN packet_observations po ON po.packet_hash = p.packet_hash @@ -2774,16 +2790,20 @@ type ListPacketsAfterIDParams struct { } type ListPacketsAfterIDRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` - ScopeName *string `json:"scope_name"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + ScopeName *string `json:"scope_name"` } // Returns packets with observations after the given observation ID, ordered oldest first. @@ -2814,6 +2834,10 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID &i.LatestObserverID, &i.LatestObserverName, &i.LatestObserverIata, + &i.LatestObserverPathLengthByte, + &i.LatestObserverHashSize, + &i.LatestObserverHopCount, + &i.LatestObserverPathBytes, &i.ScopeName, ); err != nil { return nil, err @@ -2839,7 +2863,11 @@ SELECT (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes FROM ( SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at FROM unnest($1::bpchar[]) AS req(iata) @@ -2870,7 +2898,7 @@ FROM ( ) sh JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( - SELECT observer_id, iata + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -2894,18 +2922,22 @@ type ListPacketsByIATAsParams struct { } type ListPacketsByIATAsRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` } // IATA-filtered packet list, driven from idx_observations_iata_heard. @@ -2949,6 +2981,10 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs &i.LatestObserverID, &i.LatestObserverName, &i.LatestObserverIata, + &i.LatestObserverPathLengthByte, + &i.LatestObserverHashSize, + &i.LatestObserverHopCount, + &i.LatestObserverPathBytes, ); err != nil { return nil, err } diff --git a/docs/docs.go b/docs/docs.go index bde75e92..f8d9256c 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -3139,6 +3139,30 @@ const docTemplate = `{ }, "id": { "type": "string" + }, + "pathBytes": { + "description": "hex-encoded accumulated path hashes", + "type": "string" + }, + "pathLength": { + "description": "PathLength/PathBytes are cheap -- already-stored columns on packet_observations -- and\npopulated everywhere PacketLatestObserver appears: the REST list/backfill endpoints and\nthe WS feed alike. ResolvedPath/ResolvedSource/ResolvedDestination require a per-hash DB\nresolution lookup; they're populated on the WS feed (already computed once at ingest, so\neffectively free there) but deliberately left nil on the REST endpoints, which are\npaginated/high-volume and used only for scrollback and reconnect-gap backfill -- full\nresolution stays a GET /packets/{packetHash}-only feature.", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketPathLength" + } + ] + }, + "resolvedDestination": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" + }, + "resolvedPath": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" + } + }, + "resolvedSource": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" } } }, diff --git a/docs/swagger.json b/docs/swagger.json index 67f6bd23..e9eaf319 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -3137,6 +3137,30 @@ }, "id": { "type": "string" + }, + "pathBytes": { + "description": "hex-encoded accumulated path hashes", + "type": "string" + }, + "pathLength": { + "description": "PathLength/PathBytes are cheap -- already-stored columns on packet_observations -- and\npopulated everywhere PacketLatestObserver appears: the REST list/backfill endpoints and\nthe WS feed alike. ResolvedPath/ResolvedSource/ResolvedDestination require a per-hash DB\nresolution lookup; they're populated on the WS feed (already computed once at ingest, so\neffectively free there) but deliberately left nil on the REST endpoints, which are\npaginated/high-volume and used only for scrollback and reconnect-gap backfill -- full\nresolution stays a GET /packets/{packetHash}-only feature.", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketPathLength" + } + ] + }, + "resolvedDestination": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" + }, + "resolvedPath": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" + } + }, + "resolvedSource": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" } } }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 74fff865..3e0a74c6 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -612,6 +612,28 @@ definitions: type: string id: type: string + pathBytes: + description: hex-encoded accumulated path hashes + type: string + pathLength: + allOf: + - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketPathLength' + description: |- + PathLength/PathBytes are cheap -- already-stored columns on packet_observations -- and + populated everywhere PacketLatestObserver appears: the REST list/backfill endpoints and + the WS feed alike. ResolvedPath/ResolvedSource/ResolvedDestination require a per-hash DB + resolution lookup; they're populated on the WS feed (already computed once at ingest, so + effectively free there) but deliberately left nil on the REST endpoints, which are + paginated/high-volume and used only for scrollback and reconnect-gap backfill -- full + resolution stays a GET /packets/{packetHash}-only feature. + resolvedDestination: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop' + resolvedPath: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop' + type: array + resolvedSource: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop' type: object github_com_MeshCore-Beacon_beacon-server_internal_api.PacketObservationDetail: properties: diff --git a/internal/api/packets.go b/internal/api/packets.go index 516be02c..305fcfb9 100644 --- a/internal/api/packets.go +++ b/internal/api/packets.go @@ -17,6 +17,18 @@ type PacketLatestObserver struct { ID uuid.UUID `json:"id"` DisplayName *string `json:"displayName,omitempty"` IATA string `json:"iata"` + // PathLength/PathBytes are cheap -- already-stored columns on packet_observations -- and + // populated everywhere PacketLatestObserver appears: the REST list/backfill endpoints and + // the WS feed alike. ResolvedPath/ResolvedSource/ResolvedDestination require a per-hash DB + // resolution lookup; they're populated on the WS feed (already computed once at ingest, so + // effectively free there) but deliberately left nil on the REST endpoints, which are + // paginated/high-volume and used only for scrollback and reconnect-gap backfill -- full + // resolution stays a GET /packets/{packetHash}-only feature. + PathLength *PacketPathLength `json:"pathLength,omitempty"` + PathBytes *string `json:"pathBytes,omitempty"` // hex-encoded accumulated path hashes + ResolvedPath []ResolvedHop `json:"resolvedPath,omitempty"` + ResolvedSource *ResolvedHop `json:"resolvedSource,omitempty"` + ResolvedDestination *ResolvedHop `json:"resolvedDestination,omitempty"` } // PacketSummary is the minimal packet representation used in list responses. From 2cbd211e118df685dfacb5bb3faec01fcdfec070 Mon Sep 17 00:00:00 2001 From: "Enot (ded) Skelly" Date: Thu, 30 Jul 2026 13:00:29 -0700 Subject: [PATCH 005/141] add IATA validation checks on ingest closes #89 --- internal/ingest/ingest.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index ff907a33..1f41f905 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -277,6 +277,21 @@ func (w *Worker) subscribe(client mqtt.Client) { } } +// isValidIATA reports whether s is 3 uppercase ASCII letters, matching the +// iata_codes.iata CHAR(3) column. MQTT topic segments are attacker/observer +// controlled and must be validated before touching the DB. +func isValidIATA(s string) bool { + if len(s) != 3 { + return false + } + for i := 0; i < 3; i++ { + if s[i] < 'A' || s[i] > 'Z' { + return false + } + } + return true +} + // handleMessage dispatches incoming MQTT messages by subtopic. // Each message is processed with a 30s timeout to prevent slow DB calls // from blocking the MQTT receive goroutine indefinitely. @@ -288,6 +303,13 @@ func (w *Worker) handleMessage(msg mqtt.Message) { } iata, pubkeyHex, subtopic := parts[1], parts[2], parts[3] + // iata_codes.iata is CHAR(3); anything else would fail the DB insert + // downstream, so reject malformed topic segments here instead. + if !isValidIATA(iata) { + log.Printf("ingest[%s]: dropped packet with malformed IATA %q on topic %s", w.cfg.BrokerName, iata, msg.Topic()) + return + } + // Drop packets from IATAs outside the configured geographic filter. if w.cfg.AllowedIATAs != nil { if _, ok := w.cfg.AllowedIATAs[iata]; !ok { From b44cae89d7401330e0d44cb3e3d61d4705b69d65 Mon Sep 17 00:00:00 2001 From: "Enot (ded) Skelly" Date: Thu, 30 Jul 2026 13:19:18 -0700 Subject: [PATCH 006/141] add neighbor message parsing closes #94 --- db/migrations/022_neighbor_scopes.sql | 20 +++++ db/nodes.go | 11 +-- db/observers.go | 11 +++ db/queries/queries.sql | 21 ++++-- db/sqlc/mock/querier.go | 14 ++++ db/sqlc/models.go | 2 + db/sqlc/querier.go | 12 ++- db/sqlc/queries.sql.go | 51 +++++++++---- internal/ingest/ingest.go | 22 +++++- internal/ingest/ingest_test.go | 6 +- internal/ingest/neighbors.go | 104 ++++++++++++++++++++++++++ internal/ingest/packet.go | 4 +- internal/ingest/side_effects.go | 4 +- 13 files changed, 249 insertions(+), 33 deletions(-) create mode 100644 db/migrations/022_neighbor_scopes.sql create mode 100644 internal/ingest/neighbors.go diff --git a/db/migrations/022_neighbor_scopes.sql b/db/migrations/022_neighbor_scopes.sql new file mode 100644 index 00000000..2a533826 --- /dev/null +++ b/db/migrations/022_neighbor_scopes.sql @@ -0,0 +1,20 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Supports the observer /neighbors MQTT topic: a periodic report listing the +-- OTA-configured region scope of the observer itself ("self") and of each +-- zero-hop neighbor the observer was able to query. +-- +-- NOTE: these are plaintext OTA "region scope" strings (comma-separated, +-- e.g. "*" or "US,CA") read from device config over the air. They are +-- unrelated to the transport_scopes / observer_scopes / default_scope_id +-- machinery elsewhere in this schema, which matches encrypted transport +-- packets to named routing scopes by key fingerprint. Deliberately named +-- differently (region_scope, not scope) to avoid confusion between the two. + +ALTER TABLE observers ADD COLUMN region_scope TEXT; + +-- Per-neighbor-edge, not per-node: the OTA scope query is answered by +-- whichever node reports it, in the context of a specific observer's +-- zero-hop neighbor list, so it lives alongside snr on node_neighbors. +ALTER TABLE node_neighbors ADD COLUMN region_scope TEXT; diff --git a/db/nodes.go b/db/nodes.go index 30bea8ab..a3a10744 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -61,12 +61,13 @@ func (s *Store) UpsertNodeShortID(ctx context.Context, nodeID uuid.UUID, iata st }) } -func (s *Store) UpsertNodeNeighbor(ctx context.Context, nodeID, neighborID uuid.UUID, iata string, snr *float32) error { +func (s *Store) UpsertNodeNeighbor(ctx context.Context, nodeID, neighborID uuid.UUID, iata string, snr *float32, regionScope *string) error { return s.q.UpsertNodeNeighbor(ctx, sqlc.UpsertNodeNeighborParams{ - NodeID: nodeID, - NeighborID: neighborID, - Iata: iata, - Snr: snr, + NodeID: nodeID, + NeighborID: neighborID, + Iata: iata, + Snr: snr, + RegionScope: regionScope, }) } diff --git a/db/observers.go b/db/observers.go index 5ff37060..a58569b5 100644 --- a/db/observers.go +++ b/db/observers.go @@ -315,6 +315,17 @@ func (s *Store) UpsertObserverScope(ctx context.Context, observerID uuid.UUID, s }) } +// UpdateObserverRegionScope records the observer's own OTA-reported region scope +// (the "self" field of a /neighbors report). Unrelated to UpsertObserverScope +// above, which links an observer to a named transport_scopes row matched by +// key fingerprint; this stores the raw OTA-configured scope string instead. +func (s *Store) UpdateObserverRegionScope(ctx context.Context, observerID uuid.UUID, regionScope string) error { + return s.q.UpdateObserverRegionScope(ctx, sqlc.UpdateObserverRegionScopeParams{ + ID: observerID, + RegionScope: ®ionScope, + }) +} + func (s *Store) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) { return s.q.GetObserverScopes(ctx, observerID) } diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 6d716ec2..0938f1eb 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1114,15 +1114,24 @@ ORDER BY hop_count ASC, last_seen DESC; -- Records or updates a neighbor relationship between two nodes observed in the same IATA. -- node_id is the advertising node, neighbor_id is the first-hop forwarder. -- snr is optional; pass NULL when no signal reading is available (the --- common case). On conflict, snr is only overwritten when a new non-null --- value is supplied, so a later no-SNR observation doesn't erase an --- earlier real reading. -INSERT INTO node_neighbors (node_id, neighbor_id, iata, observation_count, snr) -VALUES ($1, $2, $3, 1, $4) +-- common case). regionScope is optional too; pass NULL whenever the OTA +-- scope query for this neighbor didn't succeed (status != "responded"), +-- so a failed/timed-out query doesn't erase a previously known scope. +-- On conflict, snr and region_scope are only overwritten when a new +-- non-null value is supplied. +INSERT INTO node_neighbors (node_id, neighbor_id, iata, observation_count, snr, region_scope) +VALUES ($1, $2, $3, 1, $4, $5) ON CONFLICT (node_id, neighbor_id, iata) DO UPDATE SET last_seen = NOW(), observation_count = node_neighbors.observation_count + 1, - snr = COALESCE(EXCLUDED.snr, node_neighbors.snr); + snr = COALESCE(EXCLUDED.snr, node_neighbors.snr), + region_scope = COALESCE(EXCLUDED.region_scope, node_neighbors.region_scope); + +-- name: UpdateObserverRegionScope :exec +-- Records the observer's own OTA-reported region scope, from the "self" +-- field of a /neighbors report. Always known (not queried OTA), so this +-- unconditionally overwrites, unlike the neighbor-side region_scope. +UPDATE observers SET region_scope = $2 WHERE id = $1; -- name: GetNodeNeighbors :many -- Returns the neighbors of a node with details, ordered by most recently seen. diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index d6caa7d9..f5b951b5 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -1311,6 +1311,20 @@ func (mr *MockQuerierMockRecorder) TouchPackets(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TouchPackets", reflect.TypeOf((*MockQuerier)(nil).TouchPackets), ctx, arg) } +// UpdateObserverRegionScope mocks base method. +func (m *MockQuerier) UpdateObserverRegionScope(ctx context.Context, arg db.UpdateObserverRegionScopeParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateObserverRegionScope", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateObserverRegionScope indicates an expected call of UpdateObserverRegionScope. +func (mr *MockQuerierMockRecorder) UpdateObserverRegionScope(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateObserverRegionScope", reflect.TypeOf((*MockQuerier)(nil).UpdateObserverRegionScope), ctx, arg) +} + // UpdateObserverStatus mocks base method. func (m *MockQuerier) UpdateObserverStatus(ctx context.Context, arg db.UpdateObserverStatusParams) (uuid.UUID, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index af80eeb4..b5516333 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -165,6 +165,7 @@ type NodeNeighbor struct { LastSeen pgtype.Timestamptz `json:"last_seen"` ObservationCount int64 `json:"observation_count"` Snr *float32 `json:"snr"` + RegionScope *string `json:"region_scope"` } type NodeShortID struct { @@ -197,6 +198,7 @@ type Observer struct { LastSeen pgtype.Timestamptz `json:"last_seen"` ObservationCount *int64 `json:"observation_count"` Metadata []byte `json:"metadata"` + RegionScope *string `json:"region_scope"` } type ObserverBroker struct { diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index a2f8ded5..c496e8e6 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -200,6 +200,10 @@ type Querier interface { // from regressing a newer write-through (e.g. a status update). TouchObservers(ctx context.Context, arg TouchObserversParams) error TouchPackets(ctx context.Context, arg TouchPacketsParams) error + // Records the observer's own OTA-reported region scope, from the "self" + // field of a /neighbors report. Always known (not queried OTA), so this + // unconditionally overwrites, unlike the neighbor-side region_scope. + UpdateObserverRegionScope(ctx context.Context, arg UpdateObserverRegionScopeParams) error UpdateObserverStatus(ctx context.Context, arg UpdateObserverStatusParams) (uuid.UUID, error) // ============================================================ // CHANNELS @@ -242,9 +246,11 @@ type Querier interface { // Records or updates a neighbor relationship between two nodes observed in the same IATA. // node_id is the advertising node, neighbor_id is the first-hop forwarder. // snr is optional; pass NULL when no signal reading is available (the - // common case). On conflict, snr is only overwritten when a new non-null - // value is supplied, so a later no-SNR observation doesn't erase an - // earlier real reading. + // common case). regionScope is optional too; pass NULL whenever the OTA + // scope query for this neighbor didn't succeed (status != "responded"), + // so a failed/timed-out query doesn't erase a previously known scope. + // On conflict, snr and region_scope are only overwritten when a new + // non-null value is supplied. UpsertNodeNeighbor(ctx context.Context, arg UpsertNodeNeighborParams) error UpsertNodeShortID(ctx context.Context, arg UpsertNodeShortIDParams) error // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 02783b46..7f26f993 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -479,7 +479,7 @@ func (q *Queries) GetObserverBrokers(ctx context.Context, observerID uuid.UUID) } const getObserverByID = `-- name: GetObserverByID :one -SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata FROM observers WHERE id = $1 +SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope FROM observers WHERE id = $1 ` func (q *Queries) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, error) { @@ -506,12 +506,13 @@ func (q *Queries) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, &i.LastSeen, &i.ObservationCount, &i.Metadata, + &i.RegionScope, ) return i, err } const getObserverByPubkey = `-- name: GetObserverByPubkey :one -SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata FROM observers WHERE public_key = $1 +SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope FROM observers WHERE public_key = $1 ` func (q *Queries) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Observer, error) { @@ -538,6 +539,7 @@ func (q *Queries) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Ob &i.LastSeen, &i.ObservationCount, &i.Metadata, + &i.RegionScope, ) return i, err } @@ -3652,6 +3654,23 @@ func (q *Queries) TouchPackets(ctx context.Context, arg TouchPacketsParams) erro return err } +const updateObserverRegionScope = `-- name: UpdateObserverRegionScope :exec +UPDATE observers SET region_scope = $2 WHERE id = $1 +` + +type UpdateObserverRegionScopeParams struct { + ID uuid.UUID `json:"id"` + RegionScope *string `json:"region_scope"` +} + +// Records the observer's own OTA-reported region scope, from the "self" +// field of a /neighbors report. Always known (not queried OTA), so this +// unconditionally overwrites, unlike the neighbor-side region_scope. +func (q *Queries) UpdateObserverRegionScope(ctx context.Context, arg UpdateObserverRegionScopeParams) error { + _, err := q.db.Exec(ctx, updateObserverRegionScope, arg.ID, arg.RegionScope) + return err +} + const updateObserverStatus = `-- name: UpdateObserverStatus :one UPDATE observers SET display_name = COALESCE(NULLIF($2, ''), display_name), @@ -3989,19 +4008,21 @@ func (q *Queries) UpsertNodeIATA(ctx context.Context, arg UpsertNodeIATAParams) const upsertNodeNeighbor = `-- name: UpsertNodeNeighbor :exec -INSERT INTO node_neighbors (node_id, neighbor_id, iata, observation_count, snr) -VALUES ($1, $2, $3, 1, $4) +INSERT INTO node_neighbors (node_id, neighbor_id, iata, observation_count, snr, region_scope) +VALUES ($1, $2, $3, 1, $4, $5) ON CONFLICT (node_id, neighbor_id, iata) DO UPDATE SET last_seen = NOW(), observation_count = node_neighbors.observation_count + 1, - snr = COALESCE(EXCLUDED.snr, node_neighbors.snr) + snr = COALESCE(EXCLUDED.snr, node_neighbors.snr), + region_scope = COALESCE(EXCLUDED.region_scope, node_neighbors.region_scope) ` type UpsertNodeNeighborParams struct { - NodeID uuid.UUID `json:"node_id"` - NeighborID uuid.UUID `json:"neighbor_id"` - Iata string `json:"iata"` - Snr *float32 `json:"snr"` + NodeID uuid.UUID `json:"node_id"` + NeighborID uuid.UUID `json:"neighbor_id"` + Iata string `json:"iata"` + Snr *float32 `json:"snr"` + RegionScope *string `json:"region_scope"` } // ============================================================ @@ -4010,15 +4031,18 @@ type UpsertNodeNeighborParams struct { // Records or updates a neighbor relationship between two nodes observed in the same IATA. // node_id is the advertising node, neighbor_id is the first-hop forwarder. // snr is optional; pass NULL when no signal reading is available (the -// common case). On conflict, snr is only overwritten when a new non-null -// value is supplied, so a later no-SNR observation doesn't erase an -// earlier real reading. +// common case). regionScope is optional too; pass NULL whenever the OTA +// scope query for this neighbor didn't succeed (status != "responded"), +// so a failed/timed-out query doesn't erase a previously known scope. +// On conflict, snr and region_scope are only overwritten when a new +// non-null value is supplied. func (q *Queries) UpsertNodeNeighbor(ctx context.Context, arg UpsertNodeNeighborParams) error { _, err := q.db.Exec(ctx, upsertNodeNeighbor, arg.NodeID, arg.NeighborID, arg.Iata, arg.Snr, + arg.RegionScope, ) return err } @@ -4047,7 +4071,7 @@ VALUES ($1, 'unknown', NOW()) ON CONFLICT (public_key) DO UPDATE SET last_seen = NOW(), observation_count = observers.observation_count + 1 -RETURNING id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata +RETURNING id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope ` // ============================================================ @@ -4077,6 +4101,7 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe &i.LastSeen, &i.ObservationCount, &i.Metadata, + &i.RegionScope, ) return i, err } diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 1f41f905..19b80e28 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -21,6 +21,18 @@ // 1. Parse topic → extract publisher pubkey // 2. Upsert observers row (status_metadata, last_status_at, observer_type, etc.) // 3. Fan out observerStatus event to hub +// +// Pipeline per incoming /neighbors message (sent every 12-368h if mqtt.neighbors +// is enabled on the observer): +// 1. Parse topic → extract IATA + publisher pubkey +// 2. Record the observer's own "self" region scope (always known, unconditional write) +// 3. For each reported zero-hop neighbor: resolve both sides via GetNodeByPubkey +// (skip the entry if either the observer or the neighbor has no node row yet -- +// i.e. hasn't advertised) and upsert a node_neighbors edge. Absence from the +// report means nothing (10KB message cap can truncate a highly-connected node's +// list), so this never deletes existing edges. A neighbor's region_scope is only +// written when status == "responded"; "timeout" (OTA scope queries are flaky) +// leaves any previously known scope untouched. package ingest import ( @@ -178,7 +190,13 @@ type DB interface { // UpsertNodeNeighbor records or updates a neighbor relationship between two nodes. // nodeID is the advertising node, neighborID is the first-hop forwarder. // snr is optional (nil when no signal reading is available, the common case). - UpsertNodeNeighbor(ctx context.Context, nodeID, neighborID uuid.UUID, iata string, snr *float32) error + // regionScope is optional (nil when there's no fresh OTA-queried scope to + // record for this neighbor, e.g. a /neighbors report entry with a failed query). + UpsertNodeNeighbor(ctx context.Context, nodeID, neighborID uuid.UUID, iata string, snr *float32, regionScope *string) error + + // UpdateObserverRegionScope records the observer's own OTA-reported region + // scope, from the "self" field of a /neighbors report. + UpdateObserverRegionScope(ctx context.Context, observerID uuid.UUID, regionScope string) error } // ChannelKeyStore is a read-only view of the channel keys loaded from config. @@ -326,6 +344,8 @@ func (w *Worker) handleMessage(msg mqtt.Message) { w.handlePacket(ctx, iata, pubkeyHex, msg.Payload()) case "status": w.handleStatus(ctx, pubkeyHex, msg.Payload()) + case "neighbors": + w.handleNeighbors(ctx, iata, pubkeyHex, msg.Payload()) // "internal" is intentionally not handled (Role 2 access) } } diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index b01b9cc3..6e11dfef 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -286,7 +286,11 @@ func (s *stubDB) UpsertKnownRoute(_ context.Context, _ []uuid.UUID, _ [][]byte, return nil } -func (s *stubDB) UpsertNodeNeighbor(_ context.Context, _, _ uuid.UUID, _ string, _ *float32) error { +func (s *stubDB) UpsertNodeNeighbor(_ context.Context, _, _ uuid.UUID, _ string, _ *float32, _ *string) error { + return nil +} + +func (s *stubDB) UpdateObserverRegionScope(_ context.Context, _ uuid.UUID, _ string) error { return nil } diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go new file mode 100644 index 00000000..f16e08a3 --- /dev/null +++ b/internal/ingest/neighbors.go @@ -0,0 +1,104 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/hex" + "encoding/json" + "log" +) + +// neighborReportEntry is one entry in the "neighbors" array of a /neighbors report. +type neighborReportEntry struct { + PubKey string `json:"pubkey"` + SNR float32 `json:"snr"` + HeardSecsAgo int64 `json:"heard_secs_ago"` + Scopes string `json:"scopes"` + Status string `json:"status"` // "responded" or "timeout" +} + +// neighborReport is the JSON payload for an observer /neighbors message. +type neighborReport struct { + Self struct { + Scopes string `json:"scopes"` + } `json:"self"` + Neighbors []neighborReportEntry `json:"neighbors"` +} + +// handleNeighbors processes a /neighbors message: the observer's own OTA-configured +// region scope, plus a snapshot of its zero-hop neighbors and (best-effort) their +// OTA-queried region scopes. See the package doc comment for the full pipeline. +func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, raw []byte) { + var report neighborReport + if err := json.Unmarshal(raw, &report); err != nil { + log.Printf("ingest[%s]: malformed neighbors envelope from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + return + } + + pubkey, err := hex.DecodeString(pubkeyHex) + if err != nil { + log.Printf("ingest[%s]: invalid pubkey hex in neighbors from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + return + } + + observerID, _, err := w.db.UpsertObserver(ctx, pubkey) + if err != nil { + log.Printf("ingest[%s]: db: upsert observer failed in neighbors from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + return + } + if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName); err != nil { + log.Printf("ingest[%s]: db: upsert observer broker failed in neighbors from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + } + + // self.scopes is always known (it's the observer's own config, not an OTA + // query), so this is an unconditional write -- unlike the per-neighbor + // scopes below, there's no "query failed" case to protect against here. + if err := w.db.UpdateObserverRegionScope(ctx, observerID, report.Self.Scopes); err != nil { + log.Printf("ingest[%s]: db: update observer region scope failed for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + } + + // The observer's own node row (as opposed to its observers row above) is + // what node_neighbors edges hang off of. As elsewhere in this package + // (see GetNodeByPubkey's doc comment), an observer that hasn't advertised + // yet has no node row, in which case we simply can't record edges for it + // this round -- not an error, just nothing to attach the neighbors to. + observerNodeID, err := w.db.GetNodeByPubkey(ctx, pubkey) + if err != nil { + return + } + + for _, n := range report.Neighbors { + neighborPubkey, err := hex.DecodeString(n.PubKey) + if err != nil { + log.Printf("ingest[%s]: invalid neighbor pubkey hex %q from %s: %v", w.cfg.BrokerName, n.PubKey, pubkeyHex, err) + continue + } + neighborNodeID, err := w.db.GetNodeByPubkey(ctx, neighborPubkey) + if err != nil { + // Neighbor hasn't advertised yet -- nothing to link to. Its absence + // here doesn't mean it's gone; it'll show up once it advertises. + continue + } + if neighborNodeID == observerNodeID { + continue // don't record a node as its own neighbor + } + + snr := n.SNR + + // OTA region scope queries are unreliable (per the firmware author, even + // the mobile app sees this) -- only "responded" is a trustworthy read. + // A "timeout" must not be taken as "the neighbor cleared its scope", so + // we pass nil and let UpsertNodeNeighbor's COALESCE preserve whatever + // was last known. + var regionScope *string + if n.Status == "responded" { + regionScope = &n.Scopes + } + + if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, neighborNodeID, iata, &snr, regionScope); err != nil { + log.Printf("ingest[%s]: db: upsert neighbor failed for %s -> %s: %v", w.cfg.BrokerName, pubkeyHex, n.PubKey, err) + } + } +} diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index eb90db1b..44b5d2f8 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -575,7 +575,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ currEntries := resolved[hashes[i]] if len(prevEntries) == 1 && len(currEntries) == 1 { snr := snrValues[i] - if err := w.db.UpsertNodeNeighbor(ctx, currEntries[0].NodeID, prevEntries[0].NodeID, iata, &snr); err != nil { + if err := w.db.UpsertNodeNeighbor(ctx, currEntries[0].NodeID, prevEntries[0].NodeID, iata, &snr, nil); err != nil { log.Printf("ingest[%s]: failed to upsert trace neighbor: %v", w.cfg.BrokerName, err) } } @@ -655,7 +655,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ // don't insert our own node as a neighbor if oErr == nil && rErr == nil && observerNodeID != responderNodeID { rxSNR := float32(parseNumber(envelope.SNR)) - if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, responderNodeID, iata, &rxSNR); err != nil { + if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, responderNodeID, iata, &rxSNR, nil); err != nil { log.Printf("ingest[%s]: failed to upsert observer-discover neighbor: %v", w.cfg.BrokerName, err) } } diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 9a78ab19..e802f0c2 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -116,7 +116,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc if err == nil { key := hex.EncodeToString(firstHop[0]) if entries := resolved[key]; len(entries) == 1 { - if err := w.db.UpsertNodeNeighbor(ctx, nodeID, entries[0].NodeID, iata, nil); err != nil { + if err := w.db.UpsertNodeNeighbor(ctx, nodeID, entries[0].NodeID, iata, nil, nil); err != nil { log.Printf("ingest[%s]: failed to upsert node neighbor: %v", w.cfg.BrokerName, err) } } @@ -131,7 +131,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc observerNodeID, oErr := w.db.GetNodeByPubkey(ctx, observerPubkey) if oErr == nil && observerNodeID != nodeID { snr := rxSNR - if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, nodeID, iata, &snr); err != nil { + if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, nodeID, iata, &snr, nil); err != nil { log.Printf("ingest[%s]: failed to upsert observer-advert neighbor: %v", w.cfg.BrokerName, err) } } From c6d23a2b5235d5cc28e373bd8f9c51531e15d93e Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 5 Aug 2026 09:57:20 -0400 Subject: [PATCH 007/141] fix: cascade channel_messages on packet delete --- db/migrations/023_channel_messages_cascade.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 db/migrations/023_channel_messages_cascade.sql diff --git a/db/migrations/023_channel_messages_cascade.sql b/db/migrations/023_channel_messages_cascade.sql new file mode 100644 index 00000000..582a8f0c --- /dev/null +++ b/db/migrations/023_channel_messages_cascade.sql @@ -0,0 +1,12 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- The packet_hash FK had no ON DELETE action, so packets carrying a channel +-- message could never age out and DeleteOldPackets aborted. Match packet_observations. + +ALTER TABLE channel_messages + DROP CONSTRAINT IF EXISTS channel_messages_packet_hash_fkey; + +ALTER TABLE channel_messages + ADD CONSTRAINT channel_messages_packet_hash_fkey + FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; From 7d759d0a0fc256c8a4bdabe696609355b776b015 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:30:03 -0400 Subject: [PATCH 008/141] perf: known_routes retention, digest identity, and a working batched reconfirm (#98) * fix: cascade channel_messages on packet delete * rebuild known_routes keyed on (iata, path_key) * add route retention delete, batch reconfirm, path_key upsert * compute route path_key in store, add DeleteOldRoutes * add routes retention config * wire route retention into cleanup, batch reconfirm * prune routes in reconfirm task, not cleanup * expand routes retention docs in example config --- cmd/beacon/main.go | 2 +- config.yaml.example | 19 +++ db/migrations/024_known_routes_pathkey.sql | 43 ++++++ db/queries/queries.sql | 84 +++++++---- db/routes.go | 67 +++++++-- db/routes_test.go | 58 ++++++++ db/sqlc/mock/querier.go | 34 +++-- db/sqlc/models.go | 18 +-- db/sqlc/querier.go | 21 +-- db/sqlc/queries.sql.go | 153 ++++++++++++++++----- internal/background/tasks.go | 18 ++- internal/config/config.go | 61 +++++--- internal/config/config_test.go | 13 ++ 13 files changed, 473 insertions(+), 118 deletions(-) create mode 100644 db/migrations/024_known_routes_pathkey.sql diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 1067408e..fd92c27f 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -258,7 +258,7 @@ func main() { scheduler := background.New([]background.Task{ background.ViewRefreshTask(store, resolved.ViewRefreshInterval), background.CleanupTask(store, resolved.TelemetryRetention, resolved.PacketRetention, resolved.NodeDeleteAfter, resolved.CleanupInterval), - background.ReconfirmTask(store, resolved.ReconfirmInterval), + background.ReconfirmTask(store, resolved.RouteRetention, resolved.RouteGrace, int64(resolved.RouteMinObservations), resolved.ReconfirmInterval), }) go scheduler.Start(ctx) diff --git a/config.yaml.example b/config.yaml.example index ce975415..b039cad2 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -75,6 +75,25 @@ telemetry: packets: retention: 720h # 30 days +# Known-route retention. Routes are distilled packet history (the path a packet +# took, hop by hop), so they may outlive packets.retention; there is no required +# relationship between the two. +routes: + # How long a route is kept after it was last observed. Every new observation + # of the same path resets this clock, so routes the mesh still uses never + # expire -- only paths nothing has traveled for this long age out. + retention: 336h # 14 days + # Shorter window for rarely-seen routes: one observed fewer than + # min_observations times in total is dropped once it goes unobserved for + # this long. Must be shorter than retention, or it never applies. + grace: 168h # 7 days + # Lifetime observation count a route must reach to earn the full retention + # window. Below it a route is treated as flood noise -- a path recorded once + # or twice and never confirmed -- and only kept for the grace window. The + # count never resets, so once a route crosses this bar it stays in the + # retention tier for good. + min_observations: 3 + websocket: max_connections_per_ip: 5 # default: 5 diff --git a/db/migrations/024_known_routes_pathkey.sql b/db/migrations/024_known_routes_pathkey.sql new file mode 100644 index 00000000..ee121a41 --- /dev/null +++ b/db/migrations/024_known_routes_pathkey.sql @@ -0,0 +1,43 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- known_routes grew unbounded (28 GB / 32.8M rows in 24 days on prod) and its +-- UNIQUE(node_ids, iata) key indexed whole UUID arrays. Rebuild keyed on a +-- 16-byte md5 of node_ids and add reconfirm bookkeeping. No rows are pruned +-- here: retention is deployment config (routes.retention/grace), so the first +-- cleanup tick after startup enforces it; a schema migration must not bake in +-- one deployment's policy. id stays (the API serializes it) but loses its +-- index; uniqueness lives on (iata, path_key). + +CREATE TABLE known_routes_new ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + path_key BYTEA NOT NULL, + node_ids UUID[] NOT NULL, + hash_prefix BYTEA[] NOT NULL, + iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, + hop_count INT NOT NULL, + first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), + observation_count BIGINT NOT NULL DEFAULT 1, + last_reconfirmed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (iata, path_key) +); + +INSERT INTO known_routes_new + (id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count) +SELECT id, + decode(md5(array_to_string(node_ids, ',')), 'hex'), + node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count +FROM known_routes; + +SELECT setval(pg_get_serial_sequence('known_routes_new', 'id'), + (SELECT COALESCE(MAX(id), 1) FROM known_routes_new)); + +DROP TABLE known_routes; +ALTER TABLE known_routes_new RENAME TO known_routes; +ALTER TABLE known_routes RENAME CONSTRAINT known_routes_new_pkey TO known_routes_pkey; +ALTER TABLE known_routes RENAME CONSTRAINT known_routes_new_iata_fkey TO known_routes_iata_fkey; + +CREATE INDEX idx_known_routes_hop_count ON known_routes(iata, hop_count); +CREATE INDEX idx_known_routes_last_seen ON known_routes(last_seen DESC); +CREATE INDEX idx_known_routes_reconfirm ON known_routes(last_reconfirmed_at); diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 0938f1eb..22558c42 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -533,6 +533,13 @@ DELETE FROM nodes WHERE last_seen < $1 AND id NOT IN (SELECT owner_node_id FROM observer_owners WHERE owner_node_id IS NOT NULL); +-- name: DeleteOldRoutes :exec +-- Deletes routes not observed since the retention cutoff ($1), and rarely-observed +-- routes (observation_count < $2) not observed since the grace cutoff ($3). +DELETE FROM known_routes +WHERE last_seen < $1 + OR (observation_count < $2 AND last_seen < $3); + -- name: DeleteOldChannelIATAs :exec -- Keeps the channel IATA filter in step with packet retention. DELETE FROM channel_iatas WHERE last_heard < $1; @@ -1070,12 +1077,11 @@ ORDER BY t.last_heard_at DESC; -- ============================================================ -- name: UpsertKnownRoute :exec --- Inserts or updates a known route (all hops resolved to high confidence). --- node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and --- their hash bytes. last_seen is bumped on conflict. -INSERT INTO known_routes (node_ids, hash_prefix, iata, hop_count) -VALUES ($1, $2, $3, $4) -ON CONFLICT (node_ids, iata) DO UPDATE SET +-- Route identity is path_key, an md5 of node_ids computed by the caller. +-- On conflict, observation_count and last_seen are bumped. +INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) +VALUES ($1, $2, $3, $4, $5) +ON CONFLICT (iata, path_key) DO UPDATE SET last_seen = NOW(), observation_count = known_routes.observation_count + 1; @@ -1216,27 +1222,55 @@ REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata; REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets; -- name: ReconfirmRoutes :exec --- Delete known_routes where any hop node has departed from node_short_ids for --- that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node). -DELETE FROM known_routes kr -WHERE EXISTS ( - SELECT 1 - FROM unnest(kr.node_ids) AS hop_node_id - WHERE NOT EXISTS ( - SELECT 1 FROM node_short_ids ns - WHERE ns.node_id = hop_node_id - AND ns.iata = kr.iata +-- Checks the $1 least-recently-reconfirmed routes: deletes those with a departed +-- hop node or a hop prefix now matching >1 node in that IATA (length-aware: +-- 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. +WITH batch AS ( + SELECT iata, path_key, node_ids, hash_prefix + FROM known_routes + ORDER BY last_reconfirmed_at + LIMIT $1 +), +amb AS MATERIALIZED ( + SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 + UNION ALL + SELECT iata, 2, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1 + UNION ALL + SELECT iata, 3, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1 + UNION ALL + SELECT iata, 4, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1 +), +dead AS ( + SELECT b.iata, b.path_key + FROM batch b + WHERE EXISTS ( + SELECT 1 + FROM unnest(b.node_ids) AS hop_node_id + WHERE NOT EXISTS ( + SELECT 1 FROM node_short_ids ns + WHERE ns.node_id = hop_node_id + AND ns.iata = b.iata + ) ) + UNION + SELECT DISTINCT b.iata, b.path_key + FROM batch b + CROSS JOIN LATERAL unnest(b.hash_prefix) AS hp + JOIN amb a ON a.iata = b.iata AND a.len = length(hp) AND a.p = hp +), +deleted AS ( + DELETE FROM known_routes kr + USING dead d + WHERE kr.iata = d.iata AND kr.path_key = d.path_key ) -OR EXISTS ( - SELECT 1 - FROM unnest(kr.hash_prefix) AS hop_prefix - WHERE ( - SELECT COUNT(*) FROM node_short_ids ns - WHERE ns.iata = kr.iata - AND ns.prefix_4 = hop_prefix - ) > 1 -); +UPDATE known_routes kr +SET last_reconfirmed_at = NOW() +FROM batch b +WHERE kr.iata = b.iata AND kr.path_key = b.path_key + AND NOT EXISTS ( + SELECT 1 FROM dead d + WHERE d.iata = b.iata AND d.path_key = b.path_key + ); -- name: ReconfirmNeighbors :exec -- Delete node_neighbors where the neighbor has departed from node_short_ids diff --git a/db/routes.go b/db/routes.go index 54e88876..047bfec6 100644 --- a/db/routes.go +++ b/db/routes.go @@ -5,7 +5,9 @@ package db import ( "context" + "crypto/md5" "encoding/hex" + "strings" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -14,12 +16,24 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +// routePathKey is the route's identity digest: md5 over the comma-joined +// node UUIDs, matching Postgres's decode(md5(array_to_string(node_ids, ',')), 'hex'). +func routePathKey(nodeIDs []uuid.UUID) []byte { + parts := make([]string, len(nodeIDs)) + for i, id := range nodeIDs { + parts[i] = id.String() + } + sum := md5.Sum([]byte(strings.Join(parts, ","))) + return sum[:] +} + func (s *Store) UpsertKnownRoute(ctx context.Context, nodeIDs []uuid.UUID, hashPrefix [][]byte, iata string, hopCount int32) error { return s.q.UpsertKnownRoute(ctx, sqlc.UpsertKnownRouteParams{ + PathKey: routePathKey(nodeIDs), NodeIds: nodeIDs, HashPrefix: hashPrefix, Iata: iata, - HopCount: int32(hopCount), + HopCount: hopCount, }) } @@ -28,7 +42,7 @@ func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32 if !cursor.IsZero() { cursorTS = pgtype.Timestamptz{Time: cursor, Valid: true} } - rows, err := s.q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{ + sqlRows, err := s.q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{ Column1: iata, Column2: hopCount, Column3: cursorTS, @@ -37,6 +51,10 @@ func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32 if err != nil { return nil, err } + rows := make([]knownRouteRow, len(sqlRows)) + for i, r := range sqlRows { + rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount} + } ids := collectNodeIDs(rows) nodes, err := s.GetNodesByIDs(ctx, ids) if err != nil { @@ -54,7 +72,7 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st if err != nil { return nil, err } - rows, err := s.q.SearchKnownRoutes(ctx, sqlc.SearchKnownRoutesParams{ + sqlRows, err := s.q.SearchKnownRoutes(ctx, sqlc.SearchKnownRoutesParams{ Iata: iata, Column2: fromBytes, Column3: toBytes, @@ -62,6 +80,10 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st if err != nil { return nil, err } + rows := make([]knownRouteRow, len(sqlRows)) + for i, r := range sqlRows { + rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount} + } ids := collectNodeIDs(rows) nodes, err := s.GetNodesByIDs(ctx, ids) if err != nil { @@ -109,13 +131,17 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st } func (s *Store) GetKnownRoutesByNode(ctx context.Context, iata string, nodeID uuid.UUID) ([]api.KnownRoute, error) { - rows, err := s.q.GetKnownRoutesByNode(ctx, sqlc.GetKnownRoutesByNodeParams{ + sqlRows, err := s.q.GetKnownRoutesByNode(ctx, sqlc.GetKnownRoutesByNodeParams{ Iata: iata, Column2: nodeID, }) if err != nil { return nil, err } + rows := make([]knownRouteRow, len(sqlRows)) + for i, r := range sqlRows { + rows[i] = knownRouteRow{ID: r.ID, NodeIds: r.NodeIds, HashPrefix: r.HashPrefix, Iata: r.Iata, HopCount: r.HopCount, FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, ObservationCount: r.ObservationCount} + } ids := collectNodeIDs(rows) nodes, err := s.GetNodesByIDs(ctx, ids) if err != nil { @@ -260,8 +286,20 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t return results, nil } -func (s *Store) ReconfirmRoutes(ctx context.Context) error { - return s.q.ReconfirmRoutes(ctx) +// ReconfirmRoutes checks the batchSize least-recently-reconfirmed routes, +// deleting stale or ambiguous ones and stamping the survivors. +func (s *Store) ReconfirmRoutes(ctx context.Context, batchSize int32) error { + return s.q.ReconfirmRoutes(ctx, batchSize) +} + +// DeleteOldRoutes prunes routes per the retention rule: unconditionally past +// retentionCutoff, and past graceCutoff when observed fewer than minObservations times. +func (s *Store) DeleteOldRoutes(ctx context.Context, retentionCutoff time.Time, minObservations int64, graceCutoff time.Time) error { + return s.q.DeleteOldRoutes(ctx, sqlc.DeleteOldRoutesParams{ + LastSeen: pgtype.Timestamptz{Time: retentionCutoff, Valid: true}, + ObservationCount: minObservations, + LastSeen_2: pgtype.Timestamptz{Time: graceCutoff, Valid: true}, + }) } // extractFromNode returns the portion of a route starting at the given node. @@ -274,7 +312,20 @@ func extractFromNode(hops []api.RouteHop, nodeID uuid.UUID) []api.RouteHop { return hops } -func toKnownRoutes(rows []sqlc.KnownRoute, nodes map[uuid.UUID]*api.ResolvedNode) []api.KnownRoute { +// knownRouteRow normalizes the per-query sqlc row structs (identical +// columns, distinct generated types) so the helpers below share one body. +type knownRouteRow struct { + ID int64 + NodeIds []uuid.UUID + HashPrefix [][]byte + Iata string + HopCount int32 + FirstSeen pgtype.Timestamptz + LastSeen pgtype.Timestamptz + ObservationCount int64 +} + +func toKnownRoutes(rows []knownRouteRow, nodes map[uuid.UUID]*api.ResolvedNode) []api.KnownRoute { items := make([]api.KnownRoute, 0, len(rows)) for _, r := range rows { hops := make([]api.RouteHop, 0, len(r.NodeIds)) @@ -301,7 +352,7 @@ func toKnownRoutes(rows []sqlc.KnownRoute, nodes map[uuid.UUID]*api.ResolvedNode return items } -func collectNodeIDs(rows []sqlc.KnownRoute) []uuid.UUID { +func collectNodeIDs(rows []knownRouteRow) []uuid.UUID { seen := make(map[uuid.UUID]struct{}) var ids []uuid.UUID for _, r := range rows { diff --git a/db/routes_test.go b/db/routes_test.go index 1df7a6ec..977b541a 100644 --- a/db/routes_test.go +++ b/db/routes_test.go @@ -4,12 +4,70 @@ package db import ( + "bytes" + "context" + "encoding/hex" "testing" + "time" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/google/uuid" + "go.uber.org/mock/gomock" ) +func TestRoutePathKey_MatchesPostgresDigest(t *testing.T) { + // Golden vector: Postgres computes + // decode(md5(array_to_string(node_ids, ',')), 'hex') + // over lowercase-hyphenated UUIDs. Migration 024 backfilled with that + // expression; this pins the Go side to the identical bytes. + a := uuid.MustParse("00000000-0000-0000-0000-000000000001") + b := uuid.MustParse("00000000-0000-0000-0000-000000000002") + got := hex.EncodeToString(routePathKey([]uuid.UUID{a, b})) + want := "f097439148601d9f3291c474f82fa64c" + if got != want { + t.Errorf("routePathKey = %s, want %s", got, want) + } +} + +func TestUpsertKnownRoute_ComputesPathKey(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + + a := uuid.MustParse("00000000-0000-0000-0000-000000000001") + b := uuid.MustParse("00000000-0000-0000-0000-000000000002") + wantKey, _ := hex.DecodeString("f097439148601d9f3291c474f82fa64c") + + mock.EXPECT().UpsertKnownRoute(gomock.Any(), gomock.Cond(func(p sqlc.UpsertKnownRouteParams) bool { + return bytes.Equal(p.PathKey, wantKey) + })).Return(nil) + + if err := store.UpsertKnownRoute(context.Background(), []uuid.UUID{a, b}, [][]byte{{0x37}, {0xd8}}, "PRG", 2); err != nil { + t.Fatal(err) + } +} + +func TestDeleteOldRoutes_PassesCutoffs(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + + retention := time.Date(2026, 7, 23, 0, 0, 0, 0, time.UTC) + grace := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC) + + mock.EXPECT().DeleteOldRoutes(gomock.Any(), gomock.Cond(func(p sqlc.DeleteOldRoutesParams) bool { + return p.LastSeen.Time.Equal(retention) && p.ObservationCount == 3 && p.LastSeen_2.Time.Equal(grace) + })).Return(nil) + + if err := store.DeleteOldRoutes(context.Background(), retention, 3, grace); err != nil { + t.Fatal(err) + } +} + func TestExtractFromNode_Found(t *testing.T) { a, b, c := uuid.New(), uuid.New(), uuid.New() hops := []api.RouteHop{ diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index f5b951b5..e86eb2c9 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -85,6 +85,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldPackets(ctx, lastHeardAt any) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldPackets", reflect.TypeOf((*MockQuerier)(nil).DeleteOldPackets), ctx, lastHeardAt) } +// DeleteOldRoutes mocks base method. +func (m *MockQuerier) DeleteOldRoutes(ctx context.Context, arg db.DeleteOldRoutesParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldRoutes", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteOldRoutes indicates an expected call of DeleteOldRoutes. +func (mr *MockQuerierMockRecorder) DeleteOldRoutes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldRoutes", reflect.TypeOf((*MockQuerier)(nil).DeleteOldRoutes), ctx, arg) +} + // DeleteOldTelemetry mocks base method. func (m *MockQuerier) DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error { m.ctrl.T.Helper() @@ -189,10 +203,10 @@ func (mr *MockQuerierMockRecorder) GetIATABorder(ctx, iata any) *gomock.Call { } // GetKnownRoutesByNode mocks base method. -func (m *MockQuerier) GetKnownRoutesByNode(ctx context.Context, arg db.GetKnownRoutesByNodeParams) ([]db.KnownRoute, error) { +func (m *MockQuerier) GetKnownRoutesByNode(ctx context.Context, arg db.GetKnownRoutesByNodeParams) ([]db.GetKnownRoutesByNodeRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetKnownRoutesByNode", ctx, arg) - ret0, _ := ret[0].([]db.KnownRoute) + ret0, _ := ret[0].([]db.GetKnownRoutesByNodeRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -818,10 +832,10 @@ func (mr *MockQuerierMockRecorder) ListIATAs(ctx any) *gomock.Call { } // ListKnownRoutes mocks base method. -func (m *MockQuerier) ListKnownRoutes(ctx context.Context, arg db.ListKnownRoutesParams) ([]db.KnownRoute, error) { +func (m *MockQuerier) ListKnownRoutes(ctx context.Context, arg db.ListKnownRoutesParams) ([]db.ListKnownRoutesRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListKnownRoutes", ctx, arg) - ret0, _ := ret[0].([]db.KnownRoute) + ret0, _ := ret[0].([]db.ListKnownRoutesRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -1027,17 +1041,17 @@ func (mr *MockQuerierMockRecorder) ReconfirmNeighbors(ctx any) *gomock.Call { } // ReconfirmRoutes mocks base method. -func (m *MockQuerier) ReconfirmRoutes(ctx context.Context) error { +func (m *MockQuerier) ReconfirmRoutes(ctx context.Context, limit int32) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx) + ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx, limit) ret0, _ := ret[0].(error) return ret0 } // ReconfirmRoutes indicates an expected call of ReconfirmRoutes. -func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx any) *gomock.Call { +func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx, limit any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx, limit) } // RefreshHourlyStats mocks base method. @@ -1199,10 +1213,10 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal } // SearchKnownRoutes mocks base method. -func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.KnownRoute, error) { +func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SearchKnownRoutes", ctx, arg) - ret0, _ := ret[0].([]db.KnownRoute) + ret0, _ := ret[0].([]db.SearchKnownRoutesRow) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/db/sqlc/models.go b/db/sqlc/models.go index b5516333..9a403553 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -57,14 +57,16 @@ type IataCode struct { } type KnownRoute struct { - ID int64 `json:"id"` - NodeIds []uuid.UUID `json:"node_ids"` - HashPrefix [][]byte `json:"hash_prefix"` - Iata string `json:"iata"` - HopCount int32 `json:"hop_count"` - FirstSeen pgtype.Timestamptz `json:"first_seen"` - LastSeen pgtype.Timestamptz `json:"last_seen"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + PathKey []byte `json:"path_key"` + NodeIds []uuid.UUID `json:"node_ids"` + HashPrefix [][]byte `json:"hash_prefix"` + Iata string `json:"iata"` + HopCount int32 `json:"hop_count"` + FirstSeen pgtype.Timestamptz `json:"first_seen"` + LastSeen pgtype.Timestamptz `json:"last_seen"` + ObservationCount int64 `json:"observation_count"` + LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"` } type MvHourlyIataStat struct { diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index c496e8e6..dd967850 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -25,6 +25,9 @@ type Querier interface { // Deletes packets and their observations older than the given cutoff. // packet_observations cascade-delete via FK. DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error + // Deletes routes not observed since the retention cutoff ($1), and rarely-observed + // routes (observation_count < $2) not observed since the grace cutoff ($3). + DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) error // Deletes telemetry rows older than the given cutoff. Called by the cleanup goroutine. DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error // Keeps the trace IATA filter in step with packet retention. @@ -38,7 +41,7 @@ type Querier interface { // missing row (unknown IATA) is sql.ErrNoRows, same not-found distinction // GetIATA already makes. GetIATABorder(ctx context.Context, iata string) ([]byte, error) - GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]KnownRoute, error) + GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error) GetNodeByID(ctx context.Context, id uuid.UUID) (GetNodeByIDRow, error) GetNodeByPubkey(ctx context.Context, publicKey []byte) (uuid.UUID, error) // Returns the neighbors of a node with details, ordered by most recently seen. @@ -119,7 +122,7 @@ type Querier interface { // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) ListIATAs(ctx context.Context) ([]IataCode, error) - ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]KnownRoute, error) + ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) // Returns messages after the given message ID, ordered oldest first. // Used for WS reconnect backfill. ListMessagesAfterID(ctx context.Context, arg ListMessagesAfterIDParams) ([]ListMessagesAfterIDRow, error) @@ -168,9 +171,10 @@ type Querier interface { // Delete node_neighbors where the neighbor has departed from node_short_ids // for that IATA, or where its prefix_4 is now ambiguous. ReconfirmNeighbors(ctx context.Context) error - // Delete known_routes where any hop node has departed from node_short_ids for - // that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node). - ReconfirmRoutes(ctx context.Context) error + // Checks the $1 least-recently-reconfirmed routes: deletes those with a departed + // hop node or a hop prefix now matching >1 node in that IATA (length-aware: + // 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. + ReconfirmRoutes(ctx context.Context, limit int32) error RefreshHourlyStats(ctx context.Context) error RefreshPayloadBreakdown(ctx context.Context) error RefreshRadioPresets(ctx context.Context) error @@ -190,7 +194,7 @@ type Querier interface { ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. - SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]KnownRoute, error) + SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) SetNodeDefaultScope(ctx context.Context, arg SetNodeDefaultScopeParams) error SetNodeMultibytePaths(ctx context.Context, id uuid.UUID) error SetNodeMultibyteTraces(ctx context.Context, id uuid.UUID) error @@ -228,9 +232,8 @@ type Querier interface { // ============================================================ // ROUTES // ============================================================ - // Inserts or updates a known route (all hops resolved to high confidence). - // node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and - // their hash bytes. last_seen is bumped on conflict. + // Route identity is path_key, an md5 of node_ids computed by the caller. + // On conflict, observation_count and last_seen are bumped. UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error // ============================================================ // NODES diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 7f26f993..64f3d6b1 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -51,6 +51,25 @@ func (q *Queries) DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Times return err } +const deleteOldRoutes = `-- name: DeleteOldRoutes :exec +DELETE FROM known_routes +WHERE last_seen < $1 + OR (observation_count < $2 AND last_seen < $3) +` + +type DeleteOldRoutesParams struct { + LastSeen pgtype.Timestamptz `json:"last_seen"` + ObservationCount int64 `json:"observation_count"` + LastSeen_2 pgtype.Timestamptz `json:"last_seen_2"` +} + +// Deletes routes not observed since the retention cutoff ($1), and rarely-observed +// routes (observation_count < $2) not observed since the grace cutoff ($3). +func (q *Queries) DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) error { + _, err := q.db.Exec(ctx, deleteOldRoutes, arg.LastSeen, arg.ObservationCount, arg.LastSeen_2) + return err +} + const deleteOldTelemetry = `-- name: DeleteOldTelemetry :exec DELETE FROM observer_telemetry WHERE reported_at < $1 ` @@ -237,15 +256,26 @@ type GetKnownRoutesByNodeParams struct { Column2 uuid.UUID `json:"column_2"` } -func (q *Queries) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]KnownRoute, error) { +type GetKnownRoutesByNodeRow struct { + ID int64 `json:"id"` + NodeIds []uuid.UUID `json:"node_ids"` + HashPrefix [][]byte `json:"hash_prefix"` + Iata string `json:"iata"` + HopCount int32 `json:"hop_count"` + FirstSeen pgtype.Timestamptz `json:"first_seen"` + LastSeen pgtype.Timestamptz `json:"last_seen"` + ObservationCount int64 `json:"observation_count"` +} + +func (q *Queries) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error) { rows, err := q.db.Query(ctx, getKnownRoutesByNode, arg.Iata, arg.Column2) if err != nil { return nil, err } defer rows.Close() - items := []KnownRoute{} + items := []GetKnownRoutesByNodeRow{} for rows.Next() { - var i KnownRoute + var i GetKnownRoutesByNodeRow if err := rows.Scan( &i.ID, &i.NodeIds, @@ -2095,7 +2125,18 @@ type ListKnownRoutesParams struct { Limit int32 `json:"limit"` } -func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]KnownRoute, error) { +type ListKnownRoutesRow struct { + ID int64 `json:"id"` + NodeIds []uuid.UUID `json:"node_ids"` + HashPrefix [][]byte `json:"hash_prefix"` + Iata string `json:"iata"` + HopCount int32 `json:"hop_count"` + FirstSeen pgtype.Timestamptz `json:"first_seen"` + LastSeen pgtype.Timestamptz `json:"last_seen"` + ObservationCount int64 `json:"observation_count"` +} + +func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) { rows, err := q.db.Query(ctx, listKnownRoutes, arg.Column1, arg.Column2, @@ -2106,9 +2147,9 @@ func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams return nil, err } defer rows.Close() - items := []KnownRoute{} + items := []ListKnownRoutesRow{} for rows.Next() { - var i KnownRoute + var i ListKnownRoutesRow if err := rows.Scan( &i.ID, &i.NodeIds, @@ -3197,31 +3238,59 @@ func (q *Queries) ReconfirmNeighbors(ctx context.Context) error { } const reconfirmRoutes = `-- name: ReconfirmRoutes :exec -DELETE FROM known_routes kr -WHERE EXISTS ( - SELECT 1 - FROM unnest(kr.node_ids) AS hop_node_id - WHERE NOT EXISTS ( - SELECT 1 FROM node_short_ids ns - WHERE ns.node_id = hop_node_id - AND ns.iata = kr.iata +WITH batch AS ( + SELECT iata, path_key, node_ids, hash_prefix + FROM known_routes + ORDER BY last_reconfirmed_at + LIMIT $1 +), +amb AS MATERIALIZED ( + SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 + UNION ALL + SELECT iata, 2, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1 + UNION ALL + SELECT iata, 3, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1 + UNION ALL + SELECT iata, 4, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1 +), +dead AS ( + SELECT b.iata, b.path_key + FROM batch b + WHERE EXISTS ( + SELECT 1 + FROM unnest(b.node_ids) AS hop_node_id + WHERE NOT EXISTS ( + SELECT 1 FROM node_short_ids ns + WHERE ns.node_id = hop_node_id + AND ns.iata = b.iata + ) ) + UNION + SELECT DISTINCT b.iata, b.path_key + FROM batch b + CROSS JOIN LATERAL unnest(b.hash_prefix) AS hp + JOIN amb a ON a.iata = b.iata AND a.len = length(hp) AND a.p = hp +), +deleted AS ( + DELETE FROM known_routes kr + USING dead d + WHERE kr.iata = d.iata AND kr.path_key = d.path_key ) -OR EXISTS ( - SELECT 1 - FROM unnest(kr.hash_prefix) AS hop_prefix - WHERE ( - SELECT COUNT(*) FROM node_short_ids ns - WHERE ns.iata = kr.iata - AND ns.prefix_4 = hop_prefix - ) > 1 -) +UPDATE known_routes kr +SET last_reconfirmed_at = NOW() +FROM batch b +WHERE kr.iata = b.iata AND kr.path_key = b.path_key + AND NOT EXISTS ( + SELECT 1 FROM dead d + WHERE d.iata = b.iata AND d.path_key = b.path_key + ) ` -// Delete known_routes where any hop node has departed from node_short_ids for -// that IATA, or where any hop's prefix_4 is now ambiguous (matches >1 node). -func (q *Queries) ReconfirmRoutes(ctx context.Context) error { - _, err := q.db.Exec(ctx, reconfirmRoutes) +// Checks the $1 least-recently-reconfirmed routes: deletes those with a departed +// hop node or a hop prefix now matching >1 node in that IATA (length-aware: +// 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. +func (q *Queries) ReconfirmRoutes(ctx context.Context, limit int32) error { + _, err := q.db.Exec(ctx, reconfirmRoutes, limit) return err } @@ -3512,17 +3581,28 @@ type SearchKnownRoutesParams struct { Column3 []byte `json:"column_3"` } +type SearchKnownRoutesRow struct { + ID int64 `json:"id"` + NodeIds []uuid.UUID `json:"node_ids"` + HashPrefix [][]byte `json:"hash_prefix"` + Iata string `json:"iata"` + HopCount int32 `json:"hop_count"` + FirstSeen pgtype.Timestamptz `json:"first_seen"` + LastSeen pgtype.Timestamptz `json:"last_seen"` + ObservationCount int64 `json:"observation_count"` +} + // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. -func (q *Queries) SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]KnownRoute, error) { +func (q *Queries) SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) { rows, err := q.db.Query(ctx, searchKnownRoutes, arg.Iata, arg.Column2, arg.Column3) if err != nil { return nil, err } defer rows.Close() - items := []KnownRoute{} + items := []SearchKnownRoutesRow{} for rows.Next() { - var i KnownRoute + var i SearchKnownRoutesRow if err := rows.Scan( &i.ID, &i.NodeIds, @@ -3883,14 +3963,15 @@ func (q *Queries) UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsPa const upsertKnownRoute = `-- name: UpsertKnownRoute :exec -INSERT INTO known_routes (node_ids, hash_prefix, iata, hop_count) -VALUES ($1, $2, $3, $4) -ON CONFLICT (node_ids, iata) DO UPDATE SET +INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) +VALUES ($1, $2, $3, $4, $5) +ON CONFLICT (iata, path_key) DO UPDATE SET last_seen = NOW(), observation_count = known_routes.observation_count + 1 ` type UpsertKnownRouteParams struct { + PathKey []byte `json:"path_key"` NodeIds []uuid.UUID `json:"node_ids"` HashPrefix [][]byte `json:"hash_prefix"` Iata string `json:"iata"` @@ -3900,11 +3981,11 @@ type UpsertKnownRouteParams struct { // ============================================================ // ROUTES // ============================================================ -// Inserts or updates a known route (all hops resolved to high confidence). -// node_ids and hash_prefix are ordered arrays of the resolved node UUIDs and -// their hash bytes. last_seen is bumped on conflict. +// Route identity is path_key, an md5 of node_ids computed by the caller. +// On conflict, observation_count and last_seen are bumped. func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error { _, err := q.db.Exec(ctx, upsertKnownRoute, + arg.PathKey, arg.NodeIds, arg.HashPrefix, arg.Iata, diff --git a/internal/background/tasks.go b/internal/background/tasks.go index 6a5cf68f..76c1cf81 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -73,15 +73,23 @@ func CleanupTask(store *db.Store, telemetryRetention, packetRetention, nodeDelet } } -// ReconfirmTask returns a Task that prunes stale and ambiguous resolved paths -// and neighbors. Runs after routes to ensure neighbors are cleaned against -// already-reconfirmed path data. -func ReconfirmTask(store *db.Store, interval time.Duration) Task { +// reconfirmBatchSize bounds per-tick reconfirm work; at hourly ticks a 16M-row +// table gets fully re-checked roughly daily. +const reconfirmBatchSize = 750_000 + +// ReconfirmTask returns a Task that prunes aged routes first, then reconfirms +// stale and ambiguous resolved paths and neighbors, so known_routes only ever +// has one writer at a time. +func ReconfirmTask(store *db.Store, routeRetention, routeGrace time.Duration, routeMinObservations int64, interval time.Duration) Task { return Task{ Name: "reconfirm", Interval: interval, Run: func(ctx context.Context) error { - if err := store.ReconfirmRoutes(ctx); err != nil { + now := time.Now() + if err := store.DeleteOldRoutes(ctx, now.Add(-routeRetention), routeMinObservations, now.Add(-routeGrace)); err != nil { + return fmt.Errorf("route retention: %w", err) + } + if err := store.ReconfirmRoutes(ctx, reconfirmBatchSize); err != nil { return fmt.Errorf("routes: %w", err) } if err := store.ReconfirmNeighbors(ctx); err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index 82c4bb81..f7420052 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,7 @@ type Config struct { Telemetry TelemetryConfig `yaml:"telemetry"` WebSocket WebSocketConfig `yaml:"websocket"` Packets PacketsConfig `yaml:"packets"` + Routes RoutesConfig `yaml:"routes"` Ingest IngestFilterConfig `yaml:"ingest"` Scopes []ScopeConfig `yaml:"scopes"` Cache CacheConfig `yaml:"cache"` @@ -33,13 +34,16 @@ type Config struct { // ResolvedConfig holds all runtime configuration with defaults applied. type ResolvedConfig struct { - TelemetryResolution time.Duration - TelemetryRetention time.Duration - PacketRetention time.Duration - MaxConnsPerIP int - ViewRefreshInterval time.Duration - ReconfirmInterval time.Duration - CleanupInterval time.Duration + TelemetryResolution time.Duration + TelemetryRetention time.Duration + PacketRetention time.Duration + RouteRetention time.Duration + RouteGrace time.Duration + RouteMinObservations int + MaxConnsPerIP int + ViewRefreshInterval time.Duration + ReconfirmInterval time.Duration + CleanupInterval time.Duration PresenceFlushInterval time.Duration PresencePacketTTL time.Duration @@ -177,6 +181,19 @@ type PacketsConfig struct { Retention duration `yaml:"retention"` } +// RoutesConfig controls known-route retention behaviour. +type RoutesConfig struct { + // Retention is how long a route is kept after it was last observed. + // Defaults to 336h (14 days) if not set. + Retention duration `yaml:"retention"` + // Grace is how long a route observed fewer than MinObservations times is + // kept. Defaults to 168h (7 days) if not set. + Grace duration `yaml:"grace"` + // MinObservations is the observation count below which Grace applies + // instead of Retention. Defaults to 3 if not set. + MinObservations int `yaml:"min_observations"` +} + // NodesConfig controls node-derived signal thresholds. type NodesConfig struct { // ClockDriftThreshold is the |device clock - server clock| magnitude, measured from a @@ -299,13 +316,16 @@ func Load(path string) (*Config, error) { // Resolve returns a ResolvedConfig with defaults applied for any zero values. func Resolve(cfg *Config) ResolvedConfig { r := ResolvedConfig{ - TelemetryResolution: cfg.Telemetry.Resolution.Duration, - TelemetryRetention: cfg.Telemetry.Retention.Duration, - PacketRetention: cfg.Packets.Retention.Duration, - MaxConnsPerIP: cfg.WebSocket.MaxConnectionsPerIP, - ViewRefreshInterval: cfg.Background.ViewRefresh.Duration, - ReconfirmInterval: cfg.Background.Reconfirm.Duration, - CleanupInterval: cfg.Background.Cleanup.Duration, + TelemetryResolution: cfg.Telemetry.Resolution.Duration, + TelemetryRetention: cfg.Telemetry.Retention.Duration, + PacketRetention: cfg.Packets.Retention.Duration, + RouteRetention: cfg.Routes.Retention.Duration, + RouteGrace: cfg.Routes.Grace.Duration, + RouteMinObservations: cfg.Routes.MinObservations, + MaxConnsPerIP: cfg.WebSocket.MaxConnectionsPerIP, + ViewRefreshInterval: cfg.Background.ViewRefresh.Duration, + ReconfirmInterval: cfg.Background.Reconfirm.Duration, + CleanupInterval: cfg.Background.Cleanup.Duration, PresenceFlushInterval: cfg.Presence.FlushInterval.Duration, PresencePacketTTL: cfg.Presence.PacketTTL.Duration, @@ -323,6 +343,15 @@ func Resolve(cfg *Config) ResolvedConfig { if r.PacketRetention == 0 { r.PacketRetention = 30 * 24 * time.Hour } + if r.RouteRetention == 0 { + r.RouteRetention = 14 * 24 * time.Hour + } + if r.RouteGrace == 0 { + r.RouteGrace = 7 * 24 * time.Hour + } + if r.RouteMinObservations == 0 { + r.RouteMinObservations = 3 + } if r.MaxConnsPerIP == 0 { r.MaxConnsPerIP = 5 } @@ -357,8 +386,8 @@ func Resolve(cfg *Config) ResolvedConfig { func (r ResolvedConfig) String() string { return fmt.Sprintf( - "telemetryResolution=%s telemetryRetention=%s packetRetention=%s maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s", - r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, + "telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s", + r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, r.MaxConnsPerIP, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold, r.NodeStaleThreshold, r.NodeDeleteAfter, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a68e18b2..4ccd46a6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -141,3 +141,16 @@ func TestResolvedConfig_String(t *testing.T) { t.Error("expected maxConnsPerIP in string") } } + +func TestResolve_RouteDefaults(t *testing.T) { + r := Resolve(&Config{}) + if r.RouteRetention != 336*time.Hour { + t.Errorf("RouteRetention = %s, want 336h", r.RouteRetention) + } + if r.RouteGrace != 168*time.Hour { + t.Errorf("RouteGrace = %s, want 168h", r.RouteGrace) + } + if r.RouteMinObservations != 3 { + t.Errorf("RouteMinObservations = %d, want 3", r.RouteMinObservations) + } +} From ddc6dc1ca938ba6f0dda5ce5129dc38797fe0620 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 7 Aug 2026 09:27:07 -0400 Subject: [PATCH 009/141] fix: keep IATA packet paging alive past short pages A packet repeats once per observer in the site scan, so grouping can collapse a scan_depth window to fewer packets than the page asked for. That short page set hasMore=false and stranded all older history: YOW dead-ended after 45 packets, ~6 minutes back. Report scan saturation and the floor the scans covered, and treat a saturated short page as more data. Clamp the cursor to the floor so paging on cannot skip the band the scan never read. --- db/packets.go | 18 +++++++- db/packets_test.go | 101 +++++++++++++++++++++++++++++++++++++++++ db/queries/queries.sql | 75 +++++++++++++++++++----------- db/sqlc/querier.go | 12 +++-- db/sqlc/queries.sql.go | 79 +++++++++++++++++++++----------- 5 files changed, 227 insertions(+), 58 deletions(-) diff --git a/db/packets.go b/db/packets.go index 27766256..f806a42d 100644 --- a/db/packets.go +++ b/db/packets.go @@ -187,6 +187,14 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes if hasMore { rows = rows[:limit] } + // Observers duplicate a packet across the scan, so the page can collapse + // short while the site still has history below scan_floor. Stopping here + // would strand every older packet behind a hasMore=false. + var scanFloor pgtype.Timestamptz + if len(rows) > 0 && rows[0].ScanSaturated { + hasMore = true + scanFloor = rows[0].ScanFloor + } items := make([]api.PacketSummary, 0, len(rows)) for _, v := range rows { item := api.PacketSummary{ @@ -215,8 +223,14 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes // Cursor follows site-local recency, not the packet's global last_heard_at. var nextCursor *int64 if hasMore && len(rows) > 0 { - last := rows[len(rows)-1].SiteHeardAt.Time.UnixMilli() - nextCursor = &last + last := rows[len(rows)-1].SiteHeardAt.Time + // A saturated scan never read below its floor. Paging past it would + // skip that band; resuming at it only repeats packets already shown. + if scanFloor.Valid && scanFloor.Time.After(last) { + last = scanFloor.Time + } + ms := last.UnixMilli() + nextCursor = &ms } return api.Page[api.PacketSummary]{ Items: items, diff --git a/db/packets_test.go b/db/packets_test.go index 9a4a9cba..f2c8248f 100644 --- a/db/packets_test.go +++ b/db/packets_test.go @@ -527,6 +527,107 @@ func TestListPackets_IATAFilterRoutesToObservationIndex(t *testing.T) { } } +// A site whose packets are heard by more observers than scan_depth allows +// for collapses to a short page while history remains below the floor. +// The page must still report more, or the client stops paging for good. +func TestListPackets_SaturatedShortPageKeepsPaging(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + oldest := time.Date(2026, 8, 7, 12, 57, 25, 0, time.UTC) + floor := time.Date(2026, 8, 7, 12, 50, 0, 0, time.UTC) + + // Two rows for a limit of 5: the scan filled up but collapsed to a short page. + mock.EXPECT(). + ListPacketsByIATAs(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsByIATAsRow{ + { + PacketHash: []byte{0x01}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest.Add(time.Minute), Valid: true}, + ScanSaturated: true, + ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, + }, + { + PacketHash: []byte{0x02}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, + ScanSaturated: true, + ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, + }, + }, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW"}, nil, time.Time{}, time.Time{}, 0, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(page.Items) != 2 { + t.Fatalf("got %d items, want 2", len(page.Items)) + } + if !page.HasMore { + t.Error("hasMore = false on a saturated short page, want true") + } + // Oldest returned item sits above the floor, so it is the safe cursor. + if page.NextCursor == nil || *page.NextCursor != oldest.UnixMilli() { + t.Errorf("next cursor = %v, want %d (oldest item)", page.NextCursor, oldest.UnixMilli()) + } +} + +// The floor is newer than the oldest item, so paging past it would skip the +// band the saturated site never read. +func TestListPackets_CursorClampsToScanFloor(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + floor := time.Date(2026, 8, 7, 12, 50, 0, 0, time.UTC) + oldest := floor.Add(-30 * time.Minute) + + mock.EXPECT(). + ListPacketsByIATAs(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsByIATAsRow{{ + PacketHash: []byte{0x01}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, + ScanSaturated: true, + ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, + }}, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW", "YYZ"}, nil, time.Time{}, time.Time{}, 0, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if page.NextCursor == nil || *page.NextCursor != floor.UnixMilli() { + t.Errorf("next cursor = %v, want %d (clamped to floor)", page.NextCursor, floor.UnixMilli()) + } +} + +// An unsaturated scan read the site dry, so paging has to stop. +func TestListPackets_UnsaturatedShortPageEndsPaging(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + oldest := time.Date(2026, 8, 7, 12, 57, 25, 0, time.UTC) + + mock.EXPECT(). + ListPacketsByIATAs(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsByIATAsRow{{ + PacketHash: []byte{0x01}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, + ScanSaturated: false, + }}, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW"}, nil, time.Time{}, time.Time{}, 0, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if page.HasMore { + t.Error("hasMore = true on an exhausted site, want false") + } + if page.NextCursor != nil { + t.Errorf("next cursor = %v, want nil", page.NextCursor) + } +} + func TestListPackets_UnfilteredKeepsGlobalQuery(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 22558c42..1e540002 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -424,29 +424,14 @@ LIMIT $6; -- to fill a page for a quiet site; walking the site's own observation log is -- proportional to the page size instead. Results are ordered by when the -- requested sites heard the packet (site-local recency) and the cursor --- follows that ordering. scan_depth is a multiple of the page size to --- absorb per-observer duplicates; if duplication exceeds it across a --- page, pagination ends early (hasMore=false) rather than returning a --- short page, even though deeper matches exist. -SELECT - p.packet_hash, - p.payload_type, - p.route_type, - p.first_heard_at, - p.last_heard_at, - p.scope_id, - ts.name AS scope_name, - sh.site_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, - po.observer_id AS latest_observer_id, - o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes -FROM ( - SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at +-- follows that ordering. scan_depth caps how deep each site's observation +-- log is walked. A packet repeats once per observer that heard it, so a +-- page can collapse to fewer distinct packets than were asked for without +-- the site being exhausted. scan_saturated reports whether any site hit +-- that cap and scan_floor the oldest heard_at they all cover, so a short +-- page can keep paging instead of reading as the end of the data. +WITH scanned AS ( + SELECT req.iata AS req_iata, hits.packet_hash, hits.heard_at FROM unnest(@iatas::bpchar[]) AS req(iata) CROSS JOIN LATERAL ( SELECT po3.packet_hash, po3.heard_at @@ -464,15 +449,53 @@ FROM ( ORDER BY po3.heard_at DESC LIMIT @scan_depth ) hits - GROUP BY hits.packet_hash +), +-- A site that filled scan_depth still has unread history below its floor. +-- The newest such floor is the point above which every site is covered. +saturation AS ( + SELECT + COUNT(*) > 0 AS scan_saturated, + MAX(floor_ts)::timestamptz AS scan_floor + FROM ( + SELECT MIN(heard_at) AS floor_ts + FROM scanned + GROUP BY req_iata + HAVING COUNT(*) >= @scan_depth + ) filled +), +page AS ( + SELECT scanned.packet_hash, MAX(scanned.heard_at)::timestamptz AS site_heard_at + FROM scanned + GROUP BY scanned.packet_hash HAVING (@cursor_ts::timestamptz IS NULL OR NOT EXISTS ( SELECT 1 FROM packet_observations px - WHERE px.packet_hash = hits.packet_hash + WHERE px.packet_hash = scanned.packet_hash AND px.iata = ANY(@iatas::bpchar[]) AND px.heard_at >= @cursor_ts)) ORDER BY site_heard_at DESC LIMIT @page_limit -) sh +) +SELECT + p.packet_hash, + p.payload_type, + p.route_type, + p.first_heard_at, + p.last_heard_at, + p.scope_id, + ts.name AS scope_name, + sh.site_heard_at, + sat.scan_saturated, + sat.scan_floor, + (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + po.observer_id AS latest_observer_id, + o.display_name AS latest_observer_name, + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes +FROM page sh +CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index dd967850..9ee115a1 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -147,10 +147,14 @@ type Querier interface { // to fill a page for a quiet site; walking the site's own observation log is // proportional to the page size instead. Results are ordered by when the // requested sites heard the packet (site-local recency) and the cursor - // follows that ordering. scan_depth is a multiple of the page size to - // absorb per-observer duplicates; if duplication exceeds it across a - // page, pagination ends early (hasMore=false) rather than returning a - // short page, even though deeper matches exist. + // follows that ordering. scan_depth caps how deep each site's observation + // log is walked. A packet repeats once per observer that heard it, so a + // page can collapse to fewer distinct packets than were asked for without + // the site being exhausted. scan_saturated reports whether any site hit + // that cap and scan_floor the oldest heard_at they all cover, so a short + // page can keep paging instead of reading as the end of the data. + // A site that filled scan_depth still has unread history below its floor. + // The newest such floor is the point above which every site is covered. ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) // ============================================================ // REGIONS diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 64f3d6b1..1fe4a9c2 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2894,25 +2894,8 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID } const listPacketsByIATAs = `-- name: ListPacketsByIATAs :many -SELECT - p.packet_hash, - p.payload_type, - p.route_type, - p.first_heard_at, - p.last_heard_at, - p.scope_id, - ts.name AS scope_name, - sh.site_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, - po.observer_id AS latest_observer_id, - o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes -FROM ( - SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at +WITH scanned AS ( + SELECT req.iata AS req_iata, hits.packet_hash, hits.heard_at FROM unnest($1::bpchar[]) AS req(iata) CROSS JOIN LATERAL ( SELECT po3.packet_hash, po3.heard_at @@ -2930,15 +2913,51 @@ FROM ( ORDER BY po3.heard_at DESC LIMIT $8 ) hits - GROUP BY hits.packet_hash +), +saturation AS ( + SELECT + COUNT(*) > 0 AS scan_saturated, + MAX(floor_ts)::timestamptz AS scan_floor + FROM ( + SELECT MIN(heard_at) AS floor_ts + FROM scanned + GROUP BY req_iata + HAVING COUNT(*) >= $8 + ) filled +), +page AS ( + SELECT scanned.packet_hash, MAX(scanned.heard_at)::timestamptz AS site_heard_at + FROM scanned + GROUP BY scanned.packet_hash HAVING ($2::timestamptz IS NULL OR NOT EXISTS ( SELECT 1 FROM packet_observations px - WHERE px.packet_hash = hits.packet_hash + WHERE px.packet_hash = scanned.packet_hash AND px.iata = ANY($1::bpchar[]) AND px.heard_at >= $2)) ORDER BY site_heard_at DESC LIMIT $9 -) sh +) +SELECT + p.packet_hash, + p.payload_type, + p.route_type, + p.first_heard_at, + p.last_heard_at, + p.scope_id, + ts.name AS scope_name, + sh.site_heard_at, + sat.scan_saturated, + sat.scan_floor, + (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + po.observer_id AS latest_observer_id, + o.display_name AS latest_observer_name, + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes +FROM page sh +CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes @@ -2973,6 +2992,8 @@ type ListPacketsByIATAsRow struct { ScopeID *int32 `json:"scope_id"` ScopeName *string `json:"scope_name"` SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` + ScanSaturated bool `json:"scan_saturated"` + ScanFloor pgtype.Timestamptz `json:"scan_floor"` ObservationCount int64 `json:"observation_count"` LatestObserverID uuid.UUID `json:"latest_observer_id"` LatestObserverName *string `json:"latest_observer_name"` @@ -2988,10 +3009,14 @@ type ListPacketsByIATAsRow struct { // to fill a page for a quiet site; walking the site's own observation log is // proportional to the page size instead. Results are ordered by when the // requested sites heard the packet (site-local recency) and the cursor -// follows that ordering. scan_depth is a multiple of the page size to -// absorb per-observer duplicates; if duplication exceeds it across a -// page, pagination ends early (hasMore=false) rather than returning a -// short page, even though deeper matches exist. +// follows that ordering. scan_depth caps how deep each site's observation +// log is walked. A packet repeats once per observer that heard it, so a +// page can collapse to fewer distinct packets than were asked for without +// the site being exhausted. scan_saturated reports whether any site hit +// that cap and scan_floor the oldest heard_at they all cover, so a short +// page can keep paging instead of reading as the end of the data. +// A site that filled scan_depth still has unread history below its floor. +// The newest such floor is the point above which every site is covered. func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) { rows, err := q.db.Query(ctx, listPacketsByIATAs, arg.Iatas, @@ -3020,6 +3045,8 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs &i.ScopeID, &i.ScopeName, &i.SiteHeardAt, + &i.ScanSaturated, + &i.ScanFloor, &i.ObservationCount, &i.LatestObserverID, &i.LatestObserverName, From 137e3d265a20c6ea922ccc464fa46230644dd06d Mon Sep 17 00:00:00 2001 From: Ded Date: Thu, 20 Aug 2026 08:35:05 -0700 Subject: [PATCH 010/141] Revert "fix: keep IATA packet paging alive past short pages" This reverts commit ddc6dc1ca938ba6f0dda5ce5129dc38797fe0620. --- db/packets.go | 18 +------- db/packets_test.go | 101 ----------------------------------------- db/queries/queries.sql | 75 +++++++++++------------------- db/sqlc/querier.go | 12 ++--- db/sqlc/queries.sql.go | 79 +++++++++++--------------------- 5 files changed, 58 insertions(+), 227 deletions(-) diff --git a/db/packets.go b/db/packets.go index f806a42d..27766256 100644 --- a/db/packets.go +++ b/db/packets.go @@ -187,14 +187,6 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes if hasMore { rows = rows[:limit] } - // Observers duplicate a packet across the scan, so the page can collapse - // short while the site still has history below scan_floor. Stopping here - // would strand every older packet behind a hasMore=false. - var scanFloor pgtype.Timestamptz - if len(rows) > 0 && rows[0].ScanSaturated { - hasMore = true - scanFloor = rows[0].ScanFloor - } items := make([]api.PacketSummary, 0, len(rows)) for _, v := range rows { item := api.PacketSummary{ @@ -223,14 +215,8 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes // Cursor follows site-local recency, not the packet's global last_heard_at. var nextCursor *int64 if hasMore && len(rows) > 0 { - last := rows[len(rows)-1].SiteHeardAt.Time - // A saturated scan never read below its floor. Paging past it would - // skip that band; resuming at it only repeats packets already shown. - if scanFloor.Valid && scanFloor.Time.After(last) { - last = scanFloor.Time - } - ms := last.UnixMilli() - nextCursor = &ms + last := rows[len(rows)-1].SiteHeardAt.Time.UnixMilli() + nextCursor = &last } return api.Page[api.PacketSummary]{ Items: items, diff --git a/db/packets_test.go b/db/packets_test.go index f2c8248f..9a4a9cba 100644 --- a/db/packets_test.go +++ b/db/packets_test.go @@ -527,107 +527,6 @@ func TestListPackets_IATAFilterRoutesToObservationIndex(t *testing.T) { } } -// A site whose packets are heard by more observers than scan_depth allows -// for collapses to a short page while history remains below the floor. -// The page must still report more, or the client stops paging for good. -func TestListPackets_SaturatedShortPageKeepsPaging(t *testing.T) { - ctrl := gomock.NewController(t) - mock := mockdb.NewMockQuerier(ctrl) - - oldest := time.Date(2026, 8, 7, 12, 57, 25, 0, time.UTC) - floor := time.Date(2026, 8, 7, 12, 50, 0, 0, time.UTC) - - // Two rows for a limit of 5: the scan filled up but collapsed to a short page. - mock.EXPECT(). - ListPacketsByIATAs(gomock.Any(), gomock.Any()). - Return([]sqlc.ListPacketsByIATAsRow{ - { - PacketHash: []byte{0x01}, - SiteHeardAt: pgtype.Timestamptz{Time: oldest.Add(time.Minute), Valid: true}, - ScanSaturated: true, - ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, - }, - { - PacketHash: []byte{0x02}, - SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, - ScanSaturated: true, - ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, - }, - }, nil) - - store := &Store{q: mock} - page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW"}, nil, time.Time{}, time.Time{}, 0, 5) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(page.Items) != 2 { - t.Fatalf("got %d items, want 2", len(page.Items)) - } - if !page.HasMore { - t.Error("hasMore = false on a saturated short page, want true") - } - // Oldest returned item sits above the floor, so it is the safe cursor. - if page.NextCursor == nil || *page.NextCursor != oldest.UnixMilli() { - t.Errorf("next cursor = %v, want %d (oldest item)", page.NextCursor, oldest.UnixMilli()) - } -} - -// The floor is newer than the oldest item, so paging past it would skip the -// band the saturated site never read. -func TestListPackets_CursorClampsToScanFloor(t *testing.T) { - ctrl := gomock.NewController(t) - mock := mockdb.NewMockQuerier(ctrl) - - floor := time.Date(2026, 8, 7, 12, 50, 0, 0, time.UTC) - oldest := floor.Add(-30 * time.Minute) - - mock.EXPECT(). - ListPacketsByIATAs(gomock.Any(), gomock.Any()). - Return([]sqlc.ListPacketsByIATAsRow{{ - PacketHash: []byte{0x01}, - SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, - ScanSaturated: true, - ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, - }}, nil) - - store := &Store{q: mock} - page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW", "YYZ"}, nil, time.Time{}, time.Time{}, 0, 5) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if page.NextCursor == nil || *page.NextCursor != floor.UnixMilli() { - t.Errorf("next cursor = %v, want %d (clamped to floor)", page.NextCursor, floor.UnixMilli()) - } -} - -// An unsaturated scan read the site dry, so paging has to stop. -func TestListPackets_UnsaturatedShortPageEndsPaging(t *testing.T) { - ctrl := gomock.NewController(t) - mock := mockdb.NewMockQuerier(ctrl) - - oldest := time.Date(2026, 8, 7, 12, 57, 25, 0, time.UTC) - - mock.EXPECT(). - ListPacketsByIATAs(gomock.Any(), gomock.Any()). - Return([]sqlc.ListPacketsByIATAsRow{{ - PacketHash: []byte{0x01}, - SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, - ScanSaturated: false, - }}, nil) - - store := &Store{q: mock} - page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW"}, nil, time.Time{}, time.Time{}, 0, 5) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if page.HasMore { - t.Error("hasMore = true on an exhausted site, want false") - } - if page.NextCursor != nil { - t.Errorf("next cursor = %v, want nil", page.NextCursor) - } -} - func TestListPackets_UnfilteredKeepsGlobalQuery(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 1e540002..22558c42 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -424,14 +424,29 @@ LIMIT $6; -- to fill a page for a quiet site; walking the site's own observation log is -- proportional to the page size instead. Results are ordered by when the -- requested sites heard the packet (site-local recency) and the cursor --- follows that ordering. scan_depth caps how deep each site's observation --- log is walked. A packet repeats once per observer that heard it, so a --- page can collapse to fewer distinct packets than were asked for without --- the site being exhausted. scan_saturated reports whether any site hit --- that cap and scan_floor the oldest heard_at they all cover, so a short --- page can keep paging instead of reading as the end of the data. -WITH scanned AS ( - SELECT req.iata AS req_iata, hits.packet_hash, hits.heard_at +-- follows that ordering. scan_depth is a multiple of the page size to +-- absorb per-observer duplicates; if duplication exceeds it across a +-- page, pagination ends early (hasMore=false) rather than returning a +-- short page, even though deeper matches exist. +SELECT + p.packet_hash, + p.payload_type, + p.route_type, + p.first_heard_at, + p.last_heard_at, + p.scope_id, + ts.name AS scope_name, + sh.site_heard_at, + (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + po.observer_id AS latest_observer_id, + o.display_name AS latest_observer_name, + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes +FROM ( + SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at FROM unnest(@iatas::bpchar[]) AS req(iata) CROSS JOIN LATERAL ( SELECT po3.packet_hash, po3.heard_at @@ -449,53 +464,15 @@ WITH scanned AS ( ORDER BY po3.heard_at DESC LIMIT @scan_depth ) hits -), --- A site that filled scan_depth still has unread history below its floor. --- The newest such floor is the point above which every site is covered. -saturation AS ( - SELECT - COUNT(*) > 0 AS scan_saturated, - MAX(floor_ts)::timestamptz AS scan_floor - FROM ( - SELECT MIN(heard_at) AS floor_ts - FROM scanned - GROUP BY req_iata - HAVING COUNT(*) >= @scan_depth - ) filled -), -page AS ( - SELECT scanned.packet_hash, MAX(scanned.heard_at)::timestamptz AS site_heard_at - FROM scanned - GROUP BY scanned.packet_hash + GROUP BY hits.packet_hash HAVING (@cursor_ts::timestamptz IS NULL OR NOT EXISTS ( SELECT 1 FROM packet_observations px - WHERE px.packet_hash = scanned.packet_hash + WHERE px.packet_hash = hits.packet_hash AND px.iata = ANY(@iatas::bpchar[]) AND px.heard_at >= @cursor_ts)) ORDER BY site_heard_at DESC LIMIT @page_limit -) -SELECT - p.packet_hash, - p.payload_type, - p.route_type, - p.first_heard_at, - p.last_heard_at, - p.scope_id, - ts.name AS scope_name, - sh.site_heard_at, - sat.scan_saturated, - sat.scan_floor, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, - po.observer_id AS latest_observer_id, - o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes -FROM page sh -CROSS JOIN saturation sat +) sh JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 9ee115a1..dd967850 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -147,14 +147,10 @@ type Querier interface { // to fill a page for a quiet site; walking the site's own observation log is // proportional to the page size instead. Results are ordered by when the // requested sites heard the packet (site-local recency) and the cursor - // follows that ordering. scan_depth caps how deep each site's observation - // log is walked. A packet repeats once per observer that heard it, so a - // page can collapse to fewer distinct packets than were asked for without - // the site being exhausted. scan_saturated reports whether any site hit - // that cap and scan_floor the oldest heard_at they all cover, so a short - // page can keep paging instead of reading as the end of the data. - // A site that filled scan_depth still has unread history below its floor. - // The newest such floor is the point above which every site is covered. + // follows that ordering. scan_depth is a multiple of the page size to + // absorb per-observer duplicates; if duplication exceeds it across a + // page, pagination ends early (hasMore=false) rather than returning a + // short page, even though deeper matches exist. ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) // ============================================================ // REGIONS diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 1fe4a9c2..64f3d6b1 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2894,8 +2894,25 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID } const listPacketsByIATAs = `-- name: ListPacketsByIATAs :many -WITH scanned AS ( - SELECT req.iata AS req_iata, hits.packet_hash, hits.heard_at +SELECT + p.packet_hash, + p.payload_type, + p.route_type, + p.first_heard_at, + p.last_heard_at, + p.scope_id, + ts.name AS scope_name, + sh.site_heard_at, + (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + po.observer_id AS latest_observer_id, + o.display_name AS latest_observer_name, + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes +FROM ( + SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at FROM unnest($1::bpchar[]) AS req(iata) CROSS JOIN LATERAL ( SELECT po3.packet_hash, po3.heard_at @@ -2913,51 +2930,15 @@ WITH scanned AS ( ORDER BY po3.heard_at DESC LIMIT $8 ) hits -), -saturation AS ( - SELECT - COUNT(*) > 0 AS scan_saturated, - MAX(floor_ts)::timestamptz AS scan_floor - FROM ( - SELECT MIN(heard_at) AS floor_ts - FROM scanned - GROUP BY req_iata - HAVING COUNT(*) >= $8 - ) filled -), -page AS ( - SELECT scanned.packet_hash, MAX(scanned.heard_at)::timestamptz AS site_heard_at - FROM scanned - GROUP BY scanned.packet_hash + GROUP BY hits.packet_hash HAVING ($2::timestamptz IS NULL OR NOT EXISTS ( SELECT 1 FROM packet_observations px - WHERE px.packet_hash = scanned.packet_hash + WHERE px.packet_hash = hits.packet_hash AND px.iata = ANY($1::bpchar[]) AND px.heard_at >= $2)) ORDER BY site_heard_at DESC LIMIT $9 -) -SELECT - p.packet_hash, - p.payload_type, - p.route_type, - p.first_heard_at, - p.last_heard_at, - p.scope_id, - ts.name AS scope_name, - sh.site_heard_at, - sat.scan_saturated, - sat.scan_floor, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, - po.observer_id AS latest_observer_id, - o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes -FROM page sh -CROSS JOIN saturation sat +) sh JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes @@ -2992,8 +2973,6 @@ type ListPacketsByIATAsRow struct { ScopeID *int32 `json:"scope_id"` ScopeName *string `json:"scope_name"` SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` - ScanSaturated bool `json:"scan_saturated"` - ScanFloor pgtype.Timestamptz `json:"scan_floor"` ObservationCount int64 `json:"observation_count"` LatestObserverID uuid.UUID `json:"latest_observer_id"` LatestObserverName *string `json:"latest_observer_name"` @@ -3009,14 +2988,10 @@ type ListPacketsByIATAsRow struct { // to fill a page for a quiet site; walking the site's own observation log is // proportional to the page size instead. Results are ordered by when the // requested sites heard the packet (site-local recency) and the cursor -// follows that ordering. scan_depth caps how deep each site's observation -// log is walked. A packet repeats once per observer that heard it, so a -// page can collapse to fewer distinct packets than were asked for without -// the site being exhausted. scan_saturated reports whether any site hit -// that cap and scan_floor the oldest heard_at they all cover, so a short -// page can keep paging instead of reading as the end of the data. -// A site that filled scan_depth still has unread history below its floor. -// The newest such floor is the point above which every site is covered. +// follows that ordering. scan_depth is a multiple of the page size to +// absorb per-observer duplicates; if duplication exceeds it across a +// page, pagination ends early (hasMore=false) rather than returning a +// short page, even though deeper matches exist. func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) { rows, err := q.db.Query(ctx, listPacketsByIATAs, arg.Iatas, @@ -3045,8 +3020,6 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs &i.ScopeID, &i.ScopeName, &i.SiteHeardAt, - &i.ScanSaturated, - &i.ScanFloor, &i.ObservationCount, &i.LatestObserverID, &i.LatestObserverName, From b5ab592534c9dafdbcd0497c4d7512902ef9c04e Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Thu, 3 Sep 2026 21:54:58 -0400 Subject: [PATCH 011/141] fix: re-land IATA paging fix on Go 1.26.6 (#108) --- db/packets.go | 18 +++++++- db/packets_test.go | 101 +++++++++++++++++++++++++++++++++++++++++ db/queries/queries.sql | 75 +++++++++++++++++++----------- db/sqlc/querier.go | 12 +++-- db/sqlc/queries.sql.go | 79 +++++++++++++++++++++----------- go.mod | 2 +- 6 files changed, 228 insertions(+), 59 deletions(-) diff --git a/db/packets.go b/db/packets.go index 27766256..f806a42d 100644 --- a/db/packets.go +++ b/db/packets.go @@ -187,6 +187,14 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes if hasMore { rows = rows[:limit] } + // Observers duplicate a packet across the scan, so the page can collapse + // short while the site still has history below scan_floor. Stopping here + // would strand every older packet behind a hasMore=false. + var scanFloor pgtype.Timestamptz + if len(rows) > 0 && rows[0].ScanSaturated { + hasMore = true + scanFloor = rows[0].ScanFloor + } items := make([]api.PacketSummary, 0, len(rows)) for _, v := range rows { item := api.PacketSummary{ @@ -215,8 +223,14 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes // Cursor follows site-local recency, not the packet's global last_heard_at. var nextCursor *int64 if hasMore && len(rows) > 0 { - last := rows[len(rows)-1].SiteHeardAt.Time.UnixMilli() - nextCursor = &last + last := rows[len(rows)-1].SiteHeardAt.Time + // A saturated scan never read below its floor. Paging past it would + // skip that band; resuming at it only repeats packets already shown. + if scanFloor.Valid && scanFloor.Time.After(last) { + last = scanFloor.Time + } + ms := last.UnixMilli() + nextCursor = &ms } return api.Page[api.PacketSummary]{ Items: items, diff --git a/db/packets_test.go b/db/packets_test.go index 9a4a9cba..f2c8248f 100644 --- a/db/packets_test.go +++ b/db/packets_test.go @@ -527,6 +527,107 @@ func TestListPackets_IATAFilterRoutesToObservationIndex(t *testing.T) { } } +// A site whose packets are heard by more observers than scan_depth allows +// for collapses to a short page while history remains below the floor. +// The page must still report more, or the client stops paging for good. +func TestListPackets_SaturatedShortPageKeepsPaging(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + oldest := time.Date(2026, 8, 7, 12, 57, 25, 0, time.UTC) + floor := time.Date(2026, 8, 7, 12, 50, 0, 0, time.UTC) + + // Two rows for a limit of 5: the scan filled up but collapsed to a short page. + mock.EXPECT(). + ListPacketsByIATAs(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsByIATAsRow{ + { + PacketHash: []byte{0x01}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest.Add(time.Minute), Valid: true}, + ScanSaturated: true, + ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, + }, + { + PacketHash: []byte{0x02}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, + ScanSaturated: true, + ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, + }, + }, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW"}, nil, time.Time{}, time.Time{}, 0, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(page.Items) != 2 { + t.Fatalf("got %d items, want 2", len(page.Items)) + } + if !page.HasMore { + t.Error("hasMore = false on a saturated short page, want true") + } + // Oldest returned item sits above the floor, so it is the safe cursor. + if page.NextCursor == nil || *page.NextCursor != oldest.UnixMilli() { + t.Errorf("next cursor = %v, want %d (oldest item)", page.NextCursor, oldest.UnixMilli()) + } +} + +// The floor is newer than the oldest item, so paging past it would skip the +// band the saturated site never read. +func TestListPackets_CursorClampsToScanFloor(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + floor := time.Date(2026, 8, 7, 12, 50, 0, 0, time.UTC) + oldest := floor.Add(-30 * time.Minute) + + mock.EXPECT(). + ListPacketsByIATAs(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsByIATAsRow{{ + PacketHash: []byte{0x01}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, + ScanSaturated: true, + ScanFloor: pgtype.Timestamptz{Time: floor, Valid: true}, + }}, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW", "YYZ"}, nil, time.Time{}, time.Time{}, 0, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if page.NextCursor == nil || *page.NextCursor != floor.UnixMilli() { + t.Errorf("next cursor = %v, want %d (clamped to floor)", page.NextCursor, floor.UnixMilli()) + } +} + +// An unsaturated scan read the site dry, so paging has to stop. +func TestListPackets_UnsaturatedShortPageEndsPaging(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + + oldest := time.Date(2026, 8, 7, 12, 57, 25, 0, time.UTC) + + mock.EXPECT(). + ListPacketsByIATAs(gomock.Any(), gomock.Any()). + Return([]sqlc.ListPacketsByIATAsRow{{ + PacketHash: []byte{0x01}, + SiteHeardAt: pgtype.Timestamptz{Time: oldest, Valid: true}, + ScanSaturated: false, + }}, nil) + + store := &Store{q: mock} + page, err := store.ListPackets(context.Background(), nil, nil, []string{"YOW"}, nil, time.Time{}, time.Time{}, 0, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if page.HasMore { + t.Error("hasMore = true on an exhausted site, want false") + } + if page.NextCursor != nil { + t.Errorf("next cursor = %v, want nil", page.NextCursor) + } +} + func TestListPackets_UnfilteredKeepsGlobalQuery(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 22558c42..1e540002 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -424,29 +424,14 @@ LIMIT $6; -- to fill a page for a quiet site; walking the site's own observation log is -- proportional to the page size instead. Results are ordered by when the -- requested sites heard the packet (site-local recency) and the cursor --- follows that ordering. scan_depth is a multiple of the page size to --- absorb per-observer duplicates; if duplication exceeds it across a --- page, pagination ends early (hasMore=false) rather than returning a --- short page, even though deeper matches exist. -SELECT - p.packet_hash, - p.payload_type, - p.route_type, - p.first_heard_at, - p.last_heard_at, - p.scope_id, - ts.name AS scope_name, - sh.site_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, - po.observer_id AS latest_observer_id, - o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes -FROM ( - SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at +-- follows that ordering. scan_depth caps how deep each site's observation +-- log is walked. A packet repeats once per observer that heard it, so a +-- page can collapse to fewer distinct packets than were asked for without +-- the site being exhausted. scan_saturated reports whether any site hit +-- that cap and scan_floor the oldest heard_at they all cover, so a short +-- page can keep paging instead of reading as the end of the data. +WITH scanned AS ( + SELECT req.iata AS req_iata, hits.packet_hash, hits.heard_at FROM unnest(@iatas::bpchar[]) AS req(iata) CROSS JOIN LATERAL ( SELECT po3.packet_hash, po3.heard_at @@ -464,15 +449,53 @@ FROM ( ORDER BY po3.heard_at DESC LIMIT @scan_depth ) hits - GROUP BY hits.packet_hash +), +-- A site that filled scan_depth still has unread history below its floor. +-- The newest such floor is the point above which every site is covered. +saturation AS ( + SELECT + COUNT(*) > 0 AS scan_saturated, + MAX(floor_ts)::timestamptz AS scan_floor + FROM ( + SELECT MIN(heard_at) AS floor_ts + FROM scanned + GROUP BY req_iata + HAVING COUNT(*) >= @scan_depth + ) filled +), +page AS ( + SELECT scanned.packet_hash, MAX(scanned.heard_at)::timestamptz AS site_heard_at + FROM scanned + GROUP BY scanned.packet_hash HAVING (@cursor_ts::timestamptz IS NULL OR NOT EXISTS ( SELECT 1 FROM packet_observations px - WHERE px.packet_hash = hits.packet_hash + WHERE px.packet_hash = scanned.packet_hash AND px.iata = ANY(@iatas::bpchar[]) AND px.heard_at >= @cursor_ts)) ORDER BY site_heard_at DESC LIMIT @page_limit -) sh +) +SELECT + p.packet_hash, + p.payload_type, + p.route_type, + p.first_heard_at, + p.last_heard_at, + p.scope_id, + ts.name AS scope_name, + sh.site_heard_at, + sat.scan_saturated, + sat.scan_floor, + (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + po.observer_id AS latest_observer_id, + o.display_name AS latest_observer_name, + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes +FROM page sh +CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index dd967850..9ee115a1 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -147,10 +147,14 @@ type Querier interface { // to fill a page for a quiet site; walking the site's own observation log is // proportional to the page size instead. Results are ordered by when the // requested sites heard the packet (site-local recency) and the cursor - // follows that ordering. scan_depth is a multiple of the page size to - // absorb per-observer duplicates; if duplication exceeds it across a - // page, pagination ends early (hasMore=false) rather than returning a - // short page, even though deeper matches exist. + // follows that ordering. scan_depth caps how deep each site's observation + // log is walked. A packet repeats once per observer that heard it, so a + // page can collapse to fewer distinct packets than were asked for without + // the site being exhausted. scan_saturated reports whether any site hit + // that cap and scan_floor the oldest heard_at they all cover, so a short + // page can keep paging instead of reading as the end of the data. + // A site that filled scan_depth still has unread history below its floor. + // The newest such floor is the point above which every site is covered. ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) // ============================================================ // REGIONS diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 64f3d6b1..1fe4a9c2 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2894,25 +2894,8 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID } const listPacketsByIATAs = `-- name: ListPacketsByIATAs :many -SELECT - p.packet_hash, - p.payload_type, - p.route_type, - p.first_heard_at, - p.last_heard_at, - p.scope_id, - ts.name AS scope_name, - sh.site_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, - po.observer_id AS latest_observer_id, - o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes -FROM ( - SELECT hits.packet_hash, MAX(hits.heard_at)::timestamptz AS site_heard_at +WITH scanned AS ( + SELECT req.iata AS req_iata, hits.packet_hash, hits.heard_at FROM unnest($1::bpchar[]) AS req(iata) CROSS JOIN LATERAL ( SELECT po3.packet_hash, po3.heard_at @@ -2930,15 +2913,51 @@ FROM ( ORDER BY po3.heard_at DESC LIMIT $8 ) hits - GROUP BY hits.packet_hash +), +saturation AS ( + SELECT + COUNT(*) > 0 AS scan_saturated, + MAX(floor_ts)::timestamptz AS scan_floor + FROM ( + SELECT MIN(heard_at) AS floor_ts + FROM scanned + GROUP BY req_iata + HAVING COUNT(*) >= $8 + ) filled +), +page AS ( + SELECT scanned.packet_hash, MAX(scanned.heard_at)::timestamptz AS site_heard_at + FROM scanned + GROUP BY scanned.packet_hash HAVING ($2::timestamptz IS NULL OR NOT EXISTS ( SELECT 1 FROM packet_observations px - WHERE px.packet_hash = hits.packet_hash + WHERE px.packet_hash = scanned.packet_hash AND px.iata = ANY($1::bpchar[]) AND px.heard_at >= $2)) ORDER BY site_heard_at DESC LIMIT $9 -) sh +) +SELECT + p.packet_hash, + p.payload_type, + p.route_type, + p.first_heard_at, + p.last_heard_at, + p.scope_id, + ts.name AS scope_name, + sh.site_heard_at, + sat.scan_saturated, + sat.scan_floor, + (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + po.observer_id AS latest_observer_id, + o.display_name AS latest_observer_name, + po.iata AS latest_observer_iata, + po.path_length_byte AS latest_observer_path_length_byte, + po.hash_size AS latest_observer_hash_size, + po.hop_count AS latest_observer_hop_count, + po.path_bytes AS latest_observer_path_bytes +FROM page sh +CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes @@ -2973,6 +2992,8 @@ type ListPacketsByIATAsRow struct { ScopeID *int32 `json:"scope_id"` ScopeName *string `json:"scope_name"` SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` + ScanSaturated bool `json:"scan_saturated"` + ScanFloor pgtype.Timestamptz `json:"scan_floor"` ObservationCount int64 `json:"observation_count"` LatestObserverID uuid.UUID `json:"latest_observer_id"` LatestObserverName *string `json:"latest_observer_name"` @@ -2988,10 +3009,14 @@ type ListPacketsByIATAsRow struct { // to fill a page for a quiet site; walking the site's own observation log is // proportional to the page size instead. Results are ordered by when the // requested sites heard the packet (site-local recency) and the cursor -// follows that ordering. scan_depth is a multiple of the page size to -// absorb per-observer duplicates; if duplication exceeds it across a -// page, pagination ends early (hasMore=false) rather than returning a -// short page, even though deeper matches exist. +// follows that ordering. scan_depth caps how deep each site's observation +// log is walked. A packet repeats once per observer that heard it, so a +// page can collapse to fewer distinct packets than were asked for without +// the site being exhausted. scan_saturated reports whether any site hit +// that cap and scan_floor the oldest heard_at they all cover, so a short +// page can keep paging instead of reading as the end of the data. +// A site that filled scan_depth still has unread history below its floor. +// The newest such floor is the point above which every site is covered. func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) { rows, err := q.db.Query(ctx, listPacketsByIATAs, arg.Iatas, @@ -3020,6 +3045,8 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs &i.ScopeID, &i.ScopeName, &i.SiteHeardAt, + &i.ScanSaturated, + &i.ScanFloor, &i.ObservationCount, &i.LatestObserverID, &i.LatestObserverName, diff --git a/go.mod b/go.mod index 6f981fab..ac5e38c3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/MeshCore-Beacon/beacon-server -go 1.26.5 +go 1.26.6 require ( github.com/alicebob/miniredis/v2 v2.38.0 From dfaa31d1ddb4cad06306cdf7b59d31694ea3367e Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 10:20:24 -0400 Subject: [PATCH 012/141] perf(db): enrich nodes after selecting the directory page (#111) --- db/nodes_integration_test.go | 185 +++++++++++++++++++++++++++++++++++ db/queries/queries.sql | 31 +++--- db/sqlc/querier.go | 1 + db/sqlc/queries.sql.go | 29 +++--- 4 files changed, 219 insertions(+), 27 deletions(-) create mode 100644 db/nodes_integration_test.go diff --git a/db/nodes_integration_test.go b/db/nodes_integration_test.go new file mode 100644 index 00000000..03ddcd07 --- /dev/null +++ b/db/nodes_integration_test.go @@ -0,0 +1,185 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +//go:embed queries/queries.sql +var nodeTestQueries string + +// Run against a migrated PostgreSQL database with BEACON_TEST_POSTGRES_DSN. +// All fixture writes are transaction-local temporary tables, rolled back on exit. +func TestListNodesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"nodes", "node_iatas", "node_neighbors", "observers", "transport_scopes"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO transport_scopes (id, name, transport_key, key_fingerprint) VALUES (1, '#test', decode(repeat('00', 16), 'hex'), decode(repeat('00', 8), 'hex')); +INSERT INTO nodes (id, public_key, node_type, name, last_seen, default_scope_id, supports_multibyte_paths, supports_multibyte_traces) +SELECT md5(i::text)::uuid, decode(lpad(to_hex(i), 64, '0'), 'hex'), (i % 4 + 1)::smallint, + 'node-' || i, '2026-01-01'::timestamptz - i * interval '1 second', + CASE WHEN i % 2 = 0 THEN 1 END, i % 2 = 0, i % 3 = 0 +FROM generate_series(1, 20000) i; +INSERT INTO node_iatas (node_id, iata, last_heard) +SELECT md5(i::text)::uuid, 'YVR', '2026-01-01'::timestamptz - i * interval '1 second' +FROM generate_series(1, 19999) i; +INSERT INTO node_iatas (node_id, iata, last_heard) +SELECT md5(i::text)::uuid, 'YYJ', '2026-01-01'::timestamptz - i * interval '1 second' - interval '1 hour' +FROM generate_series(2, 19998, 2) i; +INSERT INTO node_neighbors (node_id, neighbor_id, iata) +SELECT md5(i::text)::uuid, md5((i+1)::text)::uuid, iata +FROM generate_series(1, 19999) i CROSS JOIN (VALUES ('YVR'), ('YYJ')) regions(iata); +INSERT INTO observers (id, public_key) +SELECT id, public_key FROM nodes WHERE name = 'node-10'; +ANALYZE nodes; ANALYZE node_iatas; ANALYZE node_neighbors; ANALYZE observers; ANALYZE transport_scopes; +`) + if err != nil { + t.Fatal(err) + } + query := sqlc.New(tx) + base := sqlc.ListNodesParams{Column1: int16(0), Column3: "any", Column4: "any", Column6: "", Limit: 51} + for _, tc := range []struct { + name string + change func(*sqlc.ListNodesParams) + want []int + }{ + {"page", func(p *sqlc.ListNodesParams) { p.Limit = 3 }, []int{1, 2, 3}}, + {"type", func(p *sqlc.ListNodesParams) { p.Column1 = 2; p.Limit = 3 }, []int{1, 5, 9}}, + {"multi IATA dedup", func(p *sqlc.ListNodesParams) { p.Column2 = []string{"YVR", "YYJ"}; p.Limit = 3 }, []int{1, 2, 3}}, + {"IATA", func(p *sqlc.ListNodesParams) { p.Column2 = []string{"YYJ"}; p.Limit = 3 }, []int{2, 4, 6}}, + {"unknown IATA", func(p *sqlc.ListNodesParams) { p.Column2 = []string{"ZZZ"} }, nil}, + {"scope", func(p *sqlc.ListNodesParams) { p.Column9 = "#test"; p.Limit = 3 }, []int{2, 4, 6}}, + {"capabilities", func(p *sqlc.ListNodesParams) { p.Column3 = "true"; p.Column4 = "false"; p.Limit = 3 }, []int{2, 4, 8}}, + {"name", func(p *sqlc.ListNodesParams) { p.Column6 = "NODE-19"; p.Limit = 3 }, []int{19, 190, 191}}, + {"cursor", func(p *sqlc.ListNodesParams) { + p.Column7 = pgtype.Timestamptz{Time: time.Date(2026, 1, 1, 0, 0, -3, 0, time.UTC), Valid: true} + p.Limit = 3 + }, []int{4, 5, 6}}, + {"public key", func(p *sqlc.ListNodesParams) { p.Column5, _ = hex.DecodeString(fmt.Sprintf("%064x", 10)) }, []int{10}}, + {"public key prefix", func(p *sqlc.ListNodesParams) { p.Column11 = strings.Repeat("0", 61) + "00A" }, []int{10}}, + {"no IATA", func(p *sqlc.ListNodesParams) { p.Column5, _ = hex.DecodeString(fmt.Sprintf("%064x", 20000)) }, []int{20000}}, + } { + for _, neighbors := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/neighbors=%t", tc.name, neighbors), func(t *testing.T) { + params := base + tc.change(¶ms) + params.Column10 = neighbors + rows, err := query.ListNodes(ctx, params) + if err != nil { + t.Fatal(err) + } + if len(rows) != len(tc.want) { + t.Fatalf("got %d rows, want %d", len(rows), len(tc.want)) + } + for i, number := range tc.want { + row := rows[i] + if row.Name == nil || *row.Name != fmt.Sprintf("node-%d", number) { + t.Fatalf("wrong row at %d: %v", i, row.Name) + } + var iatas []struct { + IATA string `json:"iata"` + } + if len(row.Iatas) > 0 { + if err := json.Unmarshal(row.Iatas, &iatas); err != nil { + t.Fatal(err) + } + } + wantIATAs := 1 + if number%2 == 0 { + wantIATAs = 2 + } + wantNeighbors := int64(1) + if number == 20000 { + wantIATAs, wantNeighbors = 0, 0 + } + if len(iatas) != wantIATAs || (len(iatas) > 0 && iatas[0].IATA != "YVR") { + t.Errorf("wrong IATAs: %s", row.Iatas) + } + if row.KnownNeighborCount != wantNeighbors { + t.Errorf("neighbor count %d, want %d", row.KnownNeighborCount, wantNeighbors) + } + if neighbors && len(row.NeighborIds) != int(wantNeighbors) { + t.Errorf("neighbor IDs %v", row.NeighborIds) + } + if !neighbors && len(row.NeighborIds) != 0 { + t.Errorf("unrequested neighbor IDs %v", row.NeighborIds) + } + if row.IsObserver != (number == 10) { + t.Error("wrong observer flag") + } + if number == 10 && row.ObserverID != row.ID { + t.Error("wrong observer ID") + } + } + }) + } + } + + // A small page must not aggregate every node's IATA membership. Assert work, + // not elapsed time: shared CI and Pi hosts have variable scheduling latency. + queries := strings.ReplaceAll(nodeTestQueries, "\r\n", "\n") + sql := strings.Split(strings.Split(queries, "-- name: ListNodes :many\n")[1], "\n-- name:")[0] + var planJSON []byte + err = tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+sql, + int16(0), []string(nil), "any", "any", []byte(nil), "", nil, int32(51), "", false, "").Scan(&planJSON) + if err != nil { + t.Fatal(err) + } + var plans []struct { + Plan map[string]any + ExecutionTime float64 `json:"Execution Time"` + } + if err := json.Unmarshal(planJSON, &plans); err != nil { + t.Fatal(err) + } + var visited float64 + var walk func(map[string]any) + walk = func(plan map[string]any) { + if plan["Relation Name"] == "node_iatas" { + visited += plan["Actual Rows"].(float64) * plan["Actual Loops"].(float64) + } + if children, ok := plan["Plans"].([]any); ok { + for _, child := range children { + walk(child.(map[string]any)) + } + } + } + walk(plans[0].Plan) + t.Logf("20,000 nodes / 51-row page: execution %.3f ms; IATA rows processed %.0f", plans[0].ExecutionTime, visited) + if visited > 200 { + t.Errorf("small node page processed %.0f IATA rows; budget 200", visited) + } +} diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 1e540002..b6213db3 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -659,20 +659,11 @@ WHERE id = ANY($1::uuid[]); SELECT id FROM nodes WHERE public_key = $1; -- name: ListNodes :many +-- Limit the filtered node page before enriching IATA membership and neighbours. +WITH page AS ( SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, - n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, - ts.name AS default_scope_name, - json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) FILTER (WHERE ni.iata IS NOT NULL) AS iatas, - EXISTS (SELECT 1 FROM observers o WHERE o.public_key = n.public_key) AS is_observer, - (SELECT o.id FROM observers o WHERE o.public_key = n.public_key LIMIT 1) AS observer_id, - (SELECT COUNT(DISTINCT nn.neighbor_id) FROM node_neighbors nn WHERE nn.node_id = n.id)::bigint AS known_neighbor_count, - -- CASE short-circuits: the array_agg subquery only runs when $10 is true, - -- so requests that don't ask for neighbor IDs don't pay for it. -(CASE WHEN $10::bool THEN - (SELECT COALESCE(array_agg(DISTINCT nn.neighbor_id), '{}'::uuid[]) FROM node_neighbors nn WHERE nn.node_id = n.id) - ELSE NULL END)::uuid[] AS neighbor_ids + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, ts.name AS default_scope_name FROM nodes n -LEFT JOIN node_iatas ni ON ni.node_id = n.id LEFT JOIN transport_scopes ts ON ts.id = n.default_scope_id WHERE ($1 = 0 OR n.node_type = $1) @@ -692,9 +683,21 @@ WHERE AND ($7::timestamptz IS NULL OR n.last_seen < $7) AND ($9::text = '' OR ts.name = $9::text) AND ($11::text = '' OR encode(n.public_key, 'hex') ILIKE $11 || '%') -GROUP BY n.id, ts.name ORDER BY n.last_seen DESC -LIMIT $8; +LIMIT $8 +) +SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, n.default_scope_name, + (SELECT json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) + FROM node_iatas ni WHERE ni.node_id = n.id) AS iatas, + EXISTS (SELECT 1 FROM observers o WHERE o.public_key = n.public_key) AS is_observer, + (SELECT o.id FROM observers o WHERE o.public_key = n.public_key LIMIT 1) AS observer_id, + (SELECT COUNT(DISTINCT nn.neighbor_id) FROM node_neighbors nn WHERE nn.node_id = n.id)::bigint AS known_neighbor_count, + (CASE WHEN $10::bool THEN + (SELECT COALESCE(array_agg(DISTINCT nn.neighbor_id), '{}'::uuid[]) FROM node_neighbors nn WHERE nn.node_id = n.id) + ELSE NULL END)::uuid[] AS neighbor_ids +FROM page n +ORDER BY n.last_seen DESC; -- name: ListNodeObservations :many SELECT po.id, encode(po.packet_hash, 'hex') AS packet_hash_hex, diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 9ee115a1..9e2ab2b8 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -127,6 +127,7 @@ type Querier interface { // Used for WS reconnect backfill. ListMessagesAfterID(ctx context.Context, arg ListMessagesAfterIDParams) ([]ListMessagesAfterIDRow, error) ListNodeObservations(ctx context.Context, arg ListNodeObservationsParams) ([]ListNodeObservationsRow, error) + // Limit the filtered node page before enriching IATA membership and neighbours. ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNodesRow, error) ListObservationsForPacket(ctx context.Context, packetHash []byte) ([]ListObservationsForPacketRow, error) // Returns advert packets (payload_type=4) heard by a specific observer. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 1fe4a9c2..670f7adf 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2302,20 +2302,10 @@ func (q *Queries) ListNodeObservations(ctx context.Context, arg ListNodeObservat } const listNodes = `-- name: ListNodes :many +WITH page AS ( SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, - n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, - ts.name AS default_scope_name, - json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) FILTER (WHERE ni.iata IS NOT NULL) AS iatas, - EXISTS (SELECT 1 FROM observers o WHERE o.public_key = n.public_key) AS is_observer, - (SELECT o.id FROM observers o WHERE o.public_key = n.public_key LIMIT 1) AS observer_id, - (SELECT COUNT(DISTINCT nn.neighbor_id) FROM node_neighbors nn WHERE nn.node_id = n.id)::bigint AS known_neighbor_count, - -- CASE short-circuits: the array_agg subquery only runs when $10 is true, - -- so requests that don't ask for neighbor IDs don't pay for it. -(CASE WHEN $10::bool THEN - (SELECT COALESCE(array_agg(DISTINCT nn.neighbor_id), '{}'::uuid[]) FROM node_neighbors nn WHERE nn.node_id = n.id) - ELSE NULL END)::uuid[] AS neighbor_ids + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, ts.name AS default_scope_name FROM nodes n -LEFT JOIN node_iatas ni ON ni.node_id = n.id LEFT JOIN transport_scopes ts ON ts.id = n.default_scope_id WHERE ($1 = 0 OR n.node_type = $1) @@ -2335,9 +2325,21 @@ WHERE AND ($7::timestamptz IS NULL OR n.last_seen < $7) AND ($9::text = '' OR ts.name = $9::text) AND ($11::text = '' OR encode(n.public_key, 'hex') ILIKE $11 || '%') -GROUP BY n.id, ts.name ORDER BY n.last_seen DESC LIMIT $8 +) +SELECT n.id, n.public_key, n.node_type, n.name, n.latitude, n.longitude, n.last_seen, + n.radio_freq_mhz, n.radio_sf, n.radio_bw_khz, n.default_scope_name, + (SELECT json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) + FROM node_iatas ni WHERE ni.node_id = n.id) AS iatas, + EXISTS (SELECT 1 FROM observers o WHERE o.public_key = n.public_key) AS is_observer, + (SELECT o.id FROM observers o WHERE o.public_key = n.public_key LIMIT 1) AS observer_id, + (SELECT COUNT(DISTINCT nn.neighbor_id) FROM node_neighbors nn WHERE nn.node_id = n.id)::bigint AS known_neighbor_count, + (CASE WHEN $10::bool THEN + (SELECT COALESCE(array_agg(DISTINCT nn.neighbor_id), '{}'::uuid[]) FROM node_neighbors nn WHERE nn.node_id = n.id) + ELSE NULL END)::uuid[] AS neighbor_ids +FROM page n +ORDER BY n.last_seen DESC ` type ListNodesParams struct { @@ -2373,6 +2375,7 @@ type ListNodesRow struct { NeighborIds []uuid.UUID `json:"neighbor_ids"` } +// Limit the filtered node page before enriching IATA membership and neighbours. func (q *Queries) ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNodesRow, error) { rows, err := q.db.Query(ctx, listNodes, arg.Column1, From 30327cd98d0d1c8f40323c6e354f0b10a380eaaf Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 10:20:45 -0400 Subject: [PATCH 013/141] fix(api): bound list limits consistently (#112) --- docs/docs.go | 96 +++++++++++---- docs/swagger.json | 80 ++++++++++--- docs/swagger.yaml | 93 +++++++++++--- internal/api/handlers/channels.go | 32 ++--- internal/api/handlers/limits.go | 33 +++++ internal/api/handlers/limits_test.go | 173 +++++++++++++++++++++++++++ internal/api/handlers/messages.go | 35 ++---- internal/api/handlers/nodes.go | 28 ++--- internal/api/handlers/observers.go | 28 ++--- internal/api/handlers/packets.go | 28 ++--- internal/api/handlers/routes.go | 12 +- internal/api/handlers/stats.go | 70 ++++------- internal/api/handlers/traces.go | 12 +- 13 files changed, 508 insertions(+), 212 deletions(-) create mode 100644 internal/api/handlers/limits.go create mode 100644 internal/api/handlers/limits_test.go diff --git a/docs/docs.go b/docs/docs.go index f8d9256c..94cc9198 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4,11 +4,11 @@ package docs import "github.com/swaggo/swag" const docTemplate = `{ - "schemes": [[ marshal .Schemes ]], + "schemes": {{ marshal .Schemes }}, "swagger": "2.0", "info": { - "description": "[[escape .Description]]", - "title": "[[.Title]]", + "description": "{{escape .Description}}", + "title": "{{.Title}}", "termsOfService": "https://github.com/MeshCore-Beacon/beacon-server", "contact": { "name": "MeshCore Beacon", @@ -17,10 +17,10 @@ const docTemplate = `{ "license": { "name": "AGPL-3-or-later" }, - "version": "[[.Version]]" + "version": "{{.Version}}" }, - "host": "[[.Host]]", - "basePath": "[[.BasePath]]", + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", "paths": { "/brokers": { "get": { @@ -79,8 +79,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -201,8 +203,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -387,8 +391,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -457,8 +463,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 100)", + "description": "Max results (default 100); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -583,8 +591,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -718,8 +728,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -817,8 +829,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -909,8 +923,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1090,8 +1106,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1178,8 +1196,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 100)", + "description": "Max results (default 100); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1352,8 +1372,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1368,6 +1390,12 @@ const docTemplate = `{ } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -1612,8 +1640,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1955,8 +1985,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2009,8 +2041,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2069,8 +2103,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2129,8 +2165,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2213,8 +2251,10 @@ const docTemplate = `{ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2229,6 +2269,12 @@ const docTemplate = `{ } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -3936,8 +3982,8 @@ var SwaggerInfo = &swag.Spec{ Description: "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, - LeftDelim: "[[", - RightDelim: "]]", + LeftDelim: "{{", + RightDelim: "}}", } func init() { diff --git a/docs/swagger.json b/docs/swagger.json index e9eaf319..c943d3b2 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -77,8 +77,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -199,8 +201,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -385,8 +389,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -455,8 +461,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 100)", + "description": "Max results (default 100); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -581,8 +589,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -716,8 +726,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -815,8 +827,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -907,8 +921,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1088,8 +1104,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1176,8 +1194,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 100)", + "description": "Max results (default 100); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1350,8 +1370,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1366,6 +1388,12 @@ } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -1610,8 +1638,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -1953,8 +1983,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2007,8 +2039,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2067,8 +2101,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2127,8 +2163,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 10)", + "description": "Max results (default 10); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2211,8 +2249,10 @@ "in": "query" }, { + "maximum": 200, + "minimum": 1, "type": "integer", - "description": "Max results (default 50)", + "description": "Max results (default 50); must be positive, values above 200 are clamped", "name": "limit", "in": "query" } @@ -2227,6 +2267,12 @@ } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 3e0a74c6..c6667f39 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1177,8 +1177,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1258,8 +1261,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1380,8 +1386,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1426,8 +1435,11 @@ paths: in: query name: scope type: string - - description: Max results (default 100) + - description: Max results (default 100); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1510,8 +1522,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1598,8 +1613,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1663,8 +1681,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1723,8 +1744,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1845,8 +1869,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -1935,8 +1962,11 @@ paths: in: query name: scope type: string - - description: Max results (default 100) + - description: Max results (default 100); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2018,8 +2048,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2031,6 +2064,10 @@ paths: items: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute' type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' "500": description: Internal Server Error schema: @@ -2189,8 +2226,11 @@ paths: in: query name: region type: string - - description: Max results (default 10) + - description: Max results (default 10); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2412,8 +2452,11 @@ paths: in: query name: since type: integer - - description: Max results (default 10) + - description: Max results (default 10); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2447,8 +2490,11 @@ paths: in: query name: region type: string - - description: Max results (default 10) + - description: Max results (default 10); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2486,8 +2532,11 @@ paths: in: query name: since type: integer - - description: Max results (default 10) + - description: Max results (default 10); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2525,8 +2574,11 @@ paths: in: query name: since type: integer - - description: Max results (default 10) + - description: Max results (default 10); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2581,8 +2633,11 @@ paths: in: query name: cursor type: integer - - description: Max results (default 50) + - description: Max results (default 50); must be positive, values above 200 + are clamped in: query + maximum: 200 + minimum: 1 name: limit type: integer produces: @@ -2594,6 +2649,10 @@ paths: items: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.TraceTagSummary' type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' "500": description: Internal Server Error schema: diff --git a/internal/api/handlers/channels.go b/internal/api/handlers/channels.go index f394058b..d0f6f7f4 100644 --- a/internal/api/handlers/channels.go +++ b/internal/api/handlers/channels.go @@ -38,21 +38,17 @@ func ChannelsRouter(reader api.Reader) http.Handler { // @Param iata query string false "Filter by IATA code" // @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ" // @Param cursor query int false "last_seen epoch ms of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} api.Page[api.ChannelSummary] // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /channels [get] func listChannels(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - var limit int64 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = l + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } iatas := parseIATAs(r) var cursor int64 @@ -77,7 +73,7 @@ func listChannels(reader api.Reader) http.HandlerFunc { } hashHex = h } - channels, err := reader.ListChannels(r.Context(), int32(limit), hashHex, iatas, cursor) + channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, cursor) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return @@ -128,7 +124,7 @@ func getChannel(reader api.Reader) http.HandlerFunc { // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param scope query string false "Filter by transport scope name e.g. %23bc (URL-encoded #bc)" // @Param cursor query int false "Message ID of last item for pagination (results ordered newest first)" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} object // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -144,14 +140,10 @@ func listChannelMessages(reader api.Reader) http.HandlerFunc { } id = i } - var limit int64 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = l + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var since time.Time if sinceParam := r.URL.Query().Get("since"); sinceParam != "" { @@ -174,7 +166,7 @@ func listChannelMessages(reader api.Reader) http.HandlerFunc { } scope := r.URL.Query().Get("scope") chanID := int32(id) - messages, err := reader.ListChannelMessages(r.Context(), &chanID, since, int32(limit), iatas, scope, cursor) + messages, err := reader.ListChannelMessages(r.Context(), &chanID, since, limit, iatas, scope, cursor) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/limits.go b/internal/api/handlers/limits.go new file mode 100644 index 00000000..8ee6fb40 --- /dev/null +++ b/internal/api/handlers/limits.go @@ -0,0 +1,33 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "errors" + "log" + "net/http" + "strconv" +) + +const maxListLimit int32 = 200 + +// parseLimit caps the requested result size while retaining each endpoint's default. +func parseLimit(r *http.Request, defaultLimit int32) (int32, error) { + value := r.URL.Query().Get("limit") + if value == "" { + return defaultLimit, nil + } + limit, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return 0, errors.New("limit must be an integer") + } + if limit <= 0 { + return 0, errors.New("limit must be positive") + } + if limit > int64(maxListLimit) { + log.Printf("api: clamped list limit from %d to %d for %s on %s", limit, maxListLimit, r.RemoteAddr, r.URL.Path) + return maxListLimit, nil + } + return int32(limit), nil +} diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go new file mode 100644 index 00000000..ff63a95e --- /dev/null +++ b/internal/api/handlers/limits_test.go @@ -0,0 +1,173 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" + "github.com/google/uuid" +) + +func TestListLimits(t *testing.T) { + var got int32 + var calls int + reader := stubReader{ + listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) { + calls++ + got = limit + return api.Page[api.ChannelSummary]{}, nil + }, + listChannelMessages: func(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) { + calls++ + got = limit + return api.Page[api.ChannelMessage]{}, nil + }, + listChannelMessagesByHash: func(ctx context.Context, hash []byte, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) { + calls++ + got = limit + return api.Page[api.ChannelMessage]{}, nil + }, + listMessagesAfterID: func(ctx context.Context, afterID int64, iatas []string, scope string, limit int32) ([]api.ChannelMessage, error) { + calls++ + got = limit + return nil, nil + }, + listObservers: func(ctx context.Context, iatas []string, observerType, broker, status, name, scope string, cursor int64, limit int32) (api.Page[api.ObserverSummary], error) { + calls++ + got = limit + return api.Page[api.ObserverSummary]{}, nil + }, + listObserverAdverts: func(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) { + calls++ + got = limit + return api.Page[api.AdvertObservation]{}, nil + }, + listNodes: func(ctx context.Context, nodeType int16, iatas []string, supportsMultibytePaths, supportsMultibyteTraces *bool, pubkey []byte, pubkeyPrefix, name, scope string, cursor int64, limit int32, includeNeighbors bool) (api.Page[api.NodeSummary], error) { + calls++ + got = limit + return api.Page[api.NodeSummary]{}, nil + }, + listNodeObservations: func(ctx context.Context, nodeID uuid.UUID, cursor int64, limit int32) (api.Page[api.PacketObservationSummary], error) { + calls++ + got = limit + return api.Page[api.PacketObservationSummary]{}, nil + }, + listPackets: func(ctx context.Context, payloadTypes, routeTypes []int16, iatas []string, scopes []string, since, until time.Time, cursor int64, limit int32) (api.Page[api.PacketSummary], error) { + calls++ + got = limit + return api.Page[api.PacketSummary]{}, nil + }, + listPacketsAfterID: func(ctx context.Context, afterObservationID int64, payloadType, routeType int16, iatas []string, scope string, limit int32) ([]api.PacketSummary, error) { + calls++ + got = limit + return nil, nil + }, + getStatsTopNodes: func(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { + calls++ + got = limit + return nil, nil + }, + getStatsTopObservers: func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { + calls++ + got = limit + return nil, nil + }, + getStatsTopAdvertisers: func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { + calls++ + got = limit + return nil, nil + }, + getStatsClockDrift: func(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { + calls++ + got = limit + return nil, nil + }, + getStatsTopTalkers: func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { + calls++ + got = limit + return nil, nil + }, + listTraceTags: func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) { + calls++ + got = limit + return nil, nil + }, + listKnownRoutes: func(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) { + calls++ + got = limit + return nil, nil + }, + } + router := chi.NewRouter() + router.Mount("/nodes", NodesRouter(reader)) + router.Mount("/observers", ObserversRouter(reader)) + router.Mount("/packets", PacketsRouter(reader)) + router.Mount("/channels", ChannelsRouter(reader)) + router.Mount("/messages", MessagesRouter(reader)) + router.Mount("/routes", RoutesRouter(reader)) + router.Mount("/traces", TracesRouter(reader)) + router.Mount("/stats", StatsRouter(reader)) + for _, endpoint := range []struct { + path string + defaultLimit int32 + }{ + {"/nodes", 50}, + {"/nodes/00000000-0000-0000-0000-000000000001/observations", 50}, + {"/observers", 50}, + {"/observers/00000000-0000-0000-0000-000000000001/adverts", 50}, + {"/packets", 50}, + {"/packets/backfill?afterObservationId=1", 100}, + {"/channels", 50}, + {"/channels/1/messages", 50}, + {"/messages", 50}, + {"/messages?channelHash=11", 50}, + {"/messages/backfill?afterId=1", 100}, + {"/routes", 50}, + {"/traces", 50}, + {"/stats/top-nodes", 10}, + {"/stats/top-observers", 10}, + {"/stats/top-advertisers", 10}, + {"/stats/clock-drift", 10}, + {"/stats/top-talkers", 10}, + } { + t.Run(endpoint.path, func(t *testing.T) { + for _, tc := range []struct { + value string + want int32 + status int + }{ + {"", endpoint.defaultLimit, 200}, {"1", 1, 200}, {"200", 200, 200}, + {"201", 200, 200}, {"1000000", 200, 200}, {"2147483647", 200, 200}, + {"0", 0, 400}, {"-1", 0, 400}, {"bad", 0, 400}, {"1.5", 0, 400}, + {"2147483648", 0, 400}, + } { + t.Run(tc.value, func(t *testing.T) { + got, calls = 0, 0 + req := httptest.NewRequest(http.MethodGet, endpoint.path, nil) + query := req.URL.Query() + query.Set("limit", tc.value) + req.URL.RawQuery = query.Encode() + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != tc.status { + t.Fatalf("status %d, want %d: %s", rec.Code, tc.status, rec.Body.String()) + } + if tc.status == http.StatusBadRequest { + if calls != 0 { + t.Fatal("invalid limit reached the reader") + } + } else if calls != 1 || got != tc.want { + t.Fatalf("reader calls=%d limit=%d, want 1 and %d", calls, got, tc.want) + } + }) + } + }) + } +} diff --git a/internal/api/handlers/messages.go b/internal/api/handlers/messages.go index da5f8eee..7d10fb82 100644 --- a/internal/api/handlers/messages.go +++ b/internal/api/handlers/messages.go @@ -37,7 +37,7 @@ func MessagesRouter(reader api.Reader) http.Handler { // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param scope query string false "Filter by transport scope name e.g. %23bc (URL-encoded #bc)" // @Param cursor query int false "Message ID of last item for pagination (results ordered newest first)" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} object // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -60,14 +60,10 @@ func listMessages(reader api.Reader) http.HandlerFunc { } id = i } - var limit int64 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = l + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var since time.Time if sinceParam := r.URL.Query().Get("since"); sinceParam != "" { @@ -90,7 +86,6 @@ func listMessages(reader api.Reader) http.HandlerFunc { iatas := parseIATAs(r) scope := r.URL.Query().Get("scope") var messages api.Page[api.ChannelMessage] - var err error if channelHashParam != "" { hashHex, decodeErr := hex.DecodeString(channelHashParam) if decodeErr != nil { @@ -101,12 +96,12 @@ func listMessages(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "channel hash must be a single hex byte") return } - messages, err = reader.ListChannelMessagesByHash(r.Context(), hashHex, since, int32(limit), iatas, scope, cursor) + messages, err = reader.ListChannelMessagesByHash(r.Context(), hashHex, since, limit, iatas, scope, cursor) } else if channelIDParam != "" { chanID := int32(id) - messages, err = reader.ListChannelMessages(r.Context(), &chanID, since, int32(limit), iatas, scope, cursor) + messages, err = reader.ListChannelMessages(r.Context(), &chanID, since, limit, iatas, scope, cursor) } else { - messages, err = reader.ListChannelMessages(r.Context(), nil, since, int32(limit), iatas, scope, cursor) + messages, err = reader.ListChannelMessages(r.Context(), nil, since, limit, iatas, scope, cursor) } if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") @@ -126,7 +121,7 @@ func listMessages(reader api.Reader) http.HandlerFunc { // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" // @Param scope query string false "Filter by transport scope name" -// @Param limit query int false "Max results (default 100)" +// @Param limit query int false "Max results (default 100); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} []api.ChannelMessage // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -143,14 +138,10 @@ func listMessagesBackfill(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "afterId must be an integer") return } - var limit int32 = 100 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 100) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } iatas := parseIATAs(r) if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index 65f4329a..d844a307 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -49,7 +49,7 @@ func NodesRouter(reader api.Reader) http.Handler { // @Param supportsMultibyteTraces query bool false "Filter by multibyte trace support (true/false); omit for no filter" // @Param neighbors query bool false "Include each node's known neighbor IDs (neighborIds field). Bare ?neighbors or ?neighbors=true enables it; omit/false for none" // @Param cursor query int false "last_seen epoch ms of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} api.Page[api.NodeSummary] // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -67,14 +67,10 @@ func listNodes(reader api.Reader) http.HandlerFunc { } else if typeName := r.URL.Query().Get("typeName"); typeName != "" { nodeType = api.NodeTypeFromString(typeName) } - var limit int32 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var cursor int64 if cursorParam := r.URL.Query().Get("cursor"); cursorParam != "" { @@ -184,7 +180,7 @@ func getNode(reader api.Reader) http.HandlerFunc { // @Produce json // @Param nodeId path string true "Node UUID" // @Param cursor query int false "Observation ID of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} api.Page[api.PacketObservationSummary] // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -205,14 +201,10 @@ func listNodeObservations(reader api.Reader) http.HandlerFunc { } cursor = c } - var limit int32 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } observations, err := reader.ListNodeObservations(r.Context(), nodeID, cursor, limit) if err != nil { diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 34ca2c61..d559cbdb 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -45,7 +45,7 @@ func ObserversRouter(reader api.Reader) http.Handler { // @Param name query string false "Partial case-insensitive display name match" // @Param scope query string false "Filter by transport scope name e.g. %23bc (URL-encoded #bc)" // @Param cursor query int false "last_seen epoch ms of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} api.Page[api.ObserverSummary] // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -66,14 +66,10 @@ func listObservers(reader api.Reader) http.HandlerFunc { } cursor = c } - var limit int32 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } iatas := parseIATAs(r) if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { @@ -127,7 +123,7 @@ func getObserver(reader api.Reader) http.HandlerFunc { // @Produce json // @Param observerId path string true "Observer UUID" // @Param cursor query int false "Observation ID of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} api.Page[api.AdvertObservation] // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -148,14 +144,10 @@ func listObserverAdverts(reader api.Reader) http.HandlerFunc { } cursor = c } - var limit int32 = 50 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } adverts, err := reader.ListObserverAdverts(r.Context(), observerID, cursor, limit) if err != nil { diff --git a/internal/api/handlers/packets.go b/internal/api/handlers/packets.go index 43e6bb71..40ea63c7 100644 --- a/internal/api/handlers/packets.go +++ b/internal/api/handlers/packets.go @@ -45,7 +45,7 @@ func PacketsRouter(reader api.Reader) http.Handler { // @Param since query int false "Filter by first_heard_at >= since (epoch ms)" // @Param until query int false "Filter by first_heard_at <= until (epoch ms)" // @Param cursor query int false "epoch ms of last item for pagination; last_heard_at, or site-local heard_at when iatas is set" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} object // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -93,14 +93,10 @@ func listPackets(reader api.Reader) http.HandlerFunc { } cursor = c } - var limit int32 = 50 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } iatas := parseIATAs(r) if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { @@ -134,7 +130,7 @@ func listPackets(reader api.Reader) http.HandlerFunc { // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" // @Param scope query string false "Filter by transport scope name" -// @Param limit query int false "Max results (default 100)" +// @Param limit query int false "Max results (default 100); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {object} []api.PacketSummary // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -151,14 +147,10 @@ func listPacketsBackfill(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "afterObservationId must be an integer") return } - var limit int32 = 100 - if limitParam := r.URL.Query().Get("limit"); limitParam != "" { - l, err := strconv.ParseInt(limitParam, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 100) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var payloadType int16 = -1 if v := r.URL.Query().Get("payloadType"); v != "" { diff --git a/internal/api/handlers/routes.go b/internal/api/handlers/routes.go index 9e819ac2..a5bdd76c 100644 --- a/internal/api/handlers/routes.go +++ b/internal/api/handlers/routes.go @@ -33,7 +33,8 @@ func RoutesRouter(reader api.Reader) http.Handler { // @Param iata query string false "Filter by IATA code" // @Param hopCount query int false "Filter by exact hop count" // @Param cursor query int false "Epoch ms timestamp of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) +// @Failure 400 {object} handlers.APIError // @Success 200 {object} []api.KnownRoute // @Failure 500 {object} handlers.APIError // @Router /routes [get] @@ -52,11 +53,10 @@ func listKnownRoutes(reader api.Reader) http.HandlerFunc { cursor = time.UnixMilli(ms) } } - var limit int32 = 50 - if v := r.URL.Query().Get("limit"); v != "" { - if l, err := strconv.ParseInt(v, 10, 32); err == nil { - limit = int32(l) - } + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } routes, err := reader.ListKnownRoutes(r.Context(), iata, hopCount, cursor, limit) if err != nil { diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index 474c9246..b59a3031 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -166,7 +166,7 @@ func getStatsPayloadBreakdown(reader api.Reader) http.HandlerFunc { // @Param iatas query string false "Comma-separated IATA codes" // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" -// @Param limit query int false "Max results (default 10)" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopNode // @Failure 500 {object} handlers.APIError // @Router /stats/top-nodes [get] @@ -181,14 +181,10 @@ func getStatsTopNodes(reader api.Reader) http.HandlerFunc { } iatas = append(iatas, regionIATAs...) } - var limit int32 = 10 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 10) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } nodes, err := reader.GetStatsTopNodes(r.Context(), iatas, limit) if err != nil { @@ -209,7 +205,7 @@ func getStatsTopNodes(reader api.Reader) http.HandlerFunc { // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param since query int false "Start of window epoch ms (default last 24h)" -// @Param limit query int false "Max results (default 10)" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopObserver // @Failure 500 {object} handlers.APIError // @Router /stats/top-observers [get] @@ -233,14 +229,10 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { } since = time.UnixMilli(ms) } - var limit int32 = 10 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 10) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } observers, err := reader.GetStatsTopObservers(r.Context(), iatas, since, limit) if err != nil { @@ -261,7 +253,7 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param since query int false "Start of window epoch ms (default last 24h)" -// @Param limit query int false "Max results (default 10)" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopAdvertiser // @Failure 500 {object} handlers.APIError // @Router /stats/top-advertisers [get] @@ -285,14 +277,10 @@ func getStatsTopAdvertisers(reader api.Reader) http.HandlerFunc { } since = time.UnixMilli(ms) } - var limit int32 = 10 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 10) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } advertisers, err := reader.GetStatsTopAdvertisers(r.Context(), iatas, since, limit) if err != nil { @@ -312,7 +300,7 @@ func getStatsTopAdvertisers(reader api.Reader) http.HandlerFunc { // @Param iatas query string false "Comma-separated IATA codes" // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" -// @Param limit query int false "Max results (default 10)" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.ClockDriftEntry // @Failure 500 {object} handlers.APIError // @Router /stats/clock-drift [get] @@ -327,14 +315,10 @@ func getStatsClockDrift(reader api.Reader) http.HandlerFunc { } iatas = append(iatas, regionIATAs...) } - var limit int32 = 10 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 10) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } entries, err := reader.GetStatsClockDrift(r.Context(), iatas, limit) if err != nil { @@ -355,7 +339,7 @@ func getStatsClockDrift(reader api.Reader) http.HandlerFunc { // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" // @Param since query int false "Start of window epoch ms (default last 24h)" -// @Param limit query int false "Max results (default 10)" +// @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopTalker // @Failure 500 {object} handlers.APIError // @Router /stats/top-talkers [get] @@ -379,14 +363,10 @@ func getStatsTopTalkers(reader api.Reader) http.HandlerFunc { } since = time.UnixMilli(ms) } - var limit int32 = 10 - if p := r.URL.Query().Get("limit"); p != "" { - l, err := strconv.ParseInt(p, 10, 32) - if err != nil { - respondError(w, http.StatusBadRequest, "limit must be an integer") - return - } - limit = int32(l) + limit, err := parseLimit(r, 10) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } talkers, err := reader.GetStatsTopTalkers(r.Context(), iatas, since, limit) if err != nil { diff --git a/internal/api/handlers/traces.go b/internal/api/handlers/traces.go index 3fd39ab5..7b2a7d47 100644 --- a/internal/api/handlers/traces.go +++ b/internal/api/handlers/traces.go @@ -37,17 +37,17 @@ func TracesRouter(reader api.Reader) http.Handler { // @Param since query int false "Filter by first_heard_at >= since (epoch ms)" // @Param until query int false "Filter by first_heard_at <= until (epoch ms)" // @Param cursor query int false "last_heard_at epoch ms of last item for pagination" -// @Param limit query int false "Max results (default 50)" +// @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) +// @Failure 400 {object} handlers.APIError // @Success 200 {object} []api.TraceTagSummary // @Failure 500 {object} handlers.APIError // @Router /traces [get] func listTraceTags(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - var limit int32 = 50 - if l := r.URL.Query().Get("limit"); l != "" { - if v, err := strconv.ParseInt(l, 10, 32); err == nil { - limit = int32(v) - } + limit, err := parseLimit(r, 50) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } var since, until, cursor time.Time if v := r.URL.Query().Get("since"); v != "" { From 69e7e196af372cc9399f3d54e65f95d022d7f892 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 10:21:18 -0400 Subject: [PATCH 014/141] fix(db): tolerate packets before their first observation (#115) --- db/packets_integration_test.go | 79 ++++++++++++++++++++++++++++++++++ db/queries/queries.sql | 18 ++++---- db/sqlc/queries.sql.go | 18 ++++---- 3 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 db/packets_integration_test.go diff --git a/db/packets_integration_test.go b/db/packets_integration_test.go new file mode 100644 index 00000000..10521e36 --- /dev/null +++ b/db/packets_integration_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +// The packet and its observation are separate ingest writes. A list request +// between them must return the packet without inventing an observer or failing. +func TestListPacketsBeforeObservationPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"packets", "packet_observations", "observers", "transport_scopes"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ('\x01', 4, 0, 1, '\x00', '\x00', NOW(), NOW())`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil { + t.Fatalf("packet without observation must remain readable: %v", err) + } + if len(page.Items) != 1 || page.Items[0].PacketHash != "01" { + t.Fatalf("packet missing: %+v", page) + } + if page.Items[0].LatestObserver != nil || page.Items[0].ObservationCount != 0 { + t.Fatalf("invented observation: %+v", page.Items[0]) + } + _, err = tx.Exec(ctx, ` +INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x02'); +INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes) +VALUES (1, '\x01', '00000000-0000-0000-0000-000000000001', 'YVR', NOW(), 2, 1, 2, '\x1122');`) + if err != nil { + t.Fatal(err) + } + for _, iatas := range [][]string{nil, {"YVR"}} { + page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].ObservationCount != 1 { + t.Fatalf("observed packet missing for %v: %+v", iatas, page) + } + observer := page.Items[0].LatestObserver + if observer == nil || observer.IATA != "YVR" || observer.PathLength == nil || observer.PathBytes == nil { + t.Fatalf("observation metadata missing: %+v", observer) + } + if observer.PathLength.Raw != "02" || observer.PathLength.HashSize != 1 || observer.PathLength.HopCount != 2 || *observer.PathBytes != "1122" { + t.Fatalf("observation path changed: %+v", observer) + } + } +} diff --git a/db/queries/queries.sql b/db/queries/queries.sql index b6213db3..f637b299 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -391,12 +391,13 @@ SELECT p.scope_id, ts.name AS scope_name, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes FROM packets p LEFT JOIN LATERAL ( @@ -487,12 +488,13 @@ SELECT sat.scan_saturated, sat.scan_floor, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes FROM page sh CROSS JOIN saturation sat diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 670f7adf..7fe72e45 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2694,12 +2694,13 @@ SELECT p.scope_id, ts.name AS scope_name, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes FROM packets p LEFT JOIN LATERAL ( @@ -2952,12 +2953,13 @@ SELECT sat.scan_saturated, sat.scan_floor, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, - po.iata AS latest_observer_iata, - po.path_length_byte AS latest_observer_path_length_byte, - po.hash_size AS latest_observer_hash_size, - po.hop_count AS latest_observer_hop_count, + COALESCE(po.iata, ''::bpchar) AS latest_observer_iata, + COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, + COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, + COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes FROM page sh CROSS JOIN saturation sat From a78ea368a3842231e851e2c0ae54d21303fe7808 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 10:21:45 -0400 Subject: [PATCH 015/141] fix(background): preserve refresh failures in task outcomes (#113) --- internal/background/background.go | 5 +- internal/background/background_test.go | 156 +++++++++++++------------ internal/background/tasks.go | 52 +++++---- internal/background/tasks_test.go | 116 ++++++++++++++++++ 4 files changed, 231 insertions(+), 98 deletions(-) create mode 100644 internal/background/tasks_test.go diff --git a/internal/background/background.go b/internal/background/background.go index 5e8c1896..1ddf322f 100644 --- a/internal/background/background.go +++ b/internal/background/background.go @@ -38,9 +38,10 @@ func (s *Scheduler) Start(ctx context.Context) { case <-ticker.C: log.Printf("background[%s]: running", t.Name) if err := t.Run(ctx); err != nil { - log.Printf("background[%s]: %v", t.Name, err) + log.Printf("background[%s]: failed: %v", t.Name, err) + } else { + log.Printf("background[%s]: complete", t.Name) } - log.Printf("background[%s]: complete", t.Name) case <-ctx.Done(): return } diff --git a/internal/background/background_test.go b/internal/background/background_test.go index 18fbe851..f4978776 100644 --- a/internal/background/background_test.go +++ b/internal/background/background_test.go @@ -7,98 +7,108 @@ import ( "context" "sync/atomic" "testing" + "testing/synctest" "time" ) func TestScheduler_RunsTask(t *testing.T) { - var calls atomic.Int32 - - tasks := []Task{ - { - Name: "test-task", - Interval: 10 * time.Millisecond, - Run: func(ctx context.Context) error { - calls.Add(1) - return nil + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + + tasks := []Task{ + { + Name: "test-task", + Interval: 10 * time.Millisecond, + Run: func(ctx context.Context) error { + calls.Add(1) + return nil + }, }, - }, - } + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - s := New(tasks) - go s.Start(ctx) + s := New(tasks) + go s.Start(ctx) - time.Sleep(50 * time.Millisecond) - if calls.Load() == 0 { - t.Error("expected task to have run at least once") - } + time.Sleep(50 * time.Millisecond) + synctest.Wait() + if calls.Load() == 0 { + t.Error("expected task to have run at least once") + } + }) } func TestScheduler_StopsOnContextCancel(t *testing.T) { - var calls atomic.Int32 - - tasks := []Task{ - { - Name: "test-task", - Interval: 10 * time.Millisecond, - Run: func(ctx context.Context) error { - calls.Add(1) - return nil + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + + tasks := []Task{ + { + Name: "test-task", + Interval: 10 * time.Millisecond, + Run: func(ctx context.Context) error { + calls.Add(1) + return nil + }, }, - }, - } + } - ctx, cancel := context.WithCancel(context.Background()) - s := New(tasks) - go s.Start(ctx) + ctx, cancel := context.WithCancel(context.Background()) + s := New(tasks) + go s.Start(ctx) - time.Sleep(30 * time.Millisecond) - cancel() - time.Sleep(10 * time.Millisecond) + time.Sleep(30 * time.Millisecond) + synctest.Wait() + cancel() + synctest.Wait() - snapshot := calls.Load() - time.Sleep(30 * time.Millisecond) + snapshot := calls.Load() + time.Sleep(30 * time.Millisecond) - if calls.Load() != snapshot { - t.Errorf("expected task to stop after cancel, got %d more calls", calls.Load()-snapshot) - } + if calls.Load() != snapshot { + t.Errorf("expected task to stop after cancel, got %d more calls", calls.Load()-snapshot) + } + }) } func TestScheduler_MultipleTasksRunIndependently(t *testing.T) { - var calls1, calls2 atomic.Int32 - - tasks := []Task{ - { - Name: "task1", - Interval: 10 * time.Millisecond, - Run: func(ctx context.Context) error { - calls1.Add(1) - return nil + synctest.Test(t, func(t *testing.T) { + var calls1, calls2 atomic.Int32 + + tasks := []Task{ + { + Name: "task1", + Interval: 10 * time.Millisecond, + Run: func(ctx context.Context) error { + calls1.Add(1) + return nil + }, }, - }, - { - Name: "task2", - Interval: 10 * time.Millisecond, - Run: func(ctx context.Context) error { - calls2.Add(1) - return nil + { + Name: "task2", + Interval: 10 * time.Millisecond, + Run: func(ctx context.Context) error { + calls2.Add(1) + return nil + }, }, - }, - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - s := New(tasks) - go s.Start(ctx) - - time.Sleep(50 * time.Millisecond) - if calls1.Load() == 0 { - t.Error("expected task1 to have run") - } - if calls2.Load() == 0 { - t.Error("expected task2 to have run") - } + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + s := New(tasks) + go s.Start(ctx) + + time.Sleep(50 * time.Millisecond) + synctest.Wait() + if calls1.Load() == 0 { + t.Error("expected task1 to have run") + } + if calls2.Load() == 0 { + t.Error("expected task2 to have run") + } + }) } diff --git a/internal/background/tasks.go b/internal/background/tasks.go index 76c1cf81..f2f3ec70 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -5,41 +5,47 @@ package background import ( "context" + "errors" "fmt" - "log" "time" "github.com/MeshCore-Beacon/beacon-server/db" ) +type viewRefresher interface { + RefreshHourlyStats(context.Context) error + RefreshTopNodes(context.Context) error + RefreshTopObservers(context.Context) error + RefreshPayloadBreakdown(context.Context) error + RefreshTopTalkers(context.Context) error + RefreshTopAdvertisers(context.Context) error + RefreshRadioPresets(context.Context) error +} + // ViewRefreshTask returns a Task that refreshes all materialized views. -func ViewRefreshTask(store *db.Store, interval time.Duration) Task { +func ViewRefreshTask(store viewRefresher, interval time.Duration) Task { return Task{ Name: "view_refresh", Interval: interval, Run: func(ctx context.Context) error { - if err := store.RefreshHourlyStats(ctx); err != nil { - log.Printf("background[view_refresh]: hourly stats: %v", err) - } - if err := store.RefreshTopNodes(ctx); err != nil { - log.Printf("background[view_refresh]: top nodes: %v", err) - } - if err := store.RefreshTopObservers(ctx); err != nil { - log.Printf("background[view_refresh]: top observers: %v", err) - } - if err := store.RefreshPayloadBreakdown(ctx); err != nil { - log.Printf("background[view_refresh]: payload breakdown: %v", err) + var errs []error + for _, view := range []struct { + name string + refresh func(context.Context) error + }{ + {"hourly stats", store.RefreshHourlyStats}, + {"top nodes", store.RefreshTopNodes}, + {"top observers", store.RefreshTopObservers}, + {"payload breakdown", store.RefreshPayloadBreakdown}, + {"top talkers", store.RefreshTopTalkers}, + {"top advertisers", store.RefreshTopAdvertisers}, + {"radio presets", store.RefreshRadioPresets}, + } { + if err := view.refresh(ctx); err != nil { + errs = append(errs, fmt.Errorf("%s: %w", view.name, err)) + } } - if err := store.RefreshTopTalkers(ctx); err != nil { - log.Printf("background[view_refresh]: top talkers: %v", err) - } - if err := store.RefreshTopAdvertisers(ctx); err != nil { - log.Printf("background[view_refresh]: top advertisers: %v", err) - } - if err := store.RefreshRadioPresets(ctx); err != nil { - log.Printf("background[view_refresh]: radio presets: %v", err) - } - return nil + return errors.Join(errs...) }, } } diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go new file mode 100644 index 00000000..cee90ff5 --- /dev/null +++ b/internal/background/tasks_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "log" + "strings" + "testing" + "time" +) + +type refreshStub struct { + calls []string + errs map[string]error +} + +func (s *refreshStub) refresh(ctx context.Context, name string) error { + s.calls = append(s.calls, name) + if err := ctx.Err(); err != nil { + return err + } + return s.errs[name] +} +func (s *refreshStub) RefreshHourlyStats(ctx context.Context) error { + return s.refresh(ctx, "hourly stats") +} +func (s *refreshStub) RefreshTopNodes(ctx context.Context) error { return s.refresh(ctx, "top nodes") } +func (s *refreshStub) RefreshTopObservers(ctx context.Context) error { + return s.refresh(ctx, "top observers") +} +func (s *refreshStub) RefreshPayloadBreakdown(ctx context.Context) error { + return s.refresh(ctx, "payload breakdown") +} +func (s *refreshStub) RefreshTopTalkers(ctx context.Context) error { + return s.refresh(ctx, "top talkers") +} +func (s *refreshStub) RefreshTopAdvertisers(ctx context.Context) error { + return s.refresh(ctx, "top advertisers") +} +func (s *refreshStub) RefreshRadioPresets(ctx context.Context) error { + return s.refresh(ctx, "radio presets") +} + +func TestViewRefreshTask(t *testing.T) { + first, second := errors.New("first failure"), errors.New("second failure") + for _, tc := range []struct { + name string + errs map[string]error + }{ + {"success", nil}, + {"partial failure", map[string]error{"hourly stats": first, "top talkers": second}}, + } { + t.Run(tc.name, func(t *testing.T) { + store := &refreshStub{errs: tc.errs} + task := ViewRefreshTask(store, time.Minute) + err := task.Run(context.Background()) + if len(store.calls) != 7 { + t.Fatalf("refreshed %d views, want 7", len(store.calls)) + } + if len(tc.errs) == 0 && err != nil { + t.Fatal(err) + } + for name, cause := range tc.errs { + if !errors.Is(err, cause) { + t.Errorf("missing cause %v in %v", cause, err) + } + if err == nil || !strings.Contains(err.Error(), name) { + t.Errorf("missing view name %q in %v", name, err) + } + } + }) + } +} + +func TestViewRefreshTaskCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := ViewRefreshTask(&refreshStub{}, time.Minute).Run(ctx) + if !errors.Is(err, context.Canceled) { + t.Fatalf("got %v, want cancellation", err) + } +} + +type taskLog chan string + +func (w taskLog) Write(p []byte) (int, error) { + line := string(p) + if strings.Contains(line, "complete") || strings.Contains(line, "failed:") { + w <- line + } + return len(p), nil +} + +func TestSchedulerFailureIsNotComplete(t *testing.T) { + lines := make(taskLog, 1) + previous := log.Writer() + log.SetOutput(lines) + defer log.SetOutput(previous) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + New([]Task{{Name: "fails", Interval: time.Millisecond, Run: func(context.Context) error { + cancel() + return errors.New("refresh unavailable") + }}}).Start(ctx) + select { + case line := <-lines: + if strings.Contains(line, "complete") || !strings.Contains(line, "refresh unavailable") { + t.Fatalf("incorrect failure status: %s", line) + } + case <-time.After(time.Second): + t.Fatal("scheduler did not report task outcome") + } +} From f66d43106b0d2f1f984f561c3f83817ae2f4b901 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 9 Sep 2026 18:44:12 -0400 Subject: [PATCH 016/141] fix(ingest): isolate MQTT client IDs across workers (#117) --- internal/ingest/ingest.go | 6 +- internal/ingest/mqtt_test.go | 141 +++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 internal/ingest/mqtt_test.go diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 19b80e28..16743f6b 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -37,9 +37,9 @@ package ingest import ( "context" + "crypto/rand" "encoding/binary" "encoding/json" - "fmt" "log" "strconv" "strings" @@ -234,9 +234,11 @@ func New(cfg Config, db DB, h *hub.Hub, keys ChannelKeyStore, scopes ScopeStore) // // Intended usage: go worker.Start(ctx) func (w *Worker) Start(ctx context.Context) { + // Isolate workers across deployments; Paho reuses this ID on reconnect. + // Keep it alphanumeric and within MQTT 3.1's 23-character client ID limit. opts := mqtt.NewClientOptions(). AddBroker(w.cfg.URL). - SetClientID(fmt.Sprintf("beacon-%s", w.cfg.BrokerName)). + SetClientID(rand.Text()[:23]). SetUsername(w.cfg.Username). SetPassword(w.cfg.Password). SetAutoReconnect(true). diff --git a/internal/ingest/mqtt_test.go b/internal/ingest/mqtt_test.go new file mode 100644 index 00000000..6cde148a --- /dev/null +++ b/internal/ingest/mqtt_test.go @@ -0,0 +1,141 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "net" + "regexp" + "sync" + "testing" + "time" + + "github.com/eclipse/paho.mqtt.golang/packets" +) + +type mqttConnection struct { + id string + conn net.Conn +} + +// A local protocol fixture observes the actual CONNECT packets sent by Start. +// It neither publishes nor connects to a real broker. +func mqttTestBroker(t *testing.T) (string, <-chan mqttConnection) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + connected := make(chan mqttConnection, 4) + var clients sync.WaitGroup + stopped := make(chan struct{}) + go func() { + defer close(stopped) + for { + conn, err := listener.Accept() + if err != nil { + return + } + clients.Add(1) + go func() { + defer clients.Done() + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) + first, err := packets.ReadPacket(conn) + if err != nil { + return + } + connect, ok := first.(*packets.ConnectPacket) + if !ok { + return + } + if err := packets.NewControlPacket(packets.Connack).Write(conn); err != nil { + return + } + announced := false + for { + packet, err := packets.ReadPacket(conn) + if err != nil { + return + } + switch packet := packet.(type) { + case *packets.SubscribePacket: + ack := packets.NewControlPacket(packets.Suback).(*packets.SubackPacket) + ack.MessageID = packet.MessageID + ack.ReturnCodes = []byte{1} + if err := ack.Write(conn); err != nil { + return + } + if !announced { + connected <- mqttConnection{connect.ClientIdentifier, conn} + announced = true + } + case *packets.PingreqPacket: + if err := packets.NewControlPacket(packets.Pingresp).Write(conn); err != nil { + return + } + case *packets.DisconnectPacket: + return + } + } + }() + } + }() + t.Cleanup(func() { _ = listener.Close(); <-stopped; clients.Wait() }) + return "tcp://" + listener.Addr().String(), connected +} + +func startMQTTTestWorker(t *testing.T, address string) { + t.Helper() + w, _ := newTestWorker() + w.cfg.URL = address + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); w.Start(ctx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Error("worker did not stop") + } + }) +} + +func awaitMQTTConnection(t *testing.T, connected <-chan mqttConnection) mqttConnection { + t.Helper() + select { + case c := <-connected: + return c + case <-time.After(5 * time.Second): + t.Fatal("MQTT CONNECT not received") + return mqttConnection{} + } +} + +func TestWorkersUseDistinctMQTTClientIDs(t *testing.T) { + address, connected := mqttTestBroker(t) + startMQTTTestWorker(t, address) + first := awaitMQTTConnection(t, connected) + startMQTTTestWorker(t, address) + second := awaitMQTTConnection(t, connected) + if first.id == "" || second.id == "" || first.id == second.id { + t.Fatalf("workers with the same broker name must use distinct client IDs: %q and %q", first.id, second.id) + } + valid := regexp.MustCompile(`^[A-Za-z0-9]{1,23}$`) + if !valid.MatchString(first.id) || !valid.MatchString(second.id) { + t.Fatal("client ID must be 1..23 alphanumeric characters") + } +} + +func TestReconnectKeepsMQTTClientID(t *testing.T) { + address, connected := mqttTestBroker(t) + startMQTTTestWorker(t, address) + first := awaitMQTTConnection(t, connected) + _ = first.conn.Close() + second := awaitMQTTConnection(t, connected) + if first.id != second.id { + t.Fatalf("reconnect changed client ID: %q to %q", first.id, second.id) + } +} From e234965f1915d652c691436b50090a47c7362e53 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 9 Sep 2026 18:44:27 -0400 Subject: [PATCH 017/141] fix(db): paginate complete trace tag groups (#124) --- db/queries/queries.sql | 2 +- db/sqlc/queries.sql.go | 2 +- db/trace_pagination_integration_test.go | 138 ++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 db/trace_pagination_integration_test.go diff --git a/db/queries/queries.sql b/db/queries/queries.sql index f637b299..619f8a44 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1076,9 +1076,9 @@ WITH tags AS ( AND ($2::text = '' OR p.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) AND ($3::timestamptz IS NULL OR p.first_heard_at >= $3) AND ($4::timestamptz IS NULL OR p.first_heard_at <= $4) - AND ($5::timestamptz IS NULL OR p.last_heard_at < $5) AND ($7::text = '' OR p.parsed_payload->>'type' = $7) GROUP BY p.trace_tag + HAVING ($5::timestamptz IS NULL OR MAX(p.last_heard_at) < $5) ORDER BY MAX(p.last_heard_at) DESC LIMIT $6 ) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 7fe72e45..68879b2f 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -3123,9 +3123,9 @@ WITH tags AS ( AND ($2::text = '' OR p.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) AND ($3::timestamptz IS NULL OR p.first_heard_at >= $3) AND ($4::timestamptz IS NULL OR p.first_heard_at <= $4) - AND ($5::timestamptz IS NULL OR p.last_heard_at < $5) AND ($7::text = '' OR p.parsed_payload->>'type' = $7) GROUP BY p.trace_tag + HAVING ($5::timestamptz IS NULL OR MAX(p.last_heard_at) < $5) ORDER BY MAX(p.last_heard_at) DESC LIMIT $6 ) diff --git a/db/trace_pagination_integration_test.go b/db/trace_pagination_integration_test.go new file mode 100644 index 00000000..b0ca1405 --- /dev/null +++ b/db/trace_pagination_integration_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "reflect" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +// Trace tags, not individual packets, are the entities being paginated. All +// fixture writes use temporary tables and are rolled back on exit. +func TestTraceTagPaginationPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"packets", "trace_iatas", "transport_scopes"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) VALUES + (1,'scope-one',decode(repeat('01',16),'hex'),decode(repeat('01',8),'hex')), + (2,'scope-two',decode(repeat('02',16),'hex'),decode(repeat('02',8),'hex')); +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,trace_tag,scope_id,first_heard_at,last_heard_at,parsed_payload) +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,1,'\x00','\x00',decode(tag,'hex'),scope, + '2026-01-01'::timestamptz+first*interval '1 second','2026-01-01'::timestamptz+last*interval '1 second',payload::jsonb +FROM (VALUES + (1,'aaaaaaaa',1,1,40,'{"type":"TRACE","pathHashes":["aa"],"snrValues":[1]}'), + (2,'aaaaaaaa',1,2,10,'{"type":"TRACE","pathHashes":["aa","bb"],"snrValues":[1,2]}'), + (3,'aaaaaaaa',2,3,25,'{"type":"PING","pathHashes":["aa","bb","cc"],"snrValues":[1,2,3]}'), + (4,'bbbbbbbb',1,4,30,'{"type":"TRACE","pathHashes":["bb","11","22","33"],"snrValues":[2,3,4,5]}'), + (5,'bbbbbbbb',2,5,5,'{"type":"PING","pathHashes":["bb"],"snrValues":[2]}'), + (6,'cccccccc',1,6,20,'{"type":"TRACE","pathHashes":["cc"],"snrValues":[3]}'), + (7,'dddddddd',2,7,15,'{"type":"PING","pathHashes":["dd"],"snrValues":[4]}'), + (8,NULL,1,8,50,'{"type":"TRACE","pathHashes":[],"snrValues":[]}') +) v(i,tag,scope,first,last,payload); +INSERT INTO trace_iatas (trace_tag,iata) VALUES + ('\xaaaaaaaa','YYZ'),('\xaaaaaaaa','YVR'),('\xbbbbbbbb','YYZ'),('\xcccccccc','YVR');`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + at := func(seconds int) time.Time { + if seconds == 0 { + return time.Time{} + } + return anchor.Add(time.Duration(seconds) * time.Second) + } + paths := map[string][]string{"a": {"aa", "bb", "cc"}, "b": {"bb", "11", "22", "33"}, "c": {"cc"}, "d": {"dd"}} + snrs := map[string][]float32{"a": {1, 2, 3}, "b": {2, 3, 4, 5}, "c": {3}, "d": {4}} + summary := func(tag string, first, last int, count, iatas int64, kind string) api.TraceTagSummary { + return api.TraceTagSummary{TraceTag: strings.Repeat(tag, 8), FirstHeardAt: at(first).UnixMilli(), LastHeardAt: at(last).UnixMilli(), PacketCount: count, IATACount: iatas, TraceType: kind, PathHashes: paths[tag], SNRValues: snrs[tag]} + } + a, b, c, d := summary("a", 1, 40, 3, 2, "TRACE"), summary("b", 4, 30, 2, 1, "TRACE"), summary("c", 6, 20, 1, 1, "TRACE"), summary("d", 7, 15, 1, 0, "PING") + for _, tc := range []struct { + name string + iatas []string + scope, kind string + since, until, cursor int + want []api.TraceTagSummary + }{ + {"first page", nil, "", "", 0, 0, 0, []api.TraceTagSummary{a, b, c, d}}, + {"cursor at newest group", nil, "", "", 0, 0, 40, []api.TraceTagSummary{b, c, d}}, + {"cursor between packet times", nil, "", "", 0, 0, 30, []api.TraceTagSummary{c, d}}, + {"end of groups", nil, "", "", 0, 0, 15, []api.TraceTagSummary{}}, + {"region", []string{"YYZ"}, "", "", 0, 0, 35, []api.TraceTagSummary{b}}, + {"multiple regions", []string{"YYZ", "YVR", "YYZ"}, "", "", 0, 0, 35, []api.TraceTagSummary{b, c}}, + {"unknown region", []string{"ZZZ"}, "", "", 0, 0, 0, []api.TraceTagSummary{}}, + {"scope", nil, "scope-one", "", 0, 0, 30, []api.TraceTagSummary{c}}, + {"scope defines group", nil, "scope-two", "", 0, 0, 20, []api.TraceTagSummary{d, summary("b", 5, 5, 1, 1, "PING")}}, + {"type", nil, "", "TRACE", 0, 0, 35, []api.TraceTagSummary{summary("b", 4, 30, 1, 1, "TRACE"), c}}, + {"type defines group", nil, "", "PING", 0, 0, 30, []api.TraceTagSummary{summary("a", 3, 25, 1, 2, "PING"), d, summary("b", 5, 5, 1, 1, "PING")}}, + {"time window defines group", nil, "", "", 3, 6, 26, []api.TraceTagSummary{summary("a", 3, 25, 1, 2, "PING"), c}}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := store.ListTraceTags(ctx, tc.iatas, tc.scope, tc.kind, at(tc.since), at(tc.until), at(tc.cursor), 20) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("trace summaries differ: got %+v, want %+v", got, tc.want) + } + }) + } + // Reassemble pages using the same epoch-millisecond cursor exposed by the API. + reference := map[string]api.TraceTagSummary{a.TraceTag: a, b.TraceTag: b, c.TraceTag: c, d.TraceTag: d} + seen := map[string]bool{} + var cursor time.Time + total := 0 + for page := 0; page < 10; page++ { + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, cursor, 2) + if err != nil { + t.Fatal(err) + } + if len(rows) == 0 { + break + } + for _, row := range rows { + total++ + if seen[row.TraceTag] { + t.Errorf("pagination repeated trace tag %s", row.TraceTag) + } + seen[row.TraceTag] = true + if !reflect.DeepEqual(row, reference[row.TraceTag]) { + t.Errorf("pagination changed summary for %s", row.TraceTag) + } + } + cursor = time.UnixMilli(rows[len(rows)-1].LastHeardAt) + } + t.Logf("two-row pages returned %d summaries for %d distinct trace tags", total, len(seen)) + if total != 4 || len(seen) != 4 { + t.Errorf("expected exactly four complete trace summaries") + } +} From 04d059d7b3362eb5550535d7b8cbdc1e3cb2644e Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 9 Sep 2026 18:44:39 -0400 Subject: [PATCH 018/141] perf(db): enrich clock-drift nodes after selecting the page (#122) --- db/clock_drift_integration_test.go | 191 +++++++++++++++++++++++++++++ db/queries/queries.sql | 15 ++- db/sqlc/querier.go | 1 + db/sqlc/queries.sql.go | 13 +- 4 files changed, 211 insertions(+), 9 deletions(-) create mode 100644 db/clock_drift_integration_test.go diff --git a/db/clock_drift_integration_test.go b/db/clock_drift_integration_test.go new file mode 100644 index 00000000..fbe0ddc7 --- /dev/null +++ b/db/clock_drift_integration_test.go @@ -0,0 +1,191 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "os" + "slices" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +//go:embed queries/queries.sql +var clockTestQueries string + +// Fixture writes are confined to temporary tables in a rolled-back transaction. +func TestClockDriftPagePostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"nodes", "node_iatas"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO nodes (id,public_key,node_type,name,device_clock_drift_seconds,last_advert_at) +SELECT ('00000000-0000-0000-0000-'||lpad(i::text,12,'0'))::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),kind, + CASE WHEN i=2 THEN NULL ELSE 'node-'||i END,drift,CASE WHEN i=2 THEN NULL ELSE '2026-01-01'::timestamptz END +FROM (VALUES (1,2,120),(2,3,-240),(3,1,99999),(4,4,-99999),(5,2,NULL),(6,2,60),(7,3,-60),(8,2,180)) v(i,kind,drift); +INSERT INTO node_iatas (node_id,iata,last_heard) VALUES + ('00000000-0000-0000-0000-000000000001','YVR','2026-01-01'), + ('00000000-0000-0000-0000-000000000001','YYJ','2026-01-01 01:00:00+00'), + ('00000000-0000-0000-0000-000000000002','YYJ','2026-01-01');`) + if err != nil { + t.Fatal(err) + } + q := sqlc.New(tx) + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + iatas []string + threshold, limit int32 + want []int + }{ + {"global", nil, 60, 10, []int{2, 8, 1}}, + {"empty region array", []string{}, 60, 10, []int{2, 8, 1}}, + {"page limit", nil, 60, 2, []int{2, 8}}, + {"YVR", []string{"YVR"}, 60, 10, []int{1}}, + {"YYJ", []string{"YYJ"}, 60, 10, []int{2, 1}}, + {"multiple regions", []string{"YVR", "YYJ"}, 60, 10, []int{2, 1}}, + {"duplicate region", []string{"YYJ", "YYJ"}, 60, 10, []int{2, 1}}, + {"unknown region", []string{"ZZZ"}, 60, 10, nil}, + {"strict threshold", nil, 180, 10, []int{2}}, + {"boundary", nil, 240, 10, nil}, + {"positive and negative boundary", nil, 0, 10, []int{2, 8, 1, 6, 7}}, + } { + t.Run(tc.name, func(t *testing.T) { + rows, err := q.GetStatsClockDrift(ctx, sqlc.GetStatsClockDriftParams{Column1: tc.threshold, Column2: tc.iatas, Limit: tc.limit}) + if err != nil { + t.Fatal(err) + } + if len(rows) != len(tc.want) { + t.Fatalf("got %d rows, want %d", len(rows), len(tc.want)) + } + previous := int64(1 << 62) + seen := map[int]bool{} + for _, row := range rows { + var number int + if _, err := fmt.Sscanf(row.ID.String(), "00000000-0000-0000-0000-%d", &number); err != nil { + t.Fatal(err) + } + if !slices.Contains(tc.want, number) || seen[number] { + t.Fatalf("unexpected/duplicate node %d", number) + } + seen[number] = true + if row.DeviceClockDriftSeconds == nil { + t.Fatal("null drift returned") + } + wantDrift := map[int]int32{1: 120, 2: -240, 6: 60, 7: -60, 8: 180}[number] + if *row.DeviceClockDriftSeconds != wantDrift || (row.NodeType != 2 && row.NodeType != 3) { + t.Fatal("drift sign or role changed") + } + magnitude := int64(wantDrift) + if magnitude < 0 { + magnitude = -magnitude + } + if magnitude > previous { + t.Fatal("not ordered worst first") + } + previous = magnitude + if number == 2 { + if row.Name != nil || row.LastAdvertAt.Valid { + t.Fatal("nullable node fields changed") + } + } else if row.Name == nil || *row.Name != fmt.Sprintf("node-%d", number) || !row.LastAdvertAt.Valid || !row.LastAdvertAt.Time.Equal(anchor) { + t.Fatal("node fields changed") + } + var got []api.NodeIATA + if len(row.Iatas) > 0 { + if err := json.Unmarshal(row.Iatas, &got); err != nil { + t.Fatal(err) + } + } + var want []api.NodeIATA + if number == 1 { + want = []api.NodeIATA{{IATA: "YYJ", LastHeard: anchor.Add(time.Hour).UnixMilli()}, {IATA: "YVR", LastHeard: anchor.UnixMilli()}} + } + if number == 2 { + want = []api.NodeIATA{{IATA: "YYJ", LastHeard: anchor.UnixMilli()}} + } + if !slices.Equal(got, want) { + t.Fatalf("node %d memberships/order: got %v, want %v", number, got, want) + } + } + }) + } + // Use a separate large population for the work-budget measurement. + _, err = tx.Exec(ctx, ` +TRUNCATE pg_temp.node_iatas,pg_temp.nodes; +INSERT INTO nodes (id,public_key,node_type,name,device_clock_drift_seconds,last_advert_at) +SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),2,'clock-'||i,3600+i,'2026-01-01' +FROM generate_series(1,20000) i; +INSERT INTO node_iatas (node_id,iata,last_heard) +SELECT md5(i::text)::uuid,iata,'2026-01-01'::timestamptz-i*interval '1 second' +FROM generate_series(1,20000) i CROSS JOIN (VALUES ('YVR'),('YYJ')) regions(iata); +ANALYZE nodes; ANALYZE node_iatas;`) + if err != nil { + t.Fatal(err) + } + queries := strings.ReplaceAll(clockTestQueries, "\r\n", "\n") + query := strings.Split(strings.Split(queries, "-- name: GetStatsClockDrift :many\n")[1], "\n-- name:")[0] + if _, err := tx.Exec(ctx, "PREPARE clock_page AS "+query); err != nil { + t.Fatal(err) + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err := tx.Exec(ctx, "SET LOCAL plan_cache_mode = "+mode); err != nil { + t.Fatal(err) + } + var raw []byte + if err := tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE clock_page(60,NULL,50)").Scan(&raw); err != nil { + t.Fatal(err) + } + var plans []struct { + Plan map[string]any + ExecutionTime float64 `json:"Execution Time"` + } + if err := json.Unmarshal(raw, &plans); err != nil { + t.Fatal(err) + } + var memberships float64 + var walk func(map[string]any) + walk = func(p map[string]any) { + if p["Relation Name"] == "node_iatas" { + memberships += p["Actual Rows"].(float64) * p["Actual Loops"].(float64) + } + if children, ok := p["Plans"].([]any); ok { + for _, child := range children { + walk(child.(map[string]any)) + } + } + } + walk(plans[0].Plan) + t.Logf("20,000 nodes / 50-row clock page %s: %.3f ms, %.0f membership rows processed", mode, plans[0].ExecutionTime, memberships) + if memberships > 150 { + t.Errorf("clock page processed %.0f membership rows; budget 150", memberships) + } + } +} diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 619f8a44..68b8427e 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -964,22 +964,27 @@ LIMIT $3; -- Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds -- the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's -- latest measured drift, not an aggregate over a period. +-- Select the filtered page before aggregating its IATA memberships. +WITH page AS ( SELECT n.id, n.name, n.node_type, n.device_clock_drift_seconds, - n.last_advert_at, - json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) FILTER (WHERE ni.iata IS NOT NULL) AS iatas + n.last_advert_at FROM nodes n -LEFT JOIN node_iatas ni ON ni.node_id = n.id WHERE n.node_type IN (2, 3) AND n.device_clock_drift_seconds IS NOT NULL AND ABS(n.device_clock_drift_seconds) > $1::int AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) -GROUP BY n.id, n.name, n.node_type, n.device_clock_drift_seconds, n.last_advert_at ORDER BY ABS(n.device_clock_drift_seconds) DESC -LIMIT $3; +LIMIT $3 +) +SELECT n.id, n.name, n.node_type, n.device_clock_drift_seconds, n.last_advert_at, + (SELECT json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) + FROM node_iatas ni WHERE ni.node_id = n.id) AS iatas +FROM page n +ORDER BY ABS(n.device_clock_drift_seconds) DESC; -- name: GetStatsTopTalkers :many -- Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 9e2ab2b8..bbca860c 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -72,6 +72,7 @@ type Querier interface { // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds // the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's // latest measured drift, not an aggregate over a period. + // Select the filtered page before aggregating its IATA memberships. GetStatsClockDrift(ctx context.Context, arg GetStatsClockDriftParams) ([]GetStatsClockDriftRow, error) // Returns node counts grouped by type, optionally filtered by IATA. GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]GetStatsNodeTypesRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 68879b2f..1c7833ff 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1190,22 +1190,26 @@ func (q *Queries) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]Ge } const getStatsClockDrift = `-- name: GetStatsClockDrift :many +WITH page AS ( SELECT n.id, n.name, n.node_type, n.device_clock_drift_seconds, - n.last_advert_at, - json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) FILTER (WHERE ni.iata IS NOT NULL) AS iatas + n.last_advert_at FROM nodes n -LEFT JOIN node_iatas ni ON ni.node_id = n.id WHERE n.node_type IN (2, 3) AND n.device_clock_drift_seconds IS NOT NULL AND ABS(n.device_clock_drift_seconds) > $1::int AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) -GROUP BY n.id, n.name, n.node_type, n.device_clock_drift_seconds, n.last_advert_at ORDER BY ABS(n.device_clock_drift_seconds) DESC LIMIT $3 +) +SELECT n.id, n.name, n.node_type, n.device_clock_drift_seconds, n.last_advert_at, + (SELECT json_agg(json_build_object('iata', ni.iata, 'lastHeard', (extract(epoch from ni.last_heard) * 1000)::bigint) ORDER BY ni.last_heard DESC) + FROM node_iatas ni WHERE ni.node_id = n.id) AS iatas +FROM page n +ORDER BY ABS(n.device_clock_drift_seconds) DESC ` type GetStatsClockDriftParams struct { @@ -1226,6 +1230,7 @@ type GetStatsClockDriftRow struct { // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds // the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's // latest measured drift, not an aggregate over a period. +// Select the filtered page before aggregating its IATA memberships. func (q *Queries) GetStatsClockDrift(ctx context.Context, arg GetStatsClockDriftParams) ([]GetStatsClockDriftRow, error) { rows, err := q.db.Query(ctx, getStatsClockDrift, arg.Column1, arg.Column2, arg.Limit) if err != nil { From 2125df5b61dba81ca5ee8cf5fa085a69f87b69db Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 9 Sep 2026 18:44:52 -0400 Subject: [PATCH 019/141] perf(db): include observation IATAs in trace packet reads (#120) --- db/queries/queries.sql | 12 ++- db/sqlc/querier.go | 3 +- db/sqlc/queries.sql.go | 14 +++- db/traces.go | 17 +---- db/traces_integration_test.go | 133 ++++++++++++++++++++++++++++++++++ db/traces_test.go | 7 +- 6 files changed, 160 insertions(+), 26 deletions(-) create mode 100644 db/traces_integration_test.go diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 68b8427e..c039e594 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -362,14 +362,22 @@ LEFT JOIN channel_messages cm ON cm.packet_hash = p.packet_hash WHERE p.packet_hash = $1; -- name: GetPacketsByTraceTag :many --- Returns all packets for a given trace tag with observations. +-- Return distinct observation IATAs in first-heard order for path resolution, +-- without fetching full observations separately for every trace packet. SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, p.route_type, p.first_heard_at, p.last_heard_at, p.parsed_payload, p.scope_id, - ts.name AS scope_name + ts.name AS scope_name, + ARRAY( + SELECT po.iata + FROM packet_observations po + WHERE po.packet_hash = p.packet_hash + GROUP BY po.iata + ORDER BY MIN(po.heard_at), po.iata + )::bpchar[] AS iatas FROM packets p LEFT JOIN transport_scopes ts ON ts.id = p.scope_id WHERE p.trace_tag = decode($1, 'hex') diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index bbca860c..118d1a04 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -57,7 +57,8 @@ type Querier interface { GetObserverTelemetryBucketed(ctx context.Context, arg GetObserverTelemetryBucketedParams) ([]GetObserverTelemetryBucketedRow, error) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPacketByHashRow, error) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) - // Returns all packets for a given trace tag with observations. + // Return distinct observation IATAs in first-heard order for path resolution, + // without fetching full observations separately for every trace packet. GetPacketsByTraceTag(ctx context.Context, decode string) ([]GetPacketsByTraceTagRow, error) GetRadioPresets(ctx context.Context, arg GetRadioPresetsParams) ([]MvRadioPreset, error) GetRegion(ctx context.Context, id int32) (GetRegionRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 1c7833ff..ef7550c2 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -860,7 +860,14 @@ SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, p.last_heard_at, p.parsed_payload, p.scope_id, - ts.name AS scope_name + ts.name AS scope_name, + ARRAY( + SELECT po.iata + FROM packet_observations po + WHERE po.packet_hash = p.packet_hash + GROUP BY po.iata + ORDER BY MIN(po.heard_at), po.iata + )::bpchar[] AS iatas FROM packets p LEFT JOIN transport_scopes ts ON ts.id = p.scope_id WHERE p.trace_tag = decode($1, 'hex') @@ -875,9 +882,11 @@ type GetPacketsByTraceTagRow struct { ParsedPayload []byte `json:"parsed_payload"` ScopeID *int32 `json:"scope_id"` ScopeName *string `json:"scope_name"` + Iatas []string `json:"iatas"` } -// Returns all packets for a given trace tag with observations. +// Return distinct observation IATAs in first-heard order for path resolution, +// without fetching full observations separately for every trace packet. func (q *Queries) GetPacketsByTraceTag(ctx context.Context, decode string) ([]GetPacketsByTraceTagRow, error) { rows, err := q.db.Query(ctx, getPacketsByTraceTag, decode) if err != nil { @@ -895,6 +904,7 @@ func (q *Queries) GetPacketsByTraceTag(ctx context.Context, decode string) ([]Ge &i.ParsedPayload, &i.ScopeID, &i.ScopeName, + &i.Iatas, ); err != nil { return nil, err } diff --git a/db/traces.go b/db/traces.go index 2624564d..d3cd6153 100644 --- a/db/traces.go +++ b/db/traces.go @@ -110,21 +110,8 @@ func (s *Store) GetTraceByTag(ctx context.Context, tag string) (*api.TraceDetail } packet.RawPath = rawPath } - // fetch observations to get IATAs for route resolution - packetHashBytes, err := hex.DecodeString(r.PacketHashHex) - if err == nil { - obsRows, err := s.q.ListObservationsForPacket(ctx, packetHashBytes) - if err == nil && len(obsRows) > 0 { - iatas := make([]string, 0, len(obsRows)) - seen := make(map[string]struct{}) - for _, v := range obsRows { - if _, ok := seen[v.Iata]; !ok { - seen[v.Iata] = struct{}{} - iatas = append(iatas, v.Iata) - } - } - packet.ResolvedRoute = s.resolveTraceRoute(ctx, &parsed, iatas) - } + if len(r.Iatas) > 0 { + packet.ResolvedRoute = s.resolveTraceRoute(ctx, &parsed, r.Iatas) } detail.Packets = append(detail.Packets, packet) } diff --git a/db/traces_integration_test.go b/db/traces_integration_test.go new file mode 100644 index 00000000..7850988c --- /dev/null +++ b/db/traces_integration_test.go @@ -0,0 +1,133 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +type traceQueryCounter struct { + sqlc.DBTX + queries, observations int +} + +func (c *traceQueryCounter) Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) { + c.queries++ + if strings.Contains(sql, "-- name: ListObservationsForPacket ") { + c.observations++ + } + return c.DBTX.Query(ctx, sql, args...) +} + +// Real SQL and Store mapping are exercised against temporary tables only. +func TestTraceDetailPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"packets", "packet_observations", "observers", "nodes", "node_short_ids", "transport_scopes"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO nodes (id,public_key,node_type,name) VALUES + ('00000000-0000-0000-0000-000000000001',decode('aa'||repeat('00',31),'hex'),2,'YVR relay'), + ('00000000-0000-0000-0000-000000000002',decode('aa01'||repeat('00',30),'hex'),2,'YYJ relay'); +INSERT INTO node_short_ids (node_id,iata,prefix_4) VALUES + ('00000000-0000-0000-0000-000000000001','YVR','\xaa000000'), + ('00000000-0000-0000-0000-000000000002','YYJ','\xaa010000'); +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,trace_tag,parsed_payload,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,1,'\x00','\x00','\x01020304', + CASE WHEN i=100 THEN '{}'::jsonb ELSE '{"flags":0,"pathHashes":["aa"],"snrValues":[12.5]}'::jsonb END, + '2026-01-01'::timestamptz+i*interval '1 second', '2026-01-01'::timestamptz+i*interval '1 second'+interval '1 second' +FROM generate_series(1,100) i; +INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT i,decode(lpad(to_hex(i),64,'0'),'hex'),md5(i::text)::uuid,'YVR','2026-01-01'::timestamptz+interval '2 seconds',1,1,1 +FROM generate_series(1,100) i WHERE i<>99; +-- The first packet was heard in YYJ before YVR. A repeat YYJ observation must +-- not trigger another region lookup or change equal-confidence precedence. +INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES + (101,decode(lpad('1',64,'0'),'hex'),md5('101')::uuid,'YYJ','2026-01-01'::timestamptz,1,1,1), + (102,decode(lpad('1',64,'0'),'hex'),md5('102')::uuid,'YYJ','2026-01-01'::timestamptz+interval '3 seconds',1,1,1); +ANALYZE packets; ANALYZE packet_observations; ANALYZE nodes; ANALYZE node_short_ids;`) + if err != nil { + t.Fatal(err) + } + counter := &traceQueryCounter{DBTX: tx} + store := &Store{q: sqlc.New(counter)} + start := time.Now() + detail, err := store.GetTraceByTag(ctx, "01020304") + elapsed := time.Since(start) + if err != nil { + t.Fatal(err) + } + if detail == nil || detail.TraceTag != "01020304" || len(detail.Packets) != 100 { + t.Fatal("trace packets missing") + } + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i, p := range detail.Packets { + number := i + 1 + if p.PacketHash != fmt.Sprintf("%064x", number) || p.RouteType != 1 || p.FirstHeardAt != anchor.Add(time.Duration(number)*time.Second).UnixMilli() || p.LastHeardAt != p.FirstHeardAt+1000 { + t.Fatalf("wrong packet fields or order at %d", number) + } + if number == 100 { + if len(p.RawPath) != 0 || len(p.ResolvedRoute) != 0 { + t.Fatal("invented path for empty payload") + } + continue + } + if len(p.RawPath) != 1 || p.RawPath[0].Hash != "aa" || p.RawPath[0].SNR == nil || *p.RawPath[0].SNR != 12.5 { + t.Fatalf("wrong raw path/SNR at %d", number) + } + if number == 99 { + if len(p.ResolvedRoute) != 0 { + t.Fatal("unobserved packet must not borrow another packet's IATAs") + } + continue + } + want := "YVR relay" + if number == 1 { + want = "YYJ relay" + } + if len(p.ResolvedRoute) != 1 || p.ResolvedRoute[0].Confidence != "high" || len(p.ResolvedRoute[0].Nodes) != 1 || p.ResolvedRoute[0].Nodes[0].Name == nil || *p.ResolvedRoute[0].Nodes[0].Name != want { + t.Fatalf("wrong region precedence/resolution at %d", number) + } + if p.ResolvedRoute[0].SNR == nil || *p.ResolvedRoute[0].SNR != 12.5 { + t.Fatal("resolved SNR changed") + } + } + t.Logf("100-packet trace: %.3f ms, %d database queries, %d separate observation queries", float64(elapsed.Microseconds())/1000, counter.queries, counter.observations) + if counter.observations != 0 { + t.Errorf("extra observation queries: %d; want 0", counter.observations) + } + if counter.queries != 100 { + t.Errorf("got %d queries; want one packet query and 99 unchanged path-resolution queries", counter.queries) + } + counter.queries, counter.observations = 0, 0 + missing, err := store.GetTraceByTag(ctx, "00000000") + if err != nil || missing != nil || counter.queries != 1 { + t.Fatal("missing tag behavior changed") + } +} diff --git a/db/traces_test.go b/db/traces_test.go index 764860e3..e7566979 100644 --- a/db/traces_test.go +++ b/db/traces_test.go @@ -112,15 +112,10 @@ func TestGetTraceByTag_WithPacket(t *testing.T) { FirstHeardAt: firstHeard, LastHeardAt: lastHeard, ParsedPayload: parsedPayload, + Iatas: []string{"YVR"}, }, }, nil) - mock.EXPECT(). - ListObservationsForPacket(gomock.Any(), []byte{0xaa, 0xbb, 0xcc, 0xdd}). - Return([]sqlc.ListObservationsForPacketRow{ - {Iata: "YVR"}, - }, nil) - mock.EXPECT(). ResolvePathHashesP2(gomock.Any(), sqlc.ResolvePathHashesP2Params{ Iata: "YVR", From 88bc0dc88ac7217a8f7e690eeefdaf400e24e4ef Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 9 Sep 2026 18:45:08 -0400 Subject: [PATCH 020/141] feat(retention): age out unreferenced observers when enabled (#126) --- README.md | 9 ++ cmd/beacon/main.go | 13 +- config.yaml.example | 7 ++ db/observer_retention_integration_test.go | 147 ++++++++++++++++++++++ db/observers.go | 4 + db/queries/queries.sql | 19 +++ db/sqlc/mock/querier.go | 15 +++ db/sqlc/querier.go | 3 + db/sqlc/queries.sql.go | 40 ++++++ internal/background/tasks.go | 28 +++++ internal/background/tasks_test.go | 68 ++++++++++ internal/config/config.go | 19 ++- internal/config/config_test.go | 12 ++ internal/presence/coalescer.go | 60 +++++++-- internal/presence/coalescer_test.go | 105 ++++++++++++++++ 15 files changed, 530 insertions(+), 19 deletions(-) create mode 100644 db/observer_retention_integration_test.go create mode 100644 internal/presence/coalescer_test.go diff --git a/README.md b/README.md index 53e1fd7b..ef3026db 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,15 @@ nodes: delete_after: 720h # delete a node entirely after this long unseen (default: 30 days, same default as packets.retention) clock_drift_threshold: 5m # |device clock - server clock| above which clockOutOfSync=true for a repeater/room server (default: 5m) +# Optional observer age-out (disabled by default; set e.g. 720h to opt in). +# Enable only when one Beacon ingest process owns the database: presence-cache +# preparation is local to that process, including both of its MQTT workers. +# Deletes at most 1000 observers per background.cleanup interval, only when +# last_seen/last_status_at are old and no observations, telemetry or ownership remain. +# Broker/location/scope metadata cascades; a returning observer receives a new ID. +observers: + delete_after: 0s # omitted or nonpositive disables deletion + # Redis caching layer (optional). # Caches read-heavy, slow-changing responses to reduce PostgreSQL load. # Connection details (address, password, database) are set via environment diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index fd92c27f..ef46a383 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -29,6 +29,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/presence" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" "github.com/joho/godotenv" ) @@ -255,11 +256,19 @@ func main() { go broker1.Start(ctx) go broker2.Start(ctx) - scheduler := background.New([]background.Task{ + tasks := []background.Task{ background.ViewRefreshTask(store, resolved.ViewRefreshInterval), background.CleanupTask(store, resolved.TelemetryRetention, resolved.PacketRetention, resolved.NodeDeleteAfter, resolved.CleanupInterval), background.ReconfirmTask(store, resolved.RouteRetention, resolved.RouteGrace, int64(resolved.RouteMinObservations), resolved.ReconfirmInterval), - }) + } + if resolved.ObserverDeleteAfter > 0 { + var onDelete func(context.Context, uuid.UUID) + if cr, ok := reader.(*cache.CachedReader); ok { + onDelete = cr.InvalidateObserver + } + tasks = append(tasks, background.ObserverCleanupTask(coalescer, resolved.ObserverDeleteAfter, resolved.CleanupInterval, onDelete)) + } + scheduler := background.New(tasks) go scheduler.Start(ctx) // ── HTTP server ────────────────────────────────────────────────────────── diff --git a/config.yaml.example b/config.yaml.example index b039cad2..cc91a39f 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -103,6 +103,13 @@ websocket: # delete_after: 720h # default: 30 days, same default as packets.retention # clock_drift_threshold: 5m # default: 5m +# Optional observer age-out. Omitted or nonpositive leaves observers intact. +# Enable only with one Beacon ingest process for this database. +# At most 1000 old observers are deleted per background.cleanup interval; +# retained observations, telemetry and ownership records prevent deletion. +#observers: +# delete_after: 720h # opt in to 30 days unseen + # CORS configuration. # Controls which origins, methods and headers are allowed for cross-origin requests. # Defaults to allowing all origins with GET/HEAD/OPTIONS if omitted — appropriate diff --git a/db/observer_retention_integration_test.go b/db/observer_retention_integration_test.go new file mode 100644 index 00000000..8672ea42 --- /dev/null +++ b/db/observer_retention_integration_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/presence" + "github.com/jackc/pgx/v5" +) + +func TestObserverRetentionPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + exec := func(query string, args ...any) { + t.Helper() + if _, err := tx.Exec(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + for _, table := range []string{"observers", "packet_observations", "observer_telemetry", "observer_owners", "observer_brokers", "observer_locations", "observer_scopes"} { + exec("CREATE TEMP TABLE " + table + " (LIKE public." + table + " INCLUDING ALL) ON COMMIT DROP") + if table != "observers" { + fk := "ALTER TABLE pg_temp." + table + " ADD FOREIGN KEY (observer_id) REFERENCES pg_temp.observers(id)" + if table != "packet_observations" { + fk += " ON DELETE CASCADE" + } + exec(fk) + } + } + store := &Store{q: sqlc.New(tx)} + cutoff := time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC) + exec(`INSERT INTO observers(public_key,display_name,last_seen,last_status_at) +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),label,$1::timestamptz+seen*interval '1 second', + $1::timestamptz+status*interval '1 second' +FROM (VALUES (1,'expired',-1,NULL),(2,'boundary',0,NULL),(3,'recent',1,NULL), + (4,'recent status',-1,1),(5,'status boundary',-1,0), + (6,'packet history',-1,NULL),(7,'telemetry',-1,NULL),(8,'owned',-1,NULL)) v(i,label,seen,status)`, cutoff) + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT 1,'\x01',id,'YYZ',last_seen,0,1,0 FROM observers WHERE display_name='packet history'; +INSERT INTO observer_telemetry(id,observer_id,reported_at) +SELECT 1,id,last_seen FROM observers WHERE display_name='telemetry'; +INSERT INTO observer_owners(observer_id,notes) +SELECT id,'fixture ownership without a node link' FROM observers WHERE display_name='owned'; +INSERT INTO observer_brokers(observer_id,broker_name) +SELECT id,'fixture' FROM observers WHERE display_name='expired'; +INSERT INTO observer_locations(observer_id,reported_at) +SELECT id,last_seen FROM observers WHERE display_name='expired'; +INSERT INTO observer_scopes(observer_id,scope_id) +SELECT id,1 FROM observers WHERE display_name='expired';`) + ids, err := store.DeleteOldObservers(ctx, cutoff) + if err != nil || len(ids) != 1 { + t.Fatalf("expected one unreferenced expired observer deleted, got %d: %v", len(ids), err) + } + var remaining string + if err := tx.QueryRow(ctx, "SELECT string_agg(display_name,',' ORDER BY display_name) FROM observers").Scan(&remaining); err != nil { + t.Fatal(err) + } + if remaining != "boundary,owned,packet history,recent,recent status,status boundary,telemetry" { + t.Fatalf("incorrect retained observers: %s", remaining) + } + var history, metadata int + if err := tx.QueryRow(ctx, `SELECT + (SELECT count(*) FROM packet_observations)+(SELECT count(*) FROM observer_telemetry)+(SELECT count(*) FROM observer_owners), + (SELECT count(*) FROM observer_brokers)+(SELECT count(*) FROM observer_locations)+(SELECT count(*) FROM observer_scopes)`).Scan(&history, &metadata); err != nil { + t.Fatal(err) + } + if history != 3 || metadata != 0 { + t.Fatalf("history=%d, cascading metadata=%d; want 3, 0", history, metadata) + } + if ids, err := store.DeleteOldObservers(ctx, cutoff); err != nil || len(ids) != 0 { + t.Fatalf("repeated cleanup changed protected rows: %v, %v", ids, err) + } + + // All relations below are temporary; no persistent fixtures or sequences are used. + exec("TRUNCATE pg_temp.observers CASCADE") + coalescer := presence.New(store, time.Second, time.Second) + pubkey := bytes.Repeat([]byte{42}, 32) + oldID, _, err := coalescer.UpsertObserver(ctx, pubkey) + if err != nil { + t.Fatal(err) + } + if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture"); err != nil { + t.Fatal(err) + } + // A future cutoff simulates this cached observer having aged past retention. + ids, err = coalescer.DeleteOldObservers(ctx, time.Now().Add(time.Hour)) + if err != nil || len(ids) != 1 || ids[0] != oldID { + t.Fatalf("cached expired observer was not deleted: %v, %v", ids, err) + } + newID, _, err := coalescer.UpsertObserver(ctx, pubkey) + if err != nil || newID == oldID { + t.Fatalf("returning observer reused a deleted ID: %v", err) + } + if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture"); err != nil { + t.Fatal(err) + } + var brokers int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM observer_brokers WHERE observer_id=$1", newID).Scan(&brokers); err != nil || brokers != 1 { + t.Fatalf("returning observer's broker was not recreated: count=%d, err=%v", brokers, err) + } + // The persisted row looks stale while fresh activity is still coalesced. + cutoff = time.Now().Add(-24 * time.Hour) + exec("UPDATE observers SET last_seen=$1 WHERE id=$2", cutoff.Add(-time.Hour), newID) + if _, _, err := coalescer.UpsertObserver(ctx, pubkey); err != nil { + t.Fatal(err) + } + if ids, err := coalescer.DeleteOldObservers(ctx, cutoff); err != nil || len(ids) != 0 { + t.Fatalf("pending presence did not protect a returning observer: %v, %v", ids, err) + } + var fresh bool + if err := tx.QueryRow(ctx, "SELECT last_seen >= $1 AND observation_count=1 FROM observers WHERE id=$2", cutoff, newID).Scan(&fresh); err != nil || !fresh { + t.Fatalf("pending presence was not persisted before cleanup: %v", err) + } + + exec("TRUNCATE pg_temp.observers CASCADE") + exec(`INSERT INTO observers(public_key,last_seen) +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),$1::timestamptz-interval '1 second' +FROM generate_series(1,1005) i`, cutoff) + for _, want := range []int{1000, 5, 0} { + ids, err := store.DeleteOldObservers(ctx, cutoff) + if err != nil || len(ids) != want { + t.Fatalf("bounded cleanup removed %d observers, want %d: %v", len(ids), want, err) + } + t.Logf("bounded cleanup deleted %d observers", len(ids)) + } +} diff --git a/db/observers.go b/db/observers.go index a58569b5..c8036ca4 100644 --- a/db/observers.go +++ b/db/observers.go @@ -338,3 +338,7 @@ func (s *Store) IsObserverByPubkey(ctx context.Context, pubkey []byte) bool { func (s *Store) DeleteOldTelemetry(ctx context.Context, cutoff time.Time) error { return s.q.DeleteOldTelemetry(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } + +func (s *Store) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { + return s.q.DeleteOldObservers(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) +} diff --git a/db/queries/queries.sql b/db/queries/queries.sql index c039e594..9771a265 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -287,6 +287,25 @@ LIMIT $3; -- Deletes telemetry rows older than the given cutoff. Called by the cleanup goroutine. DELETE FROM observer_telemetry WHERE reported_at < $1; +-- name: DeleteOldObservers :many +-- Opt-in age-out: preserve retained history and manually recorded ownership. +-- Bound deletions per cleanup tick and skip observers being updated by ingest. +WITH expired AS ( + SELECT o.id + FROM observers o + WHERE o.last_seen < $1 + AND (o.last_status_at IS NULL OR o.last_status_at < $1) + AND NOT EXISTS (SELECT 1 FROM packet_observations po WHERE po.observer_id = o.id) + AND NOT EXISTS (SELECT 1 FROM observer_telemetry ot WHERE ot.observer_id = o.id) + AND NOT EXISTS (SELECT 1 FROM observer_owners oo WHERE oo.observer_id = o.id) + ORDER BY o.last_seen, o.id + LIMIT 1000 + FOR UPDATE OF o SKIP LOCKED +) +DELETE FROM observers o USING expired e +WHERE o.id = e.id +RETURNING o.id; + -- ============================================================ -- OBSERVER BROKERS -- ============================================================ diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index e86eb2c9..c34fe770 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -71,6 +71,21 @@ func (mr *MockQuerierMockRecorder) DeleteOldNodes(ctx, lastSeen any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldNodes", reflect.TypeOf((*MockQuerier)(nil).DeleteOldNodes), ctx, lastSeen) } +// DeleteOldObservers mocks base method. +func (m *MockQuerier) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldObservers", ctx, lastSeen) + ret0, _ := ret[0].([]uuid.UUID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteOldObservers indicates an expected call of DeleteOldObservers. +func (mr *MockQuerierMockRecorder) DeleteOldObservers(ctx, lastSeen any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldObservers", reflect.TypeOf((*MockQuerier)(nil).DeleteOldObservers), ctx, lastSeen) +} + // DeleteOldPackets mocks base method. func (m *MockQuerier) DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 118d1a04..1c5f8c93 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -22,6 +22,9 @@ type Querier interface { // dangling in old routes there, but ReconfirmTask already prunes stale/ambiguous routes // periodically and will clean those up on its own schedule. DeleteOldNodes(ctx context.Context, lastSeen pgtype.Timestamptz) error + // Opt-in age-out: preserve retained history and manually recorded ownership. + // Bound deletions per cleanup tick and skip observers being updated by ingest. + DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) // Deletes packets and their observations older than the given cutoff. // packet_observations cascade-delete via FK. DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index ef7550c2..8581c4c0 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -40,6 +40,46 @@ func (q *Queries) DeleteOldNodes(ctx context.Context, lastSeen pgtype.Timestampt return err } +const deleteOldObservers = `-- name: DeleteOldObservers :many +WITH expired AS ( + SELECT o.id + FROM observers o + WHERE o.last_seen < $1 + AND (o.last_status_at IS NULL OR o.last_status_at < $1) + AND NOT EXISTS (SELECT 1 FROM packet_observations po WHERE po.observer_id = o.id) + AND NOT EXISTS (SELECT 1 FROM observer_telemetry ot WHERE ot.observer_id = o.id) + AND NOT EXISTS (SELECT 1 FROM observer_owners oo WHERE oo.observer_id = o.id) + ORDER BY o.last_seen, o.id + LIMIT 1000 + FOR UPDATE OF o SKIP LOCKED +) +DELETE FROM observers o USING expired e +WHERE o.id = e.id +RETURNING o.id +` + +// Opt-in age-out: preserve retained history and manually recorded ownership. +// Bound deletions per cleanup tick and skip observers being updated by ingest. +func (q *Queries) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) { + rows, err := q.db.Query(ctx, deleteOldObservers, lastSeen) + if err != nil { + return nil, err + } + defer rows.Close() + items := []uuid.UUID{} + for rows.Next() { + var id uuid.UUID + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const deleteOldPackets = `-- name: DeleteOldPackets :exec DELETE FROM packets WHERE last_heard_at < $1 ` diff --git a/internal/background/tasks.go b/internal/background/tasks.go index f2f3ec70..a19b6e4b 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -10,6 +10,7 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/google/uuid" ) type viewRefresher interface { @@ -83,6 +84,33 @@ func CleanupTask(store *db.Store, telemetryRetention, packetRetention, nodeDelet // table gets fully re-checked roughly daily. const reconfirmBatchSize = 750_000 +type observerCleaner interface { + DeleteOldObservers(context.Context, time.Time) ([]uuid.UUID, error) +} + +// ObserverCleanupTask ages out unreferenced observers through the presence +// coalescer, then invalidates any cached details for the deleted IDs. +func ObserverCleanupTask(store observerCleaner, deleteAfter, interval time.Duration, onDelete func(context.Context, uuid.UUID)) Task { + return Task{ + Name: "observer_cleanup", Interval: interval, + Run: func(ctx context.Context) error { + if deleteAfter <= 0 { + return nil + } + ids, err := store.DeleteOldObservers(ctx, time.Now().Add(-deleteAfter)) + if err != nil { + return err + } + if onDelete != nil { + for _, id := range ids { + onDelete(ctx, id) + } + } + return nil + }, + } +} + // ReconfirmTask returns a Task that prunes aged routes first, then reconfirms // stale and ambiguous resolved paths and neighbors, so known_routes only ever // has one writer at a time. diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go index cee90ff5..b84973a4 100644 --- a/internal/background/tasks_test.go +++ b/internal/background/tasks_test.go @@ -10,6 +10,8 @@ import ( "strings" "testing" "time" + + "github.com/google/uuid" ) type refreshStub struct { @@ -17,6 +19,72 @@ type refreshStub struct { errs map[string]error } +type observerCleanupStub struct { + ids []uuid.UUID + err error + calls int + cutoff time.Time +} + +func (s *observerCleanupStub) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { + s.calls++ + s.cutoff = cutoff + if err := ctx.Err(); err != nil { + return nil, err + } + return s.ids, s.err +} + +func TestObserverCleanupTask(t *testing.T) { + failure := errors.New("cleanup unavailable") + for _, tc := range []struct { + name string + retention time.Duration + err error + cancelled bool + }{ + {"disabled", 0, nil, false}, + {"negative disables", -time.Hour, nil, false}, + {"enabled", 24 * time.Hour, nil, false}, + {"failure", 24 * time.Hour, failure, false}, + {"cancelled", 24 * time.Hour, context.Canceled, true}, + } { + t.Run(tc.name, func(t *testing.T) { + store := &observerCleanupStub{ids: []uuid.UUID{uuid.New(), uuid.New()}, err: tc.err} + var invalidated []uuid.UUID + task := ObserverCleanupTask(store, tc.retention, time.Minute, func(_ context.Context, id uuid.UUID) { + invalidated = append(invalidated, id) + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if tc.cancelled { + cancel() + } + before := time.Now().Add(-tc.retention) + err := task.Run(ctx) + if !errors.Is(err, tc.err) || task.Name != "observer_cleanup" || task.Interval != time.Minute { + t.Fatalf("unexpected task outcome: %v, %+v", err, task) + } + if tc.retention <= 0 { + if store.calls != 0 || len(invalidated) != 0 { + t.Fatal("disabled retention performed work") + } + return + } + if store.calls != 1 || store.cutoff.Before(before) || store.cutoff.After(time.Now().Add(-tc.retention)) { + t.Fatal("incorrect cleanup cutoff or number of calls") + } + if tc.err != nil { + if len(invalidated) != 0 { + t.Fatal("failed cleanup invalidated cache entries") + } + } else if len(invalidated) != 2 || invalidated[0] != store.ids[0] || invalidated[1] != store.ids[1] { + t.Fatal("deleted observers were not invalidated") + } + }) + } +} + func (s *refreshStub) refresh(ctx context.Context, name string) error { s.calls = append(s.calls, name) if err := ctx.Err(); err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index f7420052..feffb27b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,6 +30,7 @@ type Config struct { Background BackgroundConfig `yaml:"background"` Presence PresenceConfig `yaml:"presence"` Nodes NodesConfig `yaml:"nodes"` + Observers ObserversConfig `yaml:"observers"` } // ResolvedConfig holds all runtime configuration with defaults applied. @@ -55,8 +56,9 @@ type ResolvedConfig struct { // NodeStaleThreshold and NodeDeleteAfter mirror ClockDriftThreshold's "0 means unset, // resolve to a default" pattern -- see NodesConfig. - NodeStaleThreshold time.Duration - NodeDeleteAfter time.Duration + NodeStaleThreshold time.Duration + NodeDeleteAfter time.Duration + ObserverDeleteAfter time.Duration } // PresenceConfig controls coalescing of presence bookkeeping writes @@ -209,6 +211,14 @@ type NodesConfig struct { DeleteAfter duration `yaml:"delete_after"` } +// ObserversConfig controls optional observer age-out. +type ObserversConfig struct { + // DeleteAfter is how long an observer must be unseen before it can be deleted. + // Omitted or nonpositive disables age-out. Retained packet observations, + // telemetry and ownership records always protect the observer from deletion. + DeleteAfter duration `yaml:"delete_after"` +} + // duration is a wrapper around time.Duration that supports YAML unmarshalling // from human-readable strings like "24h", "7d", "30d". type duration struct { @@ -333,6 +343,7 @@ func Resolve(cfg *Config) ResolvedConfig { ClockDriftThreshold: cfg.Nodes.ClockDriftThreshold.Duration, NodeStaleThreshold: cfg.Nodes.StaleThreshold.Duration, NodeDeleteAfter: cfg.Nodes.DeleteAfter.Duration, + ObserverDeleteAfter: cfg.Observers.DeleteAfter.Duration, } if r.TelemetryResolution == 0 { r.TelemetryResolution = time.Hour @@ -386,10 +397,10 @@ func Resolve(cfg *Config) ResolvedConfig { func (r ResolvedConfig) String() string { return fmt.Sprintf( - "telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s", + "telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s", r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, r.MaxConnsPerIP, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold, - r.NodeStaleThreshold, r.NodeDeleteAfter, + r.NodeStaleThreshold, r.NodeDeleteAfter, r.ObserverDeleteAfter, ) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4ccd46a6..75f673cf 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -38,6 +38,8 @@ regions: name: British Columbia display_order: 1 iatas: [YVR] +observers: + delete_after: 720h `) f.Close() @@ -54,6 +56,9 @@ regions: if cfg.Regions[0].Slug != "bc" { t.Errorf("expected slug bc, got %s", cfg.Regions[0].Slug) } + if Resolve(cfg).ObserverDeleteAfter != 30*24*time.Hour { + t.Error("observers.delete_after was not loaded from YAML") + } } func TestLoad_InvalidYAML(t *testing.T) { @@ -104,6 +109,9 @@ func TestResolve_Defaults(t *testing.T) { if r.NodeDeleteAfter != 30*24*time.Hour { t.Errorf("expected NodeDeleteAfter 720h (same default as PacketRetention), got %v", r.NodeDeleteAfter) } + if r.ObserverDeleteAfter != 0 { + t.Errorf("observer deletion must be disabled by default, got %v", r.ObserverDeleteAfter) + } } func TestResolve_ExplicitValues(t *testing.T) { @@ -115,6 +123,7 @@ func TestResolve_ExplicitValues(t *testing.T) { cfg.Background.ViewRefresh.Duration = 2 * time.Hour cfg.Background.Reconfirm.Duration = 3 * time.Hour cfg.Background.Cleanup.Duration = 4 * time.Hour + cfg.Observers.DeleteAfter.Duration = 45 * 24 * time.Hour r := Resolve(cfg) if r.TelemetryResolution != 30*time.Minute { @@ -126,6 +135,9 @@ func TestResolve_ExplicitValues(t *testing.T) { if r.ViewRefreshInterval != 2*time.Hour { t.Errorf("expected 2h, got %v", r.ViewRefreshInterval) } + if r.ObserverDeleteAfter != 45*24*time.Hour { + t.Errorf("expected observer delete_after 1080h, got %v", r.ObserverDeleteAfter) + } } func TestResolvedConfig_String(t *testing.T) { diff --git a/internal/presence/coalescer.go b/internal/presence/coalescer.go index c0673bb0..c8e3544a 100644 --- a/internal/presence/coalescer.go +++ b/internal/presence/coalescer.go @@ -35,11 +35,14 @@ type Store interface { // TouchPackets applies coalesced last_heard_at bumps for the given packet // hashes in one statement. TouchPackets(ctx context.Context, hashes [][]byte, heard []time.Time) error + + DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) } type identity struct { id uuid.UUID name string + seen time.Time } type observerBump struct { @@ -99,6 +102,8 @@ func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUI bump.seen = c.now() bump.count++ c.dirtyObservers[ident.id] = bump + ident.seen = bump.seen + c.identities[key] = ident c.mu.Unlock() return ident.id, ident.name, nil } @@ -109,7 +114,7 @@ func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUI return id, name, err } c.mu.Lock() - c.identities[key] = identity{id: id, name: name} + c.identities[key] = identity{id: id, name: name, seen: c.now()} c.mu.Unlock() return id, name, nil } @@ -191,6 +196,30 @@ func (c *Coalescer) UpdateObserverStatus(ctx context.Context, p ingest.UpdateObs return id, err } +// DeleteOldObservers persists cached activity before pruning and forgets IDs so +// returning observers write through. Include identities even if an earlier +// presence flush failed or is still in flight; GREATEST prevents older flushes +// from overwriting this freshness. A failed preparation must not delete rows. +func (c *Coalescer) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { + c.mu.Lock() + observers := c.dirtyObservers + for _, ident := range c.identities { + bump := observers[ident.id] + if ident.seen.After(bump.seen) { + bump.seen = ident.seen + } + observers[ident.id] = bump + } + c.dirtyObservers = make(map[uuid.UUID]observerBump) + clear(c.identities) + clear(c.knownBrokers) + c.mu.Unlock() + if err := c.flushObservers(ctx, observers); err != nil { + return nil, err + } + return c.Store.DeleteOldObservers(ctx, cutoff) +} + // Run flushes on the configured interval until ctx is cancelled, then does a // final flush so a clean shutdown loses nothing. func (c *Coalescer) Run(ctx context.Context) { @@ -228,18 +257,8 @@ func (c *Coalescer) Flush(ctx context.Context) { } c.mu.Unlock() - if len(observers) > 0 { - ids := make([]uuid.UUID, 0, len(observers)) - seen := make([]time.Time, 0, len(observers)) - counts := make([]int32, 0, len(observers)) - for id, bump := range observers { - ids = append(ids, id) - seen = append(seen, bump.seen) - counts = append(counts, bump.count) - } - if err := c.Store.TouchObservers(ctx, ids, seen, counts); err != nil { - log.Printf("presence: flush observers failed (%d rows dropped): %v", len(ids), err) - } + if err := c.flushObservers(ctx, observers); err != nil { + log.Printf("presence: flush observers failed (%d rows dropped): %v", len(observers), err) } if len(brokers) > 0 { @@ -268,3 +287,18 @@ func (c *Coalescer) Flush(ctx context.Context) { } } } + +func (c *Coalescer) flushObservers(ctx context.Context, observers map[uuid.UUID]observerBump) error { + if len(observers) == 0 { + return nil + } + ids := make([]uuid.UUID, 0, len(observers)) + seen := make([]time.Time, 0, len(observers)) + counts := make([]int32, 0, len(observers)) + for id, bump := range observers { + ids = append(ids, id) + seen = append(seen, bump.seen) + counts = append(counts, bump.count) + } + return c.Store.TouchObservers(ctx, ids, seen, counts) +} diff --git a/internal/presence/coalescer_test.go b/internal/presence/coalescer_test.go new file mode 100644 index 00000000..0900f5f7 --- /dev/null +++ b/internal/presence/coalescer_test.go @@ -0,0 +1,105 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package presence + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" +) + +type retentionStore struct { + Store + id uuid.UUID + touch func(context.Context, []uuid.UUID, []time.Time, []int32) error + prune func(context.Context, time.Time) ([]uuid.UUID, error) +} + +func (s *retentionStore) UpsertObserver(context.Context, []byte) (uuid.UUID, string, error) { + return s.id, "fixture", nil +} +func (s *retentionStore) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error { + return s.touch(ctx, ids, seen, counts) +} +func (s *retentionStore) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { + return s.prune(ctx, cutoff) +} + +func TestObserverRetentionAbortOnFlushFailure(t *testing.T) { + for _, cause := range []error{errors.New("flush unavailable"), context.Canceled} { + t.Run(cause.Error(), func(t *testing.T) { + store := &retentionStore{id: uuid.New()} + store.touch = func(context.Context, []uuid.UUID, []time.Time, []int32) error { return cause } + store.prune = func(context.Context, time.Time) ([]uuid.UUID, error) { + t.Error("deletion attempted after failed preparation") + return nil, nil + } + c := New(store, time.Second, time.Second) + if _, _, err := c.UpsertObserver(context.Background(), []byte("fixture")); err != nil { + t.Fatal(err) + } + if _, err := c.DeleteOldObservers(context.Background(), time.Now()); !errors.Is(err, cause) { + t.Fatalf("got %v, want %v", err, cause) + } + }) + } +} + +func TestObserverRetentionPreservesActivityAfterFlush(t *testing.T) { + for _, inFlight := range []bool{false, true} { + name := "failed flush" + if inFlight { + name = "in-flight flush" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + store := &retentionStore{id: uuid.New()} + c := New(store, time.Second, time.Second) + c.now = func() time.Time { return now } + for range 2 { + if _, _, err := c.UpsertObserver(ctx, []byte("fixture")); err != nil { + t.Fatal(err) + } + } + started, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var calls atomic.Int32 + store.touch = func(_ context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error { + if calls.Add(1) == 1 { + close(started) + <-release + return errors.New("earlier flush failed") + } + if len(ids) != 1 || ids[0] != store.id || !seen[0].Equal(now) || counts[0] != 0 { + t.Errorf("cached activity was lost or counted twice: %v, %v, %v", ids, seen, counts) + } + return nil + } + store.prune = func(context.Context, time.Time) ([]uuid.UUID, error) { + if calls.Load() != 2 { + t.Error("deletion preceded the freshness write") + } + return nil, nil + } + go func() { c.Flush(ctx); close(done) }() + <-started + if !inFlight { + close(release) + <-done + } + _, err := c.DeleteOldObservers(ctx, now.Add(-time.Hour)) + if inFlight { + close(release) + <-done + } + if err != nil { + t.Fatal(err) + } + }) + } +} From 04fc0266d13a8dec8135da49a94f5fcf1db3bea9 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 9 Sep 2026 18:45:52 -0400 Subject: [PATCH 021/141] fix(packets): preserve captured endpoint names in history (#128) --- db/endpoint_matching_integration_test.go | 219 +++++++++++++++ .../026_observation_endpoint_snapshots.sql | 9 + db/packet_endpoints_integration_test.go | 257 ++++++++++++++++++ db/packets.go | 103 ++++--- db/packets_test.go | 2 +- db/queries/queries.sql | 31 ++- db/sqlc/mock/querier.go | 15 + db/sqlc/models.go | 1 + db/sqlc/querier.go | 5 + db/sqlc/queries.sql.go | 182 +++++++++---- db/store.go | 26 ++ docs/docs.go | 4 +- docs/swagger.json | 4 +- docs/swagger.yaml | 14 +- internal/api/packets.go | 29 +- internal/api/packets_test.go | 22 ++ internal/ingest/endpoint_matching_test.go | 50 ++++ internal/ingest/ingest.go | 3 + internal/ingest/ingest_test.go | 4 + internal/ingest/packet.go | 57 ++-- internal/ingest/packet_endpoints_test.go | 123 +++++++++ 21 files changed, 1018 insertions(+), 142 deletions(-) create mode 100644 db/endpoint_matching_integration_test.go create mode 100644 db/migrations/026_observation_endpoint_snapshots.sql create mode 100644 db/packet_endpoints_integration_test.go create mode 100644 internal/ingest/endpoint_matching_test.go create mode 100644 internal/ingest/packet_endpoints_test.go diff --git a/db/endpoint_matching_integration_test.go b/db/endpoint_matching_integration_test.go new file mode 100644 index 00000000..734c2085 --- /dev/null +++ b/db/endpoint_matching_integration_test.go @@ -0,0 +1,219 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +type endpointMatchQueries struct { + sqlc.DBTX + calls int + query string +} + +func (q *endpointMatchQueries) Query(ctx context.Context, query string, args ...any) (pgx.Rows, error) { + q.calls++ + q.query = query + return q.DBTX.Query(ctx, query, args...) +} + +func TestPacketEndpointRolesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + exec := func(sql string, args ...any) { + t.Helper() + if _, err := tx.Exec(ctx, sql, args...); err != nil { + t.Fatal(err) + } + } + for _, table := range []string{"packets", "packet_observations", "observers", "transport_scopes", "channel_messages", "nodes", "node_short_ids"} { + exec("CREATE TEMP TABLE " + table + " (LIKE public." + table + " INCLUDING ALL) ON COMMIT DROP") + } + id := func(i int) uuid.UUID { return uuid.MustParse(fmt.Sprintf("00000000-0000-0000-0000-%012d", i)) } + exec(`INSERT INTO nodes(id,public_key,name,node_type) +SELECT ('00000000-0000-0000-0000-'||lpad(i::text,12,'0'))::uuid, + decode(prefix||repeat('00',28),'hex'),name,role +FROM (VALUES (1,'aa000001','Companion',1),(2,'aa000002','Repeater',2), + (3,'ee000003','Room',3),(4,'aa000004','Sensor',4),(5,'aa000005','Unknown role',0), + (6,'bb000006','Companion only',1),(7,'aa000007','Other region',1), + (8,'cc000008','Relay',2),(9,'dd000009',NULL,1),(10,'aa000010','No membership',1)) v(i,prefix,name,role); +INSERT INTO node_short_ids(node_id,iata,prefix_4) +SELECT id,CASE WHEN name='Other region' THEN 'YYZ' ELSE 'YVR' END,substring(public_key from 1 for 4) +FROM nodes WHERE name IS DISTINCT FROM 'No membership';`) + exec("INSERT INTO observers(id,public_key) VALUES ($1,'\\x01'),($2,'\\x02')", id(101), id(102)) + queries := &endpointMatchQueries{DBTX: tx} + store := &Store{q: sqlc.New(queries)} + check := func(hop *api.ResolvedHop, confidence string, members ...int) { + t.Helper() + if hop == nil || hop.Confidence != confidence || len(hop.Nodes) != len(members) { + t.Errorf("endpoint candidate roles incorrect: confidence=%s, want %s with %d nodes", func() string { + if hop == nil { + return "nil" + } + return hop.Confidence + }(), confidence, len(members)) + return + } + want := make(map[uuid.UUID]bool) + for _, member := range members { + want[id(member)] = true + } + for _, node := range hop.Nodes { + if !want[node.ID] { + t.Errorf("unexpected endpoint node %s", node.ID) + } + delete(want, node.ID) + } + if len(want) != 0 { + t.Error("endpoint candidate missing") + } + } + for i, kind := range []int{0, 1, 2, 8} { + // These encrypted envelopes carry one-byte destination/source prefixes. + hash := []byte{byte(i + 1)} + payload := append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...) + exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +VALUES ($1,$2,0,1,$3,$4,NOW(),NOW())`, hash, kind, payload, []byte{byte(kind<<2 | 1)}) + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,path_bytes,source_broker) +VALUES ($1,$2,$3,'YVR',NOW(),1,1,1,'\xcc','fixture')`, i+1, hash, id(101)) + packet, err := store.GetPacket(ctx, hash) + if err != nil || len(packet.Observations) != 1 { + t.Fatalf("packet detail: %v", err) + } + observation := packet.Observations[0] + check(observation.ResolvedSource, "ambiguous", 1, 2, 4, 5) + check(observation.ResolvedDestination, "high", 6) + if len(observation.ResolvedPath) != 1 { + t.Fatal("relay path missing") + } + check(&observation.ResolvedPath[0], "high", 8) + } + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,source_broker) +VALUES (10,'\x01',$1,'YYZ',NOW()+interval '1 second',0,1,0,'fixture')`, id(102)) + packet, err := store.GetPacket(ctx, []byte{1}) + if err != nil || len(packet.Observations) != 2 { + t.Fatalf("regional observations: %v", err) + } + check(packet.Observations[1].ResolvedSource, "high", 7) + check(packet.Observations[1].ResolvedDestination, "none") + // Intermediate relay matching keeps its existing infrastructure-only boundary. + for width := 1; width <= 4; width++ { + hash := []byte{0xaa, 0, 0, 2}[:width] + resolved, err := store.ResolvePathHashes(ctx, "YVR", [][]byte{hash}) + if err != nil { + t.Fatal(err) + } + hops := api.BuildResolvedPath([][]byte{hash}, resolved) + check(&hops[0], "high", 2) + } + t.Log("endpoint roles and unchanged relay matching checked across four payload types") + for _, hashes := range [][][]byte{nil, {}, {{}}, {{0xaa, 0xbb}}, {{0xaa}, {0xbb, 0xcc}}} { + queries.calls = 0 + got, err := store.ResolveEndpointHashes(ctx, "YVR", hashes) + if err != nil || len(got) != 0 || queries.calls != 0 { + t.Fatal("invalid endpoint width reached the database") + } + } + queries.calls = 0 + resolved, err := store.ResolveEndpointHashes(ctx, "YVR", [][]byte{{0xee}, {0xdd}, {0xff}, {0xbb}, {0xbb}}) + if err != nil || queries.calls != 1 { + t.Fatalf("batched endpoint query: %v, calls=%d", err, queries.calls) + } + hops := api.BuildResolvedPath([][]byte{{0xee}, {0xdd}, {0xff}, {0xbb}}, resolved) + check(&hops[0], "high", 3) + check(&hops[1], "high", 9) + if hops[1].Nodes[0].Name != nil { + t.Error("unnamed node acquired a name") + } + check(&hops[2], "none") + check(&hops[3], "high", 6) + missing, err := store.ResolveEndpointHashes(ctx, "ZZZ", [][]byte{{0xaa}}) + if err != nil || len(missing) != 0 { + t.Fatal("endpoint escaped its observation region") + } + + // A large nonmatching population must not turn a short-hash lookup into a full scan. + exec(`INSERT INTO nodes(id,public_key,node_type) +SELECT ('00000000-0000-0000-0000-'||lpad(i::text,12,'0'))::uuid, + decode(lpad(to_hex(i),8,'0')||repeat('00',28),'hex'),1 FROM generate_series(10000,29999) i; +INSERT INTO node_short_ids(node_id,iata,prefix_4) +SELECT id,'YVR',substring(public_key from 1 for 4) FROM nodes WHERE substring(public_key from 1 for 1)='\x00'; +ANALYZE nodes; +ANALYZE node_short_ids;`) + if _, err := store.ResolveEndpointHashes(ctx, "YVR", [][]byte{{0xaa}}); err != nil { + t.Fatal(err) + } + exec("PREPARE endpoint_role_plan AS " + queries.query) + defer tx.Exec(context.Background(), "DEALLOCATE endpoint_role_plan") + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + exec("SET LOCAL plan_cache_mode=" + mode) + var raw []byte + if err := tx.QueryRow(ctx, `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE endpoint_role_plan('YVR',ARRAY['\xaa'::bytea])`).Scan(&raw); err != nil { + t.Fatal(err) + } + var report []struct { + Plan map[string]any + Milliseconds float64 `json:"Execution Time"` + } + if err := json.Unmarshal(raw, &report); err != nil { + t.Fatal(err) + } + var examined, nodeRows float64 + indexed := false + var walk func(map[string]any) + walk = func(plan map[string]any) { + if cond, ok := plan["Index Cond"].(string); ok && strings.Contains(cond, "prefix_1") && strings.Contains(cond, "iata") { + indexed = true + } + if plan["Relation Name"] == "node_short_ids" { + rows, _ := plan["Actual Rows"].(float64) + removed, _ := plan["Rows Removed by Filter"].(float64) + loops, _ := plan["Actual Loops"].(float64) + examined += (rows + removed) * loops + } + if plan["Relation Name"] == "nodes" { + rows, _ := plan["Actual Rows"].(float64) + removed, _ := plan["Rows Removed by Filter"].(float64) + loops, _ := plan["Actual Loops"].(float64) + nodeRows += (rows + removed) * loops + } + if children, ok := plan["Plans"].([]any); ok { + for _, child := range children { + walk(child.(map[string]any)) + } + } + } + walk(report[0].Plan) + if !indexed || examined > 10 || nodeRows > 10 { + t.Errorf("%s endpoint lookup scanned %.0f short IDs and %.0f nodes, indexed=%v", mode, examined, nodeRows, indexed) + } + t.Logf("%s: %.0f short-ID rows and %.0f node rows examined in %.3f ms", mode, examined, nodeRows, report[0].Milliseconds) + } +} diff --git a/db/migrations/026_observation_endpoint_snapshots.sql b/db/migrations/026_observation_endpoint_snapshots.sql new file mode 100644 index 00000000..77a163b2 --- /dev/null +++ b/db/migrations/026_observation_endpoint_snapshots.sql @@ -0,0 +1,9 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Preserve the endpoint resolution already computed for the live observation. +-- SQL NULL means no endpoint nodes were captured; empty legacy captures also +-- permit detail-time lookup once the node is known (for example, first adverts). +-- No default/backfill: current node metadata cannot reconstruct past names. +ALTER TABLE packet_observations + ADD COLUMN IF NOT EXISTS resolved_endpoints JSONB; diff --git a/db/packet_endpoints_integration_test.go b/db/packet_endpoints_integration_test.go new file mode 100644 index 00000000..756d9cdf --- /dev/null +++ b/db/packet_endpoints_integration_test.go @@ -0,0 +1,257 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "os" + "reflect" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +type endpointQueryCounter struct { + sqlc.DBTX + calls int +} + +func (q *endpointQueryCounter) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + q.calls++ + return q.DBTX.Query(ctx, sql, args...) +} +func (q *endpointQueryCounter) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row { + q.calls++ + return q.DBTX.QueryRow(ctx, sql, args...) +} + +func TestPacketEndpointSnapshotsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + exec := func(query string, args ...any) { + t.Helper() + if _, err := tx.Exec(ctx, query, args...); err != nil { + t.Fatal(err) + } + } + for _, table := range []string{"packets", "packet_observations", "observers", "transport_scopes", "nodes", "channel_messages"} { + exec("CREATE TEMP TABLE " + table + " (LIKE public." + table + " INCLUDING ALL) ON COMMIT DROP") + } + // Also lets the unchanged Store run against the regression fixture before migration. + exec("ALTER TABLE pg_temp.packet_observations ADD COLUMN IF NOT EXISTS resolved_endpoints JSONB") + exec("CREATE TEMP SEQUENCE endpoint_observation_id START 1000") + exec("ALTER TABLE pg_temp.packet_observations ALTER COLUMN id SET DEFAULT nextval('pg_temp.endpoint_observation_id')") + observer1 := uuid.MustParse("00000000-0000-0000-0000-000000000001") + observer2 := uuid.MustParse("00000000-0000-0000-0000-000000000002") + nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000003") + key := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + name, earlier := "Captured 👋 companion", "Earlier companion" + high := &api.ResolvedHop{Confidence: "high", Nodes: []api.ResolvedNode{{ID: nodeID, Name: &name, PublicKey: key}}} + old := &api.ResolvedHop{Confidence: "high", Nodes: []api.ResolvedNode{{ID: nodeID, Name: &earlier, PublicKey: key}}} + ambiguous := &api.ResolvedHop{Confidence: "ambiguous", Nodes: []api.ResolvedNode{high.Nodes[0], {ID: observer2, PublicKey: "bb"}}} + none := &api.ResolvedHop{Confidence: "none", Nodes: []api.ResolvedNode{}} + snapshot := func(source, destination *api.ResolvedHop) []byte { + value, err := json.Marshal(struct { + Source *api.ResolvedHop `json:"source,omitempty"` + Destination *api.ResolvedHop `json:"destination,omitempty"` + }{source, destination}) + if err != nil { + t.Fatal(err) + } + return value + } + check := func(source, destination, wantSource, wantDestination *api.ResolvedHop) { + t.Helper() + if !reflect.DeepEqual(source, wantSource) || !reflect.DeepEqual(destination, wantDestination) { + t.Error("historical packet lost endpoint snapshot") + } + } + exec("INSERT INTO observers(id,public_key) VALUES ($1,'\\x01'),($2,'\\x02')", observer1, observer2) + exec("INSERT INTO nodes(id,public_key,name,node_type) VALUES ($1,decode($2,'hex'),$3,1)", nodeID, key, name) + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),kind,0,1,'\x00','\x00',decode($2,'hex'), + $1::timestamptz+i*interval '1 second',$1::timestamptz+i*interval '1 second' +FROM (VALUES (1,4),(2,2),(3,3),(4,4),(5,3)) v(i,kind)`, anchor, key) + for _, obs := range []struct { + id, packet int + observer uuid.UUID + iata string + data []byte + }{ + {1, 1, observer1, "YYZ", snapshot(old, nil)}, + {2, 1, observer2, "YVR", snapshot(high, nil)}, + {3, 2, observer1, "YYZ", snapshot(ambiguous, none)}, + {4, 3, observer2, "YVR", snapshot(nil, nil)}, + {5, 4, observer1, "YYZ", nil}, + } { + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,source_broker,resolved_endpoints) +VALUES ($1,decode($2,'hex'),$3,$4,$5,0,1,0,'fixture',$6)`, obs.id, fmt.Sprintf("%064x", obs.packet), obs.observer, obs.iata, anchor.Add(time.Duration(obs.id)*time.Second), obs.data) + } + exec("UPDATE nodes SET name='Current name'") + q := &endpointQueryCounter{DBTX: tx} + store := &Store{q: sqlc.New(q)} + for _, iatas := range [][]string{nil, {"YYZ"}, {"YVR"}, {"YYZ", "YVR", "YYZ"}} { + q.calls = 0 + page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil { + t.Fatal(err) + } + if q.calls != 1 || len(page.Items) == 0 { + t.Fatalf("list used %d queries or lost its page", q.calls) + } + for _, item := range page.Items { + switch item.PacketHash { + case fmt.Sprintf("%064x", 1): + // The existing regional roll-up also selects the global latest observer. + if item.LatestObserver == nil || item.LatestObserver.ID != observer2 || item.LatestObserver.IATA != "YVR" { + t.Fatal("latest observer selection changed") + } + check(item.LatestObserver.ResolvedSource, item.LatestObserver.ResolvedDestination, high, nil) + case fmt.Sprintf("%064x", 2): + check(item.LatestObserver.ResolvedSource, item.LatestObserver.ResolvedDestination, ambiguous, none) + case fmt.Sprintf("%064x", 3), fmt.Sprintf("%064x", 4): + check(item.LatestObserver.ResolvedSource, item.LatestObserver.ResolvedDestination, nil, nil) + case fmt.Sprintf("%064x", 5): + if item.LatestObserver != nil { + t.Fatal("invented an observation for a packet without one") + } + } + } + } + q.calls = 0 + backfill, err := store.ListPacketsAfterID(ctx, 0, -1, -1, nil, "", 50) + if err != nil || len(backfill) != 5 || q.calls != 1 { + t.Fatalf("backfill: rows=%d queries=%d err=%v", len(backfill), q.calls, err) + } + check(backfill[0].LatestObserver.ResolvedSource, backfill[0].LatestObserver.ResolvedDestination, old, nil) + check(backfill[1].LatestObserver.ResolvedSource, backfill[1].LatestObserver.ResolvedDestination, high, nil) + check(backfill[2].LatestObserver.ResolvedSource, backfill[2].LatestObserver.ResolvedDestination, ambiguous, none) + packet1 := append(make([]byte, 31), 1) + for pass := 0; pass < 2; pass++ { + q.calls = 0 + packet, err := store.GetPacket(ctx, packet1) + if err != nil || len(packet.Observations) != 2 { + t.Fatalf("packet detail: %v", err) + } + check(packet.Observations[0].ResolvedSource, packet.Observations[0].ResolvedDestination, old, nil) + check(packet.Observations[1].ResolvedSource, packet.Observations[1].ResolvedDestination, high, nil) + if q.calls != 2 { + t.Errorf("snapshot detail used %d queries, want packet plus observations only", q.calls) + } + if pass == 0 { + legacy, err := store.GetPacket(ctx, append(make([]byte, 31), 4)) + if err != nil || legacy.Observations[0].ResolvedSource == nil || *legacy.Observations[0].ResolvedSource.Nodes[0].Name != "Current name" { + t.Fatalf("legacy detail lookup changed: %v", err) + } + exec("DELETE FROM nodes") + } + } + exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),64,'0'),'hex'),4,0,1,'\x00','\x00',$1::timestamptz+i*interval '1 second',$1::timestamptz+i*interval '1 second' +FROM generate_series(6,55) i`, anchor) + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,source_broker,resolved_endpoints) +SELECT i,decode(lpad(to_hex(i),64,'0'),'hex'),$1,'YYZ',$2::timestamptz+i*interval '1 second',0,1,0,'fixture',$3 +FROM generate_series(6,55) i`, observer1, anchor, snapshot(high, nil)) + q.calls = 0 + page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil || len(page.Items) != 50 || q.calls != 1 { + t.Fatalf("50-row page: queries=%d err=%v", q.calls, err) + } + for _, item := range page.Items { + check(item.LatestObserver.ResolvedSource, item.LatestObserver.ResolvedDestination, high, nil) + } + t.Logf("50 historical rows use %d database query", q.calls) + + // Capture uses the ordinary insert; duplicate delivery cannot replace its snapshot. + exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +VALUES (decode($1,'hex'),4,0,1,'\x00','\x00',NOW(),NOW())`, fmt.Sprintf("%064x", 56)) + observation := ingest.InsertObservationParams{PacketHash: append(make([]byte, 31), 56), + ObserverID: observer1, IATA: "YYZ", HeardAt: anchor, HashSize: 1, SourceBroker: "fixture", + PayloadType: 4, ResolvedEndpoints: snapshot(high, none)} + for _, wantInserted := range []bool{true, false} { + q.calls = 0 + inserted, err := store.InsertObservation(ctx, observation) + if err != nil || inserted != wantInserted || q.calls != 1 { + t.Fatalf("snapshot insert: inserted=%v queries=%d err=%v", inserted, q.calls, err) + } + observation.ResolvedEndpoints = snapshot(old, nil) + observation.IATA = "YVR" + } + stored, err := store.GetPacket(ctx, observation.PacketHash) + if err != nil || len(stored.Observations) != 1 || stored.Observations[0].IATA != "YYZ" { + t.Fatalf("duplicate observation changed: %v", err) + } + check(stored.Observations[0].ResolvedSource, stored.Observations[0].ResolvedDestination, high, none) + // First adverts can be captured before their node exists. Empty old captures + // must allow detail-time lookup once that node has been advertised. + exec("INSERT INTO nodes(id,public_key,name,node_type) VALUES ($1,decode($2,'hex'),'Later discovered companion',1)", nodeID, key) + exec("UPDATE packets SET origin_pubkey=decode($2,'hex') WHERE packet_hash=$1", observation.PacketHash, key) + for _, raw := range []string{`{}`, `{"source":{"confidence":"none","nodes":[]}}`} { + exec("UPDATE packet_observations SET resolved_endpoints=$2::jsonb,source_broker=NULL WHERE packet_hash=$1", observation.PacketHash, raw) + q.calls = 0 + stored, err = store.GetPacket(ctx, observation.PacketHash) + if err != nil || q.calls != 4 || stored.Observations[0].SourceBroker != "" { + t.Fatalf("empty snapshot fallback or nullable legacy broker: queries=%d err=%v", q.calls, err) + } + if source := stored.Observations[0].ResolvedSource; source == nil || len(source.Nodes) != 1 || source.Nodes[0].Name == nil || *source.Nodes[0].Name != "Later discovered companion" { + t.Fatal("empty capture suppressed the later advertised node") + } + } + exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) +VALUES (decode($1,'hex'),4,0,1,'\x00','\x00',decode($2,'hex'),$3,$3)`, fmt.Sprintf("%064x", 57), key, anchor) + observation.PacketHash, observation.ResolvedEndpoints = append(make([]byte, 31), 57), nil + inserted, err := store.InsertObservation(ctx, observation) + if err != nil || !inserted { + t.Fatalf("uncaptured observation insert: %v", err) + } + var isNull bool + if err := tx.QueryRow(ctx, "SELECT resolved_endpoints IS NULL FROM packet_observations WHERE packet_hash=$1", observation.PacketHash).Scan(&isNull); err != nil || !isNull { + t.Fatalf("uncaptured resolution was not SQL NULL: %v", err) + } + empty, err := store.GetPacket(ctx, append(make([]byte, 31), 5)) + if err != nil || len(empty.Observations) != 0 { + t.Fatalf("packet awaiting its first observation: %v", err) + } +} + +func TestDecodePacketEndpointSnapshot(t *testing.T) { + for _, tc := range []struct { + raw string + captured bool + }{ + {"", false}, {"null", false}, {"[", false}, {"[]", false}, {"{}", false}, + {`{"source":{"confidence":"none","nodes":[]}}`, false}, + {`{"destination":{"confidence":"none","nodes":[]}}`, false}, + {`{"source":{},"destination":null}`, false}, + {`{"source":{"confidence":"high","nodes":[{"publicKey":"aa"}]}}`, true}, + {`{"destination":{"confidence":"ambiguous","nodes":[{"publicKey":"aa"},{"publicKey":"ab"}]}}`, true}, + } { + _, captured := decodePacketEndpointSnapshot(json.RawMessage(tc.raw)) + if captured != tc.captured { + t.Errorf("snapshot %q: captured=%v, want %v", tc.raw, captured, tc.captured) + } + } +} diff --git a/db/packets.go b/db/packets.go index f806a42d..b03eda68 100644 --- a/db/packets.go +++ b/db/packets.go @@ -84,6 +84,14 @@ func buildLatestObserverPath(pathLengthByte, hashSize, hopCount *int16, pathByte return pathLength, pathBytesHex } +func decodePacketEndpointSnapshot(raw json.RawMessage) (api.PacketEndpointSnapshot, bool) { + var snapshot *api.PacketEndpointSnapshot + if len(raw) == 0 || json.Unmarshal(raw, &snapshot) != nil || snapshot == nil || !snapshot.HasResolvedNodes() { + return api.PacketEndpointSnapshot{}, false + } + return *snapshot, true +} + func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int16, iatas []string, scopes []string, since, until time.Time, cursor int64, limit int32) (api.Page[api.PacketSummary], error) { if len(iatas) > 0 { return s.listPacketsByIATAs(ctx, payloadTypes, routeTypes, iatas, scopes, since, until, cursor, limit) @@ -130,10 +138,13 @@ func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int1 ObservationCount: int32(v.ObservationCount), } if v.LatestObserverID != (uuid.UUID{}) { + endpoints, _ := decodePacketEndpointSnapshot(v.LatestObserverResolvedEndpoints) item.LatestObserver = &api.PacketLatestObserver{ - ID: v.LatestObserverID, - DisplayName: v.LatestObserverName, - IATA: v.LatestObserverIata, + ID: v.LatestObserverID, + DisplayName: v.LatestObserverName, + IATA: v.LatestObserverIata, + ResolvedSource: endpoints.Source, + ResolvedDestination: endpoints.Destination, } item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, @@ -209,10 +220,13 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes ObservationCount: int32(v.ObservationCount), } if v.LatestObserverID != (uuid.UUID{}) { + endpoints, _ := decodePacketEndpointSnapshot(v.LatestObserverResolvedEndpoints) item.LatestObserver = &api.PacketLatestObserver{ - ID: v.LatestObserverID, - DisplayName: v.LatestObserverName, - IATA: v.LatestObserverIata, + ID: v.LatestObserverID, + DisplayName: v.LatestObserverName, + IATA: v.LatestObserverIata, + ResolvedSource: endpoints.Source, + ResolvedDestination: endpoints.Destination, } item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, @@ -265,10 +279,13 @@ func (s *Store) ListPacketsAfterID(ctx context.Context, afterObservationID int64 ObservationCount: int32(v.ObservationCount), } if v.LatestObserverID != (uuid.UUID{}) { + endpoints, _ := decodePacketEndpointSnapshot(v.LatestObserverResolvedEndpoints) item.LatestObserver = &api.PacketLatestObserver{ - ID: v.LatestObserverID, - DisplayName: v.LatestObserverName, - IATA: v.LatestObserverIata, + ID: v.LatestObserverID, + DisplayName: v.LatestObserverName, + IATA: v.LatestObserverIata, + ResolvedSource: endpoints.Source, + ResolvedDestination: endpoints.Destination, } // Inner join here (unlike ListPackets/listPacketsByIATAs' LEFT JOIN LATERAL), so // these are never nil when an observer was joined at all. @@ -343,7 +360,10 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, ObservationCount: int32(len(obsRows)), Observations: make([]api.PacketObservationDetail, 0, len(obsRows)), } - minHeardAt := obsRows[0].HeardAt.Time + var minHeardAt time.Time + if len(obsRows) > 0 { + minHeardAt = obsRows[0].HeardAt.Time + } if len(obsRows) > 1 { maxHeardAt := obsRows[0].HeardAt.Time for _, v := range obsRows[1:] { @@ -420,16 +440,9 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, destHashByte = []byte{path.Destination} } } - // ADVERT's source is an exact pubkey match, not ambiguous like the above -- and unlike - // them it doesn't depend on IATA, so resolve it once here rather than per observation. + // Legacy ADVERT sources need an exact lookup, at most once for this packet. var resolvedAdvertSource *api.ResolvedNode - if row.PayloadType == int16(meshcore.PayloadTypeAdvert) && row.OriginPubkey != nil { - if nodeID, err := s.GetNodeByPubkey(ctx, row.OriginPubkey); err == nil { - if nodes, err := s.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { - resolvedAdvertSource = nodes[nodeID] - } - } - } + advertSourceLookedUp := false for _, v := range obsRows { obs := api.PacketObservationDetail{ ID: v.ID, @@ -442,9 +455,11 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, HashSize: v.HashSize, HopCount: v.HopCount, }, - RSSI: v.Rssi, - SNR: v.Snr, - SourceBroker: *v.SourceBroker, + RSSI: v.Rssi, + SNR: v.Snr, + } + if v.SourceBroker != nil { + obs.SourceBroker = *v.SourceBroker } prop := int32(v.HeardAt.Time.Sub(minHeardAt).Milliseconds()) obs.PropagationTimeMs = &prop @@ -472,23 +487,36 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, } } obs.ResolvedPath = resolvedPath - if row.PayloadType == int16(meshcore.PayloadTypeAdvert) { - hop := api.ResolveExactNode(resolvedAdvertSource) - obs.ResolvedSource = &hop - } else if len(sourceHashByte) == 1 { - if r, err := s.ResolvePathHashes(ctx, v.Iata, [][]byte{sourceHashByte}); err != nil { - log.Printf("store: source resolution failed for observation %d: %v", v.ID, err) - } else { - hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] + if endpoints, captured := decodePacketEndpointSnapshot(v.ResolvedEndpoints); captured { + obs.ResolvedSource = endpoints.Source + obs.ResolvedDestination = endpoints.Destination + } else { + if row.PayloadType == int16(meshcore.PayloadTypeAdvert) { + if !advertSourceLookedUp && row.OriginPubkey != nil { + if nodeID, err := s.GetNodeByPubkey(ctx, row.OriginPubkey); err == nil { + if nodes, err := s.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { + resolvedAdvertSource = nodes[nodeID] + } + } + advertSourceLookedUp = true + } + hop := api.ResolveExactNode(resolvedAdvertSource) obs.ResolvedSource = &hop + } else if len(sourceHashByte) == 1 { + if r, err := s.ResolveEndpointHashes(ctx, v.Iata, [][]byte{sourceHashByte}); err != nil { + log.Printf("store: source resolution failed for observation %d: %v", v.ID, err) + } else { + hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] + obs.ResolvedSource = &hop + } } - } - if len(destHashByte) == 1 { - if r, err := s.ResolvePathHashes(ctx, v.Iata, [][]byte{destHashByte}); err != nil { - log.Printf("store: destination resolution failed for observation %d: %v", v.ID, err) - } else { - hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] - obs.ResolvedDestination = &hop + if len(destHashByte) == 1 { + if r, err := s.ResolveEndpointHashes(ctx, v.Iata, [][]byte{destHashByte}); err != nil { + log.Printf("store: destination resolution failed for observation %d: %v", v.ID, err) + } else { + hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] + obs.ResolvedDestination = &hop + } } } if row.PayloadType == int16(meshcore.PayloadTypeTrace) && len(traceRawHashes) > 0 { @@ -557,6 +585,7 @@ func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservatio CodingRate: &o.CodingRate, SourceBroker: &o.SourceBroker, PayloadType: &o.PayloadType, + ResolvedEndpoints: o.ResolvedEndpoints, } row, err := s.q.InsertObservation(ctx, params) if errors.Is(err, pgx.ErrNoRows) { diff --git a/db/packets_test.go b/db/packets_test.go index f2c8248f..6541ba35 100644 --- a/db/packets_test.go +++ b/db/packets_test.go @@ -204,7 +204,7 @@ func TestListPackets_LatestObserverPathFields(t *testing.T) { if obs.PathBytes == nil || *obs.PathBytes != "a1b2" { t.Errorf("expected pathBytes a1b2, got %v", obs.PathBytes) } - // Resolution stays a detail-view-only feature on this list endpoint -- deliberately unset. + // Legacy rows without a captured snapshot still omit resolved endpoints on lists. if obs.ResolvedPath != nil || obs.ResolvedSource != nil || obs.ResolvedDestination != nil { t.Error("expected no resolved path/source/destination on the list endpoint") } diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 9771a265..c2d73057 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -425,10 +425,11 @@ SELECT COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes + po.path_bytes AS latest_observer_path_bytes, + po.resolved_endpoints AS latest_observer_resolved_endpoints FROM packets p LEFT JOIN LATERAL ( - SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes, resolved_endpoints FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -522,12 +523,13 @@ SELECT COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes + po.path_bytes AS latest_observer_path_bytes, + po.resolved_endpoints AS latest_observer_resolved_endpoints FROM page sh CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( - SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes, resolved_endpoints FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -554,6 +556,7 @@ SELECT po.hash_size AS latest_observer_hash_size, po.hop_count AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes, + po.resolved_endpoints AS latest_observer_resolved_endpoints, ts.name AS scope_name FROM packets p JOIN packet_observations po ON po.packet_hash = p.packet_hash @@ -622,9 +625,10 @@ INSERT INTO packet_observations ( bandwidth_khz, coding_rate, source_broker, - payload_type + payload_type, + resolved_endpoints ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING RETURNING *; @@ -1236,6 +1240,21 @@ WHERE ns.iata = $1 AND n.node_type IN (2, 3) AND ns.prefix_1 = ANY($2::bytea[]); +-- name: ResolveEndpointHashes :many +-- Logical endpoints can be any advertised role, unlike intermediate relay hops. +-- Endpoint hashes are always one byte; use the existing (iata, prefix_1) index. +-- LIMIT 1 keeps generic plans on a node PK lookup per candidate instead of +-- flattening the join into a scan of all nodes. The PK already guarantees one row. +SELECT ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM node_short_ids ns +CROSS JOIN LATERAL ( + SELECT id, name, latitude, longitude, public_key + FROM nodes WHERE id = ns.node_id + LIMIT 1 +) n +WHERE ns.iata = $1 + AND ns.prefix_1 = ANY($2::bytea[]); + -- name: ResolvePathHashesP2 :many SELECT ns.prefix_4 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key FROM node_short_ids ns diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index c34fe770..5742d194 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -1167,6 +1167,21 @@ func (mr *MockQuerierMockRecorder) RefreshTopTalkers(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopTalkers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopTalkers), ctx) } +// ResolveEndpointHashes mocks base method. +func (m *MockQuerier) ResolveEndpointHashes(ctx context.Context, arg db.ResolveEndpointHashesParams) ([]db.ResolveEndpointHashesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveEndpointHashes", ctx, arg) + ret0, _ := ret[0].([]db.ResolveEndpointHashesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveEndpointHashes indicates an expected call of ResolveEndpointHashes. +func (mr *MockQuerierMockRecorder) ResolveEndpointHashes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveEndpointHashes", reflect.TypeOf((*MockQuerier)(nil).ResolveEndpointHashes), ctx, arg) +} + // ResolvePathHashesP1 mocks base method. func (m *MockQuerier) ResolvePathHashesP1(ctx context.Context, arg db.ResolvePathHashesP1Params) ([]db.ResolvePathHashesP1Row, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 9a403553..5bed2693 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -292,6 +292,7 @@ type PacketObservation struct { CodingRate *int16 `json:"coding_rate"` SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` + ResolvedEndpoints []byte `json:"resolved_endpoints"` } type Region struct { diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 1c5f8c93..7a2b051f 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -192,6 +192,11 @@ type Querier interface { RefreshTopNodes(ctx context.Context) error RefreshTopObservers(ctx context.Context) error RefreshTopTalkers(ctx context.Context) error + // Logical endpoints can be any advertised role, unlike intermediate relay hops. + // Endpoint hashes are always one byte; use the existing (iata, prefix_1) index. + // LIMIT 1 keeps generic plans on a node PK lookup per candidate instead of + // flattening the join into a scan of all nodes. The PK already guarantees one row. + ResolveEndpointHashes(ctx context.Context, arg ResolveEndpointHashesParams) ([]ResolveEndpointHashesRow, error) // ============================================================ // HELPERS // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 8581c4c0..2d2a05de 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1721,12 +1721,13 @@ INSERT INTO packet_observations ( bandwidth_khz, coding_rate, source_broker, - payload_type + payload_type, + resolved_endpoints ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING -RETURNING id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type +RETURNING id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type, resolved_endpoints ` type InsertObservationParams struct { @@ -1747,6 +1748,7 @@ type InsertObservationParams struct { CodingRate *int16 `json:"coding_rate"` SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` + ResolvedEndpoints []byte `json:"resolved_endpoints"` } // ============================================================ @@ -1771,6 +1773,7 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa arg.CodingRate, arg.SourceBroker, arg.PayloadType, + arg.ResolvedEndpoints, ) var i PacketObservation err := row.Scan( @@ -1792,6 +1795,7 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa &i.CodingRate, &i.SourceBroker, &i.PayloadType, + &i.ResolvedEndpoints, ) return i, err } @@ -2481,7 +2485,7 @@ func (q *Queries) ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNod } const listObservationsForPacket = `-- name: ListObservationsForPacket :many -SELECT po.id, po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, po.path_bytes, po.rssi, po.snr, po.propagation_time_ms, po.radio_freq_mhz, po.spread_factor, po.bandwidth_khz, po.coding_rate, po.source_broker, po.payload_type, o.display_name AS observer_name +SELECT po.id, po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, po.path_bytes, po.rssi, po.snr, po.propagation_time_ms, po.radio_freq_mhz, po.spread_factor, po.bandwidth_khz, po.coding_rate, po.source_broker, po.payload_type, po.resolved_endpoints, o.display_name AS observer_name FROM packet_observations po LEFT JOIN observers o ON o.id = po.observer_id WHERE po.packet_hash = $1 @@ -2507,6 +2511,7 @@ type ListObservationsForPacketRow struct { CodingRate *int16 `json:"coding_rate"` SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` + ResolvedEndpoints []byte `json:"resolved_endpoints"` ObserverName *string `json:"observer_name"` } @@ -2538,6 +2543,7 @@ func (q *Queries) ListObservationsForPacket(ctx context.Context, packetHash []by &i.CodingRate, &i.SourceBroker, &i.PayloadType, + &i.ResolvedEndpoints, &i.ObserverName, ); err != nil { return nil, err @@ -2756,10 +2762,11 @@ SELECT COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes + po.path_bytes AS latest_observer_path_bytes, + po.resolved_endpoints AS latest_observer_resolved_endpoints FROM packets p LEFT JOIN LATERAL ( - SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes, resolved_endpoints FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -2789,21 +2796,22 @@ type ListPacketsParams struct { } type ListPacketsRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` - LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` - LatestObserverHashSize int16 `json:"latest_observer_hash_size"` - LatestObserverHopCount int16 `json:"latest_observer_hop_count"` - LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + LatestObserverResolvedEndpoints []byte `json:"latest_observer_resolved_endpoints"` } // Returns packets with the latest observation rolled in for display. @@ -2842,6 +2850,7 @@ func (q *Queries) ListPackets(ctx context.Context, arg ListPacketsParams) ([]Lis &i.LatestObserverHashSize, &i.LatestObserverHopCount, &i.LatestObserverPathBytes, + &i.LatestObserverResolvedEndpoints, ); err != nil { return nil, err } @@ -2868,6 +2877,7 @@ SELECT po.hash_size AS latest_observer_hash_size, po.hop_count AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes, + po.resolved_endpoints AS latest_observer_resolved_endpoints, ts.name AS scope_name FROM packets p JOIN packet_observations po ON po.packet_hash = p.packet_hash @@ -2892,20 +2902,21 @@ type ListPacketsAfterIDParams struct { } type ListPacketsAfterIDRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` - LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` - LatestObserverHashSize int16 `json:"latest_observer_hash_size"` - LatestObserverHopCount int16 `json:"latest_observer_hop_count"` - LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` - ScopeName *string `json:"scope_name"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + LatestObserverResolvedEndpoints []byte `json:"latest_observer_resolved_endpoints"` + ScopeName *string `json:"scope_name"` } // Returns packets with observations after the given observation ID, ordered oldest first. @@ -2940,6 +2951,7 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID &i.LatestObserverHashSize, &i.LatestObserverHopCount, &i.LatestObserverPathBytes, + &i.LatestObserverResolvedEndpoints, &i.ScopeName, ); err != nil { return nil, err @@ -3015,12 +3027,13 @@ SELECT COALESCE(po.path_length_byte, 0::smallint) AS latest_observer_path_length_byte, COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, - po.path_bytes AS latest_observer_path_bytes + po.path_bytes AS latest_observer_path_bytes, + po.resolved_endpoints AS latest_observer_resolved_endpoints FROM page sh CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( - SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes, resolved_endpoints FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -3044,24 +3057,25 @@ type ListPacketsByIATAsParams struct { } type ListPacketsByIATAsRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` - ScanSaturated bool `json:"scan_saturated"` - ScanFloor pgtype.Timestamptz `json:"scan_floor"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` - LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` - LatestObserverHashSize int16 `json:"latest_observer_hash_size"` - LatestObserverHopCount int16 `json:"latest_observer_hop_count"` - LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` + ScanSaturated bool `json:"scan_saturated"` + ScanFloor pgtype.Timestamptz `json:"scan_floor"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + LatestObserverResolvedEndpoints []byte `json:"latest_observer_resolved_endpoints"` } // IATA-filtered packet list, driven from idx_observations_iata_heard. @@ -3115,6 +3129,7 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs &i.LatestObserverHashSize, &i.LatestObserverHopCount, &i.LatestObserverPathBytes, + &i.LatestObserverResolvedEndpoints, ); err != nil { return nil, err } @@ -3444,6 +3459,63 @@ func (q *Queries) RefreshTopTalkers(ctx context.Context) error { return err } +const resolveEndpointHashes = `-- name: ResolveEndpointHashes :many +SELECT ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM node_short_ids ns +CROSS JOIN LATERAL ( + SELECT id, name, latitude, longitude, public_key + FROM nodes WHERE id = ns.node_id + LIMIT 1 +) n +WHERE ns.iata = $1 + AND ns.prefix_1 = ANY($2::bytea[]) +` + +type ResolveEndpointHashesParams struct { + Iata string `json:"iata"` + Column2 [][]byte `json:"column_2"` +} + +type ResolveEndpointHashesRow struct { + Hash []byte `json:"hash"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` + PublicKey []byte `json:"public_key"` +} + +// Logical endpoints can be any advertised role, unlike intermediate relay hops. +// Endpoint hashes are always one byte; use the existing (iata, prefix_1) index. +// LIMIT 1 keeps generic plans on a node PK lookup per candidate instead of +// flattening the join into a scan of all nodes. The PK already guarantees one row. +func (q *Queries) ResolveEndpointHashes(ctx context.Context, arg ResolveEndpointHashesParams) ([]ResolveEndpointHashesRow, error) { + rows, err := q.db.Query(ctx, resolveEndpointHashes, arg.Iata, arg.Column2) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ResolveEndpointHashesRow{} + for rows.Next() { + var i ResolveEndpointHashesRow + if err := rows.Scan( + &i.Hash, + &i.NodeID, + &i.Name, + &i.Latitude, + &i.Longitude, + &i.PublicKey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const resolvePathHashesP1 = `-- name: ResolvePathHashesP1 :many diff --git a/db/store.go b/db/store.go index d2699a7b..a22b9600 100644 --- a/db/store.go +++ b/db/store.go @@ -83,6 +83,32 @@ func (s *Store) ResolvePathHashes(ctx context.Context, iata string, hashes [][]b return result, nil } +// ResolveEndpointHashes matches one-byte packet endpoints across advertised roles. +// Keep this separate from the infrastructure-only intermediate path resolver. +func (s *Store) ResolveEndpointHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + if len(hashes) == 0 { + return nil, nil + } + for _, hash := range hashes { + if len(hash) != 1 { + return nil, nil + } + } + rows, err := s.q.ResolveEndpointHashes(ctx, sqlc.ResolveEndpointHashesParams{Iata: iata, Column2: hashes}) + if err != nil { + return nil, err + } + result := make(map[string][]api.ResolvedPathEntry) + for _, row := range rows { + key := hex.EncodeToString(row.Hash) + result[key] = append(result[key], api.ResolvedPathEntry{ + NodeID: row.NodeID, Name: row.Name, Latitude: row.Latitude, + Longitude: row.Longitude, PublicKey: row.PublicKey, + }) + } + return result, nil +} + // nullableUUID returns nil for a zero UUID, or a pointer to the UUID otherwise. func nullableUUID(id uuid.UUID) *uuid.UUID { if id == (uuid.UUID{}) { diff --git a/docs/docs.go b/docs/docs.go index 94cc9198..b9f685a5 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -3191,7 +3191,7 @@ const docTemplate = `{ "type": "string" }, "pathLength": { - "description": "PathLength/PathBytes are cheap -- already-stored columns on packet_observations -- and\npopulated everywhere PacketLatestObserver appears: the REST list/backfill endpoints and\nthe WS feed alike. ResolvedPath/ResolvedSource/ResolvedDestination require a per-hash DB\nresolution lookup; they're populated on the WS feed (already computed once at ingest, so\neffectively free there) but deliberately left nil on the REST endpoints, which are\npaginated/high-volume and used only for scrollback and reconnect-gap backfill -- full\nresolution stays a GET /packets/{packetHash}-only feature.", + "description": "PathLength/PathBytes and captured endpoint resolutions are stored on the observation,\nso list/backfill reads need no per-hash resolution queries. Legacy observations have\nno endpoint snapshot. ResolvedPath remains a detail/opted-in WS feature.", "allOf": [ { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketPathLength" @@ -3256,7 +3256,7 @@ const docTemplate = `{ } }, "resolvedSource": { - "description": "ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type\ncarries a resolvable one: an exact match for ADVERT's full pubkey, an ambiguous\nhash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte\nsource/destination hashes. Nil when the payload type doesn't carry one at all (e.g.\nGRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and\nResolveExactNode for how each is built.", + "description": "ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type\ncarries one. Prefer the snapshot captured at ingest; legacy observations without a\nsnapshot use the current node registry. Endpoint matching itself is unchanged:\nan exact match for ADVERT's full pubkey, an ambiguous\nhash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte\nsource/destination hashes. Nil when the payload type doesn't carry one at all (e.g.\nGRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and\nResolveExactNode for how each is built.", "allOf": [ { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" diff --git a/docs/swagger.json b/docs/swagger.json index c943d3b2..7db537cc 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -3189,7 +3189,7 @@ "type": "string" }, "pathLength": { - "description": "PathLength/PathBytes are cheap -- already-stored columns on packet_observations -- and\npopulated everywhere PacketLatestObserver appears: the REST list/backfill endpoints and\nthe WS feed alike. ResolvedPath/ResolvedSource/ResolvedDestination require a per-hash DB\nresolution lookup; they're populated on the WS feed (already computed once at ingest, so\neffectively free there) but deliberately left nil on the REST endpoints, which are\npaginated/high-volume and used only for scrollback and reconnect-gap backfill -- full\nresolution stays a GET /packets/{packetHash}-only feature.", + "description": "PathLength/PathBytes and captured endpoint resolutions are stored on the observation,\nso list/backfill reads need no per-hash resolution queries. Legacy observations have\nno endpoint snapshot. ResolvedPath remains a detail/opted-in WS feature.", "allOf": [ { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketPathLength" @@ -3254,7 +3254,7 @@ } }, "resolvedSource": { - "description": "ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type\ncarries a resolvable one: an exact match for ADVERT's full pubkey, an ambiguous\nhash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte\nsource/destination hashes. Nil when the payload type doesn't carry one at all (e.g.\nGRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and\nResolveExactNode for how each is built.", + "description": "ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type\ncarries one. Prefer the snapshot captured at ingest; legacy observations without a\nsnapshot use the current node registry. Endpoint matching itself is unchanged:\nan exact match for ADVERT's full pubkey, an ambiguous\nhash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte\nsource/destination hashes. Nil when the payload type doesn't carry one at all (e.g.\nGRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and\nResolveExactNode for how each is built.", "allOf": [ { "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index c6667f39..5bb289bc 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -619,13 +619,9 @@ definitions: allOf: - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PacketPathLength' description: |- - PathLength/PathBytes are cheap -- already-stored columns on packet_observations -- and - populated everywhere PacketLatestObserver appears: the REST list/backfill endpoints and - the WS feed alike. ResolvedPath/ResolvedSource/ResolvedDestination require a per-hash DB - resolution lookup; they're populated on the WS feed (already computed once at ingest, so - effectively free there) but deliberately left nil on the REST endpoints, which are - paginated/high-volume and used only for scrollback and reconnect-gap backfill -- full - resolution stays a GET /packets/{packetHash}-only feature. + PathLength/PathBytes and captured endpoint resolutions are stored on the observation, + so list/backfill reads need no per-hash resolution queries. Legacy observations have + no endpoint snapshot. ResolvedPath remains a detail/opted-in WS feature. resolvedDestination: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop' resolvedPath: @@ -670,7 +666,9 @@ definitions: - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ResolvedHop' description: |- ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type - carries a resolvable one: an exact match for ADVERT's full pubkey, an ambiguous + carries one. Prefer the snapshot captured at ingest; legacy observations without a + snapshot use the current node registry. Endpoint matching itself is unchanged: + an exact match for ADVERT's full pubkey, an ambiguous hash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte source/destination hashes. Nil when the payload type doesn't carry one at all (e.g. GRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and diff --git a/internal/api/packets.go b/internal/api/packets.go index 305fcfb9..c75aa6f6 100644 --- a/internal/api/packets.go +++ b/internal/api/packets.go @@ -17,13 +17,9 @@ type PacketLatestObserver struct { ID uuid.UUID `json:"id"` DisplayName *string `json:"displayName,omitempty"` IATA string `json:"iata"` - // PathLength/PathBytes are cheap -- already-stored columns on packet_observations -- and - // populated everywhere PacketLatestObserver appears: the REST list/backfill endpoints and - // the WS feed alike. ResolvedPath/ResolvedSource/ResolvedDestination require a per-hash DB - // resolution lookup; they're populated on the WS feed (already computed once at ingest, so - // effectively free there) but deliberately left nil on the REST endpoints, which are - // paginated/high-volume and used only for scrollback and reconnect-gap backfill -- full - // resolution stays a GET /packets/{packetHash}-only feature. + // PathLength/PathBytes and captured endpoint resolutions are stored on the observation, + // so list/backfill reads need no per-hash resolution queries. Legacy observations have + // no endpoint snapshot. ResolvedPath remains a detail/opted-in WS feature. PathLength *PacketPathLength `json:"pathLength,omitempty"` PathBytes *string `json:"pathBytes,omitempty"` // hex-encoded accumulated path hashes ResolvedPath []ResolvedHop `json:"resolvedPath,omitempty"` @@ -71,7 +67,9 @@ type PacketObservationDetail struct { SourceBroker string `json:"sourceBroker"` ResolvedPath []ResolvedHop `json:"resolvedPath"` // per-observation resolved path hashes // ResolvedSource/ResolvedDestination are the packet's endpoints, when the payload type - // carries a resolvable one: an exact match for ADVERT's full pubkey, an ambiguous + // carries one. Prefer the snapshot captured at ingest; legacy observations without a + // snapshot use the current node registry. Endpoint matching itself is unchanged: + // an exact match for ADVERT's full pubkey, an ambiguous // hash-prefix match (like intermediate hops) for TEXT_MESSAGE/PATH/ANON_REQ's 1-byte // source/destination hashes. Nil when the payload type doesn't carry one at all (e.g. // GRP_TXT/GRP_DATA/TRACE aren't node-to-node addressed) -- see BuildResolvedPath and @@ -96,6 +94,21 @@ type ResolvedHop struct { Nodes []ResolvedNode `json:"nodes"` // empty for "none", one for "high", multiple for "ambiguous" } +// PacketEndpointSnapshot is the internal storage shape for an observation's +// endpoint resolution at ingest. Names and confidence are preserved even when +// the current node registry changes. REST/WS expose the existing per-endpoint fields. +type PacketEndpointSnapshot struct { + Source *ResolvedHop `json:"source,omitempty"` + Destination *ResolvedHop `json:"destination,omitempty"` +} + +// HasResolvedNodes distinguishes a historical capture from an unresolved lookup +// that should be retried when a node becomes known (including a first advert). +func (s PacketEndpointSnapshot) HasResolvedNodes() bool { + return (s.Source != nil && len(s.Source.Nodes) > 0) || + (s.Destination != nil && len(s.Destination.Nodes) > 0) +} + // ResolvedNode is a node reference within a resolved path hop. type ResolvedNode struct { ID uuid.UUID `json:"id"` diff --git a/internal/api/packets_test.go b/internal/api/packets_test.go index f430f60a..2befa568 100644 --- a/internal/api/packets_test.go +++ b/internal/api/packets_test.go @@ -10,6 +10,28 @@ import ( "github.com/meshcore-go/meshcore-go" ) +func TestPacketEndpointSnapshotHasResolvedNodes(t *testing.T) { + none := &api.ResolvedHop{Confidence: "none", Nodes: []api.ResolvedNode{}} + known := &api.ResolvedHop{Confidence: "high", Nodes: []api.ResolvedNode{{PublicKey: "aa"}}} + for _, tc := range []struct { + name string + snapshot api.PacketEndpointSnapshot + want bool + }{ + {"absent", api.PacketEndpointSnapshot{}, false}, + {"unresolved source", api.PacketEndpointSnapshot{Source: none}, false}, + {"both unresolved", api.PacketEndpointSnapshot{Source: none, Destination: none}, false}, + {"source only", api.PacketEndpointSnapshot{Source: known, Destination: none}, true}, + {"destination only", api.PacketEndpointSnapshot{Destination: known}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := tc.snapshot.HasResolvedNodes(); got != tc.want { + t.Fatalf("resolved nodes = %v, want %v", got, tc.want) + } + }) + } +} + func TestPayloadTypeName(t *testing.T) { tests := []struct { input int16 diff --git a/internal/ingest/endpoint_matching_test.go b/internal/ingest/endpoint_matching_test.go new file mode 100644 index 00000000..1b600628 --- /dev/null +++ b/internal/ingest/endpoint_matching_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/hex" + "reflect" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/meshcore-go/meshcore-go" +) + +type endpointRoutingDB struct { + *stubDB + endpoints []string + paths []string +} + +func (s *endpointRoutingDB) ResolveEndpointHashes(_ context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + for _, hash := range hashes { + s.endpoints = append(s.endpoints, iata+":"+hex.EncodeToString(hash)) + } + return nil, nil +} +func (s *endpointRoutingDB) ResolvePathHashes(_ context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + for _, hash := range hashes { + s.paths = append(s.paths, iata+":"+hex.EncodeToString(hash)) + } + return nil, nil +} + +func TestHandlePacketSeparatesEndpointAndRelayMatching(t *testing.T) { + for _, kind := range []uint8{meshcore.PayloadTypeReq, meshcore.PayloadTypeResponse, meshcore.PayloadTypeTxtMsg, meshcore.PayloadTypePath} { + w, base := newTestWorker() + db := &endpointRoutingDB{stubDB: base} + w.db = db + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, kind, 0), + PathLength: 1, Path: []byte{0xcc}, Payload: append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...)} + w.handlePacket(context.Background(), "YVR", "0102", packetEnvelope(t, packet)) + if !reflect.DeepEqual(db.endpoints, []string{"YVR:aa", "YVR:bb"}) { + t.Errorf("payload %d endpoint dispatch: %v", kind, db.endpoints) + } + if !reflect.DeepEqual(db.paths, []string{"YVR:cc"}) { + t.Errorf("payload %d relay dispatch: %v", kind, db.paths) + } + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 16743f6b..c2e485fc 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -153,6 +153,9 @@ type DB interface { // ResolvePathHashes returns a list of node UUIDs for the given path hash prefixes and IATA. ResolvePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) + // ResolveEndpointHashes matches one-byte logical endpoints, including companions. + ResolveEndpointHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) + // UpsertChannel upserts a channel row by (hash, keyFingerprint) and returns its integer ID. // Pass nil keyFingerprint to record a hash-only row when the key is unknown. UpsertChannel(ctx context.Context, channelHash []byte, keyFingerprint []byte, name string, hashtag string) (int, error) diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 6e11dfef..94bd6fdf 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -252,6 +252,10 @@ func (s *stubDB) ResolvePathHashes(_ context.Context, _ string, _ [][]byte) (map return nil, nil } +func (s *stubDB) ResolveEndpointHashes(_ context.Context, _ string, _ [][]byte) (map[string][]api.ResolvedPathEntry, error) { + return nil, nil +} + func (s *stubDB) UpsertChannel(_ context.Context, _ []byte, _ []byte, _, _ string) (int, error) { s.upsertChannelCalls++ return 0, nil diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 44b5d2f8..4a16a04c 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -55,6 +55,7 @@ type InsertObservationParams struct { CodingRate int16 SourceBroker string PayloadType int16 + ResolvedEndpoints json.RawMessage } // RadioSettings holds the radio configuration for an observer, populated from @@ -754,6 +755,38 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if err != nil { log.Printf("ingest[%s]: db: get observer radio failed for %s: %v", w.cfg.BrokerName, pubkeyHex, err) } + // Save the same endpoint resolution used by the live event in the observation INSERT. + var resolvedSource, resolvedDestination *api.ResolvedHop + if packet.PayloadType() == meshcore.PayloadTypeAdvert && originPubkey != nil { + // Exact match: ADVERT carries the sender's real identity pubkey, not a + // short ambiguous hash prefix like the other resolvable payload types. + if nodeID, err := w.db.GetNodeByPubkey(ctx, originPubkey); err == nil { + if nodes, err := w.db.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { + hop := api.ResolveExactNode(nodes[nodeID]) + resolvedSource = &hop + } + } + } else if len(sourceHashByte) == 1 { + if r, err := w.db.ResolveEndpointHashes(ctx, iata, [][]byte{sourceHashByte}); err == nil { + hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] + resolvedSource = &hop + } + } + if len(destHashByte) == 1 { + if r, err := w.db.ResolveEndpointHashes(ctx, iata, [][]byte{destHashByte}); err == nil { + hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] + resolvedDestination = &hop + } + } + var resolvedEndpoints json.RawMessage + snapshot := api.PacketEndpointSnapshot{Source: resolvedSource, Destination: resolvedDestination} + if snapshot.HasResolvedNodes() { + resolvedEndpoints, err = json.Marshal(snapshot) + if err != nil { + log.Printf("ingest[%s]: endpoint snapshot encoding failed: %v", w.cfg.BrokerName, err) + resolvedEndpoints = nil // optional enrichment must not discard the observation + } + } oParams := InsertObservationParams{ PacketHash: packetHash[:], ObserverID: id, @@ -772,6 +805,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ CodingRate: radio.CR, SourceBroker: w.cfg.BrokerName, PayloadType: int16(packet.PayloadType()), + ResolvedEndpoints: resolvedEndpoints, } inserted, err := w.db.InsertObservation(ctx, oParams) if err != nil { @@ -839,29 +873,6 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } w.runCapabilityDetection(ctx, packet.PayloadType(), packet.PathHashSize(), resolvedIDs) - var resolvedSource, resolvedDestination *api.ResolvedHop - if packet.PayloadType() == meshcore.PayloadTypeAdvert && originPubkey != nil { - // Exact match: ADVERT carries the sender's real identity pubkey, not a - // short ambiguous hash prefix like the other resolvable payload types. - if nodeID, err := w.db.GetNodeByPubkey(ctx, originPubkey); err == nil { - if nodes, err := w.db.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { - hop := api.ResolveExactNode(nodes[nodeID]) - resolvedSource = &hop - } - } - } else if len(sourceHashByte) == 1 { - if r, err := w.db.ResolvePathHashes(ctx, iata, [][]byte{sourceHashByte}); err == nil { - hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] - resolvedSource = &hop - } - } - if len(destHashByte) == 1 { - if r, err := w.db.ResolvePathHashes(ctx, iata, [][]byte{destHashByte}); err == nil { - hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] - resolvedDestination = &hop - } - } - if inserted { w.handlePayloadTypeSideEffects(ctx, packet, iata, packetHash[:], radio, scopeID, matchedScope, pubkeyBytes, float32(parseNumber(envelope.SNR))) evt := packetObservationEvent{} diff --git a/internal/ingest/packet_endpoints_test.go b/internal/ingest/packet_endpoints_test.go new file mode 100644 index 00000000..61a47063 --- /dev/null +++ b/internal/ingest/packet_endpoints_test.go @@ -0,0 +1,123 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "math" + "reflect" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +type endpointCaptureDB struct { + *stubDB + node api.ResolvedNode + observed []InsertObservationParams + lookups int + pathLookups int + missingNode bool +} + +func (s *endpointCaptureDB) GetNodeByPubkey(context.Context, []byte) (uuid.UUID, error) { + if s.missingNode { + return uuid.Nil, errors.New("node not yet advertised") + } + return s.node.ID, nil +} + +func (s *endpointCaptureDB) UpsertNode(context.Context, UpsertNodeParams, RadioSettings) (uuid.UUID, error) { + s.missingNode = false + return s.node.ID, nil +} +func (s *endpointCaptureDB) GetNodesByIDs(context.Context, []uuid.UUID) (map[uuid.UUID]*api.ResolvedNode, error) { + return map[uuid.UUID]*api.ResolvedNode{s.node.ID: &s.node}, nil +} +func (s *endpointCaptureDB) ResolvePathHashes(_ context.Context, _ string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + if len(hashes) > 0 { + s.pathLookups++ + } + return nil, nil // a companion must not be found by the relay-only resolver +} + +func (s *endpointCaptureDB) ResolveEndpointHashes(_ context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + if len(hashes) == 0 { + return nil, nil + } + s.lookups++ + if iata != "YYZ" || s.missingNode { + return nil, nil + } + return map[string][]api.ResolvedPathEntry{ + "aa": {{NodeID: s.node.ID, Name: s.node.Name, PublicKey: []byte{0xaa}}, {NodeID: uuid.Nil, PublicKey: []byte{0xab}}}, + }, nil +} + +func (s *endpointCaptureDB) InsertObservation(_ context.Context, observation InsertObservationParams) (bool, error) { + s.observed = append(s.observed, observation) + return true, nil +} + +func TestHandlePacketCapturesEndpoints(t *testing.T) { + name := "Companion 👋" + for _, kind := range []string{"advert", "first advert", "direct message", "unresolved direct", "unaddressed", "encoding failure"} { + t.Run(kind, func(t *testing.T) { + w, base := newTestWorker() + db := &endpointCaptureDB{stubDB: base, node: api.ResolvedNode{ID: uuid.New(), Name: &name, PublicKey: "aa"}} + w.db = db + packet := buildAdvertPacket(t, false) + want := api.PacketEndpointSnapshot{} + switch kind { + case "advert": + hop := api.ResolveExactNode(&db.node) + want.Source = &hop + case "first advert": + db.missingNode = true + case "direct message", "unresolved direct": + // Destination, source, MAC and a minimal ciphertext envelope. + packet = &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeTxtMsg, 0), Payload: append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...)} + db.missingNode = kind == "unresolved direct" + resolved, _ := db.ResolveEndpointHashes(context.Background(), "YYZ", [][]byte{{0xaa}}) + source := api.BuildResolvedPath([][]byte{{0xaa}}, resolved)[0] + destination := api.BuildResolvedPath([][]byte{{0xbb}}, nil)[0] + want.Source, want.Destination = &source, &destination + db.lookups = 0 + case "unaddressed": + packet = buildTracePacket(t) + case "encoding failure": + nan := math.NaN() + db.node.Latitude = &nan + } + w.handlePacket(context.Background(), "YYZ", hex.EncodeToString([]byte{1, 2}), packetEnvelope(t, packet)) + if len(db.observed) != 1 { + t.Fatalf("observation was lost or written twice: %d", len(db.observed)) + } + if kind == "first advert" || kind == "unresolved direct" || kind == "unaddressed" || kind == "encoding failure" { + if db.observed[0].ResolvedEndpoints != nil { + t.Fatal("empty or failed endpoint resolution must remain SQL NULL") + } + if kind == "first advert" && db.missingNode { + t.Fatal("advert side effects did not make the node available for a later lookup") + } + return + } + var got api.PacketEndpointSnapshot + if err := json.Unmarshal(db.observed[0].ResolvedEndpoints, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("captured endpoints differ: got %+v, want %+v", got, want) + } + if kind == "direct message" && (db.lookups != 2 || db.pathLookups != 0) { + t.Fatalf("wrong resolver or repeated work: endpoint=%d relay=%d", db.lookups, db.pathLookups) + } + }) + } +} From 5c5de43e8305f1f6808dac498e600796a4324414 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 9 Sep 2026 18:53:30 -0400 Subject: [PATCH 022/141] perf(db): bound route pages to the requested IATA (#118) --- .../027_known_routes_iata_last_seen.sql | 9 + db/queries/queries.sql | 20 ++- db/routes_integration_test.go | 155 ++++++++++++++++++ db/sqlc/querier.go | 3 + db/sqlc/queries.sql.go | 20 ++- 5 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 db/migrations/027_known_routes_iata_last_seen.sql create mode 100644 db/routes_integration_test.go diff --git a/db/migrations/027_known_routes_iata_last_seen.sql b/db/migrations/027_known_routes_iata_last_seen.sql new file mode 100644 index 00000000..b0bae7b2 --- /dev/null +++ b/db/migrations/027_known_routes_iata_last_seen.sql @@ -0,0 +1,9 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- RunMigrations executes each file outside an explicit transaction. Keep this +-- as one statement so other subscribers can keep writing during the build. +-- If interrupted, check pg_index.indisvalid and remove this named index before +-- retrying; do not silently accept an invalid index with IF NOT EXISTS. +CREATE INDEX CONCURRENTLY idx_known_routes_iata_last_seen +ON known_routes (iata, last_seen DESC); diff --git a/db/queries/queries.sql b/db/queries/queries.sql index c2d73057..5de50eba 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1150,12 +1150,30 @@ ON CONFLICT (iata, path_key) DO UPDATE SET observation_count = known_routes.observation_count + 1; -- name: ListKnownRoutes :many +-- Only one branch runs. Keep the IATA range ordered by the composite index: +-- generic plans can otherwise prefer scanning the global timestamp index. +-- The text equality preserves exact input matching, including trailing spaces. +( SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes -WHERE ($1 = '' OR iata = $1) +WHERE $1 = '' AND ($2 = 0 OR hop_count = $2) AND ($3::timestamptz IS NULL OR last_seen < $3) ORDER BY last_seen DESC +LIMIT $4 +) +UNION ALL +( +SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count +FROM known_routes +WHERE $1 <> '' AND iata::text = $1 + AND iata >= $1::bpchar AND iata <= $1::bpchar + AND ($2 = 0 OR hop_count = $2) + AND ($3::timestamptz IS NULL OR last_seen < $3) +ORDER BY iata, last_seen DESC +LIMIT $4 +) +ORDER BY last_seen DESC LIMIT $4; -- name: SearchKnownRoutes :many diff --git a/db/routes_integration_test.go b/db/routes_integration_test.go new file mode 100644 index 00000000..5156dc24 --- /dev/null +++ b/db/routes_integration_test.go @@ -0,0 +1,155 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + _ "embed" + "encoding/json" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +//go:embed queries/queries.sql +var routeTestQueries string + +// All synthetic rows and indexes are temporary and rolled back. The sparse +// region sits behind 100,000 newer routes to expose global timestamp scans. +func TestListKnownRoutesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + _, err = tx.Exec(ctx, ` +CREATE TEMP TABLE known_routes (LIKE public.known_routes INCLUDING ALL) ON COMMIT DROP; +INSERT INTO known_routes (id,path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) +SELECT i, decode(md5(i::text),'hex'), ARRAY[md5(i::text)::uuid], ARRAY['\x01'::bytea], + CASE WHEN i > 100000 THEN 'GPT' ELSE 'YYZ' END, i % 4 + 2, + '2026-01-01'::timestamptz - i * interval '1 second', + '2026-01-01'::timestamptz - i * interval '1 second' +FROM generate_series(1,100100) i; +ANALYZE known_routes;`) + if err != nil { + t.Fatal(err) + } + migration, err := migrationFiles.ReadFile("migrations/027_known_routes_iata_last_seen.sql") + if err != nil { + t.Fatal(err) + } + // A temporary table inside this rollback-only fixture cannot be indexed + // concurrently. Use the migration's exact definition with that option off. + if _, err := tx.Exec(ctx, strings.Replace(string(migration), "CREATE INDEX CONCURRENTLY", "CREATE INDEX", 1)); err != nil { + t.Fatal(err) + } + queries := strings.ReplaceAll(routeTestQueries, "\r\n", "\n") + querySQL := strings.Split(strings.Split(queries, "-- name: ListKnownRoutes :many\n")[1], "\n-- name:")[0] + if _, err := tx.Exec(ctx, "PREPARE route_page AS "+querySQL); err != nil { + t.Fatal(err) + } + q := sqlc.New(tx) + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + t.Run(mode, func(t *testing.T) { + if _, err := tx.Exec(ctx, "SET LOCAL plan_cache_mode = "+mode); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, iata string + hop, cursor, limit, first, stride, count int + }{ + {"global", "", 0, 0, 3, 1, 1, 3}, + {"common", "YYZ", 0, 0, 3, 1, 1, 3}, + {"sparse", "GPT", 0, 0, 50, 100001, 1, 50}, + {"sparse hop", "GPT", 2, 0, 50, 100004, 4, 25}, + {"global hop", "", 2, 0, 3, 4, 4, 3}, + {"global cursor", "", 0, 3, 3, 4, 1, 3}, + {"sparse cursor", "GPT", 0, 100010, 3, 100011, 1, 3}, + {"sparse hop cursor", "GPT", 2, 100010, 3, 100012, 4, 3}, + {"partial page", "GPT", 0, 100098, 50, 100099, 1, 2}, + {"end", "GPT", 0, 100100, 50, 0, 1, 0}, + {"unknown", "ZZZ", 0, 0, 50, 0, 1, 0}, + {"lowercase", "gpt", 0, 0, 50, 0, 1, 0}, + {"trailing space", "GPT ", 0, 0, 50, 0, 1, 0}, + {"overlong", "GPTX", 0, 0, 50, 0, 1, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + cursor := pgtype.Timestamptz{} + if tc.cursor != 0 { + cursor = pgtype.Timestamptz{Time: anchor.Add(-time.Duration(tc.cursor) * time.Second), Valid: true} + } + rows, err := q.ListKnownRoutes(ctx, sqlc.ListKnownRoutesParams{Column1: tc.iata, Column2: int32(tc.hop), Column3: cursor, Limit: int32(tc.limit)}) + if err != nil { + t.Fatal(err) + } + if len(rows) != tc.count { + t.Fatalf("got %d rows, want %d", len(rows), tc.count) + } + for i, r := range rows { + want := int64(tc.first + i*tc.stride) + wantIATA := "YYZ" + if want > 100000 { + wantIATA = "GPT" + } + if r.ID != want || r.Iata != wantIATA || r.HopCount != int32(want%4+2) || r.ObservationCount != 1 || + len(r.NodeIds) != 1 || len(r.HashPrefix) != 1 || len(r.HashPrefix[0]) != 1 || r.HashPrefix[0][0] != 1 || + !r.LastSeen.Time.Equal(anchor.Add(-time.Duration(want)*time.Second)) || !r.FirstSeen.Time.Equal(r.LastSeen.Time) { + t.Errorf("unexpected route %d at row %d; want %d", r.ID, i, want) + } + } + }) + } + for _, iata := range []string{"GPT", "YYZ", ""} { + var planJSON []byte + if err := tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE route_page('"+iata+"',0,NULL,50)").Scan(&planJSON); err != nil { + t.Fatal(err) + } + var plans []struct { + Plan map[string]any + ExecutionTime float64 `json:"Execution Time"` + } + if err := json.Unmarshal(planJSON, &plans); err != nil { + t.Fatal(err) + } + var visited float64 + var walk func(map[string]any) + walk = func(p map[string]any) { + if p["Relation Name"] == "known_routes" { + rows, _ := p["Actual Rows"].(float64) + removed, _ := p["Rows Removed by Filter"].(float64) + loops, _ := p["Actual Loops"].(float64) + visited += (rows + removed) * loops + } + if children, ok := p["Plans"].([]any); ok { + for _, child := range children { + walk(child.(map[string]any)) + } + } + } + walk(plans[0].Plan) + t.Logf("100,100 routes / 50-row page IATA=%q: %.3f ms, %.0f rows examined", iata, plans[0].ExecutionTime, visited) + if visited > 200 { + t.Errorf("page IATA=%q examined %.0f routes; budget 200", iata, visited) + } + } + }) + } +} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 7a2b051f..dc30ab23 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -127,6 +127,9 @@ type Querier interface { // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) ListIATAs(ctx context.Context) ([]IataCode, error) + // Only one branch runs. Keep the IATA range ordered by the composite index: + // generic plans can otherwise prefer scanning the global timestamp index. + // The text equality preserves exact input matching, including trailing spaces. ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) // Returns messages after the given message ID, ordered oldest first. // Used for WS reconnect backfill. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 2d2a05de..3a7ff942 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2168,13 +2168,28 @@ func (q *Queries) ListIATAs(ctx context.Context) ([]IataCode, error) { } const listKnownRoutes = `-- name: ListKnownRoutes :many +( SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes -WHERE ($1 = '' OR iata = $1) +WHERE $1 = '' AND ($2 = 0 OR hop_count = $2) AND ($3::timestamptz IS NULL OR last_seen < $3) ORDER BY last_seen DESC LIMIT $4 +) +UNION ALL +( +SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count +FROM known_routes +WHERE $1 <> '' AND iata::text = $1 + AND iata >= $1::bpchar AND iata <= $1::bpchar + AND ($2 = 0 OR hop_count = $2) + AND ($3::timestamptz IS NULL OR last_seen < $3) +ORDER BY iata, last_seen DESC +LIMIT $4 +) +ORDER BY last_seen DESC +LIMIT $4 ` type ListKnownRoutesParams struct { @@ -2195,6 +2210,9 @@ type ListKnownRoutesRow struct { ObservationCount int64 `json:"observation_count"` } +// Only one branch runs. Keep the IATA range ordered by the composite index: +// generic plans can otherwise prefer scanning the global timestamp index. +// The text equality preserves exact input matching, including trailing spaces. func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) { rows, err := q.db.Query(ctx, listKnownRoutes, arg.Column1, From f3900860589660585c5f1e8ea6f069e25c6d8d75 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 9 Sep 2026 20:25:07 -0400 Subject: [PATCH 023/141] fix(channels): preserve timestamp ties in page cursors (#132) --- db/channel_cursor_integration_test.go | 204 ++++++++++++++++++ db/channel_index_migration_test.go | 74 +++++++ db/channels.go | 33 +-- db/channels_test.go | 8 +- db/migrations/028_channels_last_seen_id.sql | 8 + db/migrations/029_drop_channels_last_seen.sql | 6 + db/queries/queries.sql | 14 +- db/sqlc/mock/querier.go | 15 ++ db/sqlc/querier.go | 3 + db/sqlc/queries.sql.go | 61 +++++- docs/docs.go | 47 ++-- docs/swagger.json | 47 ++-- docs/swagger.yaml | 35 +-- internal/api/channels.go | 50 +++++ internal/api/channels_test.go | 34 +++ internal/api/handlers/channels.go | 19 +- internal/api/handlers/channels_test.go | 37 +++- internal/api/handlers/limits_test.go | 4 +- internal/api/handlers/stub_reader_test.go | 8 +- internal/api/reader.go | 4 +- internal/cache/cache_test.go | 4 +- internal/cache/reader.go | 4 +- 22 files changed, 632 insertions(+), 87 deletions(-) create mode 100644 db/channel_cursor_integration_test.go create mode 100644 db/channel_index_migration_test.go create mode 100644 db/migrations/028_channels_last_seen_id.sql create mode 100644 db/migrations/029_drop_channels_last_seen.sql create mode 100644 internal/api/channels_test.go diff --git a/db/channel_cursor_integration_test.go b/db/channel_cursor_integration_test.go new file mode 100644 index 00000000..8f55844a --- /dev/null +++ b/db/channel_cursor_integration_test.go @@ -0,0 +1,204 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "reflect" + "strconv" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/jackc/pgx/v5" +) + +type channelCursorQueries struct { + sqlc.DBTX + query string + args []any +} + +func (q *channelCursorQueries) Query(ctx context.Context, query string, args ...any) (pgx.Rows, error) { + q.query, q.args = query, args + return q.DBTX.Query(ctx, query, args...) +} + +func TestChannelCursorPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + exec := func(sql string) { + t.Helper() + if _, err := tx.Exec(ctx, sql); err != nil { + t.Fatal(err) + } + } + exec(`CREATE TEMP TABLE channels (LIKE public.channels INCLUDING ALL) ON COMMIT DROP; +CREATE TEMP TABLE channel_iatas (LIKE public.channel_iatas INCLUDING ALL) ON COMMIT DROP; +CREATE INDEX ON channels(last_seen DESC,id DESC); +INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) VALUES + (1,'\xaa','\x01','2026-09-08 12:00:00+00'),(2,'\xaa','\x02','2026-09-08 12:00:00+00'), + (3,'\xaa','\x03','2026-09-08 12:00:00+00'),(4,'\xbb','\x04','2026-09-08 11:59:59+00'), + (5,'\xcc','\x05','2026-09-08 12:00:00.000321+00'), + (6,'\xcc','\x06','2026-09-08 12:00:00.000111+00'),(7,'\xcc','\x07','2026-09-08 12:00:00.000111+00'); +INSERT INTO channel_iatas(channel_hash,iata,last_heard) VALUES + ('\xaa','YOW',now()),('\xbb','YOW',now()),('\xcc','YYZ',now());`) + queries := &channelCursorQueries{DBTX: tx} + router := handlers.ChannelsRouter(&Store{q: sqlc.New(queries)}) + // Decode the wire contract so the same regression runs against the old server. + type wirePage struct { + Items []api.ChannelSummary `json:"items"` + NextCursor *int64 `json:"nextCursor"` + NextPageCursor *string `json:"nextPageCursor"` + HasMore bool `json:"hasMore"` + } + request := func(t *testing.T, params url.Values) wirePage { + t.Helper() + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/?"+params.Encode(), nil).WithContext(ctx)) + if w.Code != http.StatusOK { + t.Fatalf("channel page HTTP %d", w.Code) + } + var page wirePage + if err := json.Unmarshal(w.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + return page + } + for _, tc := range []struct { + name, query string + want []int + }{ + {"all ties and precision", "", []int{5, 7, 6, 3, 2, 1, 4}}, + {"hash", "hash=AA", []int{3, 2, 1}}, + {"region", "iata=yow", []int{3, 2, 1, 4}}, + {"multi region", "iatas=yow,yyz", []int{5, 7, 6, 3, 2, 1, 4}}, + {"hash and region", "hash=cc&iata=yyz", []int{5, 7, 6}}, + {"no regional match", "hash=aa&iata=yyz", nil}, + {"missing region", "iata=ZZZ", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + params, _ := url.ParseQuery(tc.query) + params.Set("limit", "2") + params.Set("cursor", "0") // Clients may retain the legacy first-page value. + var got []int + for n := 0; n < 10; n++ { + page := request(t, params) + for _, item := range page.Items { + got = append(got, item.ID) + } + if !page.HasMore { + if page.NextCursor != nil || page.NextPageCursor != nil { + t.Error("final page retains a cursor") + } + break + } + if page.NextCursor == nil { + t.Fatal("legacy numeric cursor was removed") + } + if page.NextPageCursor != nil { + params.Set("pageCursor", *page.NextPageCursor) + } else { + params.Set("cursor", strconv.FormatInt(*page.NextCursor, 10)) + } + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("paged IDs %v; want %v", got, tc.want) + } + }) + } + // The integer cursor still has its original strict millisecond semantics. + legacy := request(t, url.Values{"limit": {"50"}, "cursor": {"1788868800000"}}) + if len(legacy.Items) != 1 || legacy.Items[0].ID != 4 { + t.Error("legacy cursor semantics changed") + } + t.Run("newer activity stays ahead of the boundary", func(t *testing.T) { + first := request(t, url.Values{"limit": {"2"}}) + if first.NextPageCursor == nil { + t.Fatal("precise cursor missing") + } + exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) +VALUES (8,'\xee','\x08','2026-09-08 12:00:02+00')`) + rest := request(t, url.Values{"limit": {"50"}, "pageCursor": {*first.NextPageCursor}}) + var ids []int + for _, item := range rest.Items { + ids = append(ids, item.ID) + } + if !reflect.DeepEqual(ids, []int{6, 3, 2, 1, 4}) { + t.Errorf("new activity changed older traversal: %v", ids) + } + fresh := request(t, url.Values{"limit": {"2"}}) + if fresh.Items[0].ID != 8 { + t.Error("fresh first page missed new activity") + } + exec("DELETE FROM channels WHERE id=8") + }) + + exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) +SELECT i,'\xdd',decode(lpad(to_hex(i),16,'0'),'hex'), + '2026-09-09 12:00:00+00'::timestamptz-i*interval '1 millisecond' +FROM generate_series(1001,21000) i; +ANALYZE channels;`) + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + t.Run(mode, func(t *testing.T) { + exec("SET LOCAL plan_cache_mode = '" + mode + "'") + at := time.Date(2026, 9, 9, 12, 0, 0, 0, time.UTC).Add(-20 * time.Second) + page := request(t, url.Values{"limit": {"50"}, "pageCursor": {fmt.Sprintf("v1:%d:20000", at.UnixMicro())}}) + if len(page.Items) != 50 || page.Items[0].ID != 20001 { + t.Fatal("deep cursor returned the wrong page") + } + var raw []byte + if err := tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+queries.query, queries.args...).Scan(&raw); err != nil { + t.Fatal(err) + } + var plans []map[string]any + if err := json.Unmarshal(raw, &plans); err != nil { + t.Fatal(err) + } + rows := 0.0 + indexed := false + var visit func(map[string]any) + visit = func(plan map[string]any) { + if plan["Relation Name"] == "channels" { + n, _ := plan["Actual Rows"].(float64) + removed, _ := plan["Rows Removed by Filter"].(float64) + loops, _ := plan["Actual Loops"].(float64) + rows += (n + removed) * loops + _, indexed = plan["Index Cond"] + } + children, _ := plan["Plans"].([]any) + for _, child := range children { + visit(child.(map[string]any)) + } + } + visit(plans[0]["Plan"].(map[string]any)) + if rows > 60 || !indexed { + t.Errorf("deep page examines %.0f rows, indexed=%v", rows, indexed) + } + t.Logf("%s: examined %.0f rows; execution %v ms", mode, rows, plans[0]["Execution Time"]) + }) + } +} diff --git a/db/channel_index_migration_test.go b/db/channel_index_migration_test.go new file mode 100644 index 00000000..4e0fb02b --- /dev/null +++ b/db/channel_index_migration_test.go @@ -0,0 +1,74 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +func TestChannelIndexMigrationsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + exec := func(query string) { + t.Helper() + if _, err := conn.Exec(ctx, query); err != nil { + t.Fatal(err) + } + } + schema := pgx.Identifier{fmt.Sprintf("channel_index_test_%d", time.Now().UnixNano())}.Sanitize() + exec("CREATE SCHEMA " + schema) + defer func() { + cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := conn.Exec(cleanup, "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Error(err) + } + }() + exec("SET search_path TO " + schema) + exec(`CREATE TABLE channels (id bigint PRIMARY KEY, last_seen timestamptz NOT NULL); +CREATE INDEX idx_channels_last_seen ON channels(last_seen DESC); +INSERT INTO channels VALUES (1, '2026-09-09 12:00:00+00');`) + migrate := func(name string) { + t.Helper() + query, err := migrationFiles.ReadFile("migrations/" + name) + if err != nil { + t.Fatal(err) + } + exec(string(query)) // Concurrent DDL must remain outside an explicit transaction. + } + checkReplacement := func() { + t.Helper() + var valid bool + if err := conn.QueryRow(ctx, `SELECT indisvalid AND indisready AND indnkeyatts=2 +FROM pg_index WHERE indexrelid='idx_channels_last_seen_id'::regclass`).Scan(&valid); err != nil || !valid { + t.Fatalf("replacement index valid=%v, err=%v", valid, err) + } + } + migrate("028_channels_last_seen_id.sql") + checkReplacement() + for range 2 { // Retry after the drop succeeds but before its migration record is saved. + migrate("029_drop_channels_last_seen.sql") + var preserved bool + if err := conn.QueryRow(ctx, `SELECT to_regclass('idx_channels_last_seen') IS NULL +AND (SELECT count(*) FROM channels)=1`).Scan(&preserved); err != nil || !preserved { + t.Fatalf("old index removed and row preserved=%v, err=%v", preserved, err) + } + checkReplacement() + } +} diff --git a/db/channels.go b/db/channels.go index dbdbddc9..599230a3 100644 --- a/db/channels.go +++ b/db/channels.go @@ -77,19 +77,25 @@ func (s *Store) DeleteOldChannelIATAs(ctx context.Context, cutoff time.Time) err return s.q.DeleteOldChannelIATAs(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } -func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) { +func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { var cursorTS pgtype.Timestamptz if cursor > 0 { cursorTS = pgtype.Timestamptz{Time: time.UnixMilli(cursor), Valid: true} } - rows, err := s.q.ListChannels(ctx, sqlc.ListChannelsParams{ - ChannelHash: hash, - Iatas: iatas, - CursorTs: cursorTS, - PageLimit: limit + 1, - }) + var rows []sqlc.Channel + var err error + if pageCursor != nil { + rows, err = s.q.ListChannelsAfter(ctx, sqlc.ListChannelsAfterParams{ + ChannelHash: hash, Iatas: iatas, PageLimit: limit + 1, + CursorTs: pgtype.Timestamptz{Time: pageCursor.LastSeen, Valid: true}, CursorID: pageCursor.ID, + }) + } else { + rows, err = s.q.ListChannels(ctx, sqlc.ListChannelsParams{ + ChannelHash: hash, Iatas: iatas, CursorTs: cursorTS, PageLimit: limit + 1, + }) + } if err != nil { - return api.Page[api.ChannelSummary]{}, err + return api.ChannelPage{}, err } hasMore := len(rows) > int(limit) if hasMore { @@ -107,14 +113,17 @@ func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iata }) } var nextCursor *int64 + var nextPageCursor *string if hasMore { last := items[len(items)-1].LastSeen nextCursor = &last + row := rows[len(rows)-1] + precise := (api.ChannelCursor{LastSeen: row.LastSeen.Time, ID: row.ID}).String() + nextPageCursor = &precise } - return api.Page[api.ChannelSummary]{ - Items: items, - NextCursor: nextCursor, - HasMore: hasMore, + return api.ChannelPage{ + Page: api.Page[api.ChannelSummary]{Items: items, NextCursor: nextCursor, HasMore: hasMore}, + NextPageCursor: nextPageCursor, }, nil } diff --git a/db/channels_test.go b/db/channels_test.go index 8fdea060..0f51c631 100644 --- a/db/channels_test.go +++ b/db/channels_test.go @@ -29,7 +29,7 @@ func TestListChannels_Empty(t *testing.T) { Return([]sqlc.Channel{}, nil) store := &Store{q: mock} - page, err := store.ListChannels(context.Background(), 10, nil, nil, 0) + page, err := store.ListChannels(context.Background(), 10, nil, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -71,7 +71,7 @@ func TestListChannels_Pagination(t *testing.T) { Return(rows, nil) store := &Store{q: mock} - page, err := store.ListChannels(context.Background(), 2, nil, nil, 0) + page, err := store.ListChannels(context.Background(), 2, nil, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -95,7 +95,7 @@ func TestListChannels_DBError(t *testing.T) { Return(nil, errors.New("db error")) store := &Store{q: mock} - _, err := store.ListChannels(context.Background(), 10, nil, nil, 0) + _, err := store.ListChannels(context.Background(), 10, nil, nil, 0, nil) if err == nil { t.Fatal("expected error, got nil") } @@ -115,7 +115,7 @@ func TestListChannels_IATAFilter(t *testing.T) { Return([]sqlc.Channel{}, nil) store := &Store{q: mock} - _, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, 0) + _, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/db/migrations/028_channels_last_seen_id.sql b/db/migrations/028_channels_last_seen_id.sql new file mode 100644 index 00000000..6c295090 --- /dev/null +++ b/db/migrations/028_channels_last_seen_id.sql @@ -0,0 +1,8 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- RunMigrations executes this single statement outside an explicit transaction. +-- If interrupted, check pg_index.indisvalid and remove this named index before +-- retrying; do not silently accept an invalid index with IF NOT EXISTS. +CREATE INDEX CONCURRENTLY idx_channels_last_seen_id +ON channels (last_seen DESC, id DESC); diff --git a/db/migrations/029_drop_channels_last_seen.sql b/db/migrations/029_drop_channels_last_seen.sql new file mode 100644 index 00000000..fadcd428 --- /dev/null +++ b/db/migrations/029_drop_channels_last_seen.sql @@ -0,0 +1,6 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Migration 028 replaces this index with (last_seen DESC, id DESC). +-- Keep this separate for concurrent execution; IF EXISTS allows a completed drop to retry. +DROP INDEX CONCURRENTLY IF EXISTS idx_channels_last_seen; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 5de50eba..da3fa9bd 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -816,7 +816,19 @@ WHERE (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) WHERE ci.iata = ANY(@iatas::bpchar[]) )) AND (@cursor_ts::timestamptz IS NULL OR c.last_seen < @cursor_ts) -ORDER BY c.last_seen DESC +ORDER BY c.last_seen DESC, c.id DESC +LIMIT @page_limit; + +-- name: ListChannelsAfter :many +-- Keep the non-null tuple boundary separate from the legacy optional cursor so +-- generic prepared plans can seek directly into the composite ordered index. +SELECT c.* FROM channels c +WHERE (c.last_seen, c.id) < (@cursor_ts::timestamptz, @cursor_id::integer) + AND (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR c.channel_hash IN ( + SELECT ci.channel_hash FROM channel_iatas ci WHERE ci.iata = ANY(@iatas::bpchar[]) + )) +ORDER BY c.last_seen DESC, c.id DESC LIMIT @page_limit; -- name: GetChannelByID :one diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 5742d194..a1a5e7a3 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -831,6 +831,21 @@ func (mr *MockQuerierMockRecorder) ListChannels(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannels", reflect.TypeOf((*MockQuerier)(nil).ListChannels), ctx, arg) } +// ListChannelsAfter mocks base method. +func (m *MockQuerier) ListChannelsAfter(ctx context.Context, arg db.ListChannelsAfterParams) ([]db.Channel, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListChannelsAfter", ctx, arg) + ret0, _ := ret[0].([]db.Channel) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListChannelsAfter indicates an expected call of ListChannelsAfter. +func (mr *MockQuerierMockRecorder) ListChannelsAfter(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannelsAfter", reflect.TypeOf((*MockQuerier)(nil).ListChannelsAfter), ctx, arg) +} + // ListIATAs mocks base method. func (m *MockQuerier) ListIATAs(ctx context.Context) ([]db.IataCode, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index dc30ab23..4d1bdd19 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -126,6 +126,9 @@ type Querier interface { // (membership via channel_iatas). NULL hash / empty array skip those filters. // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) + // Keep the non-null tuple boundary separate from the legacy optional cursor so + // generic prepared plans can seek directly into the composite ordered index. + ListChannelsAfter(ctx context.Context, arg ListChannelsAfterParams) ([]Channel, error) ListIATAs(ctx context.Context) ([]IataCode, error) // Only one branch runs. Keep the IATA range ordered by the composite index: // generic plans can otherwise prefer scanning the global timestamp index. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 3a7ff942..0dce6ba9 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2085,7 +2085,7 @@ WHERE ($1::bytea IS NULL OR c.channel_hash = $1) WHERE ci.iata = ANY($2::bpchar[]) )) AND ($3::timestamptz IS NULL OR c.last_seen < $3) -ORDER BY c.last_seen DESC +ORDER BY c.last_seen DESC, c.id DESC LIMIT $4 ` @@ -2136,6 +2136,65 @@ func (q *Queries) ListChannels(ctx context.Context, arg ListChannelsParams) ([]C return items, nil } +const listChannelsAfter = `-- name: ListChannelsAfter :many +SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c +WHERE (c.last_seen, c.id) < ($1::timestamptz, $2::integer) + AND ($3::bytea IS NULL OR c.channel_hash = $3) + AND (COALESCE(cardinality($4::bpchar[]), 0) = 0 OR c.channel_hash IN ( + SELECT ci.channel_hash FROM channel_iatas ci WHERE ci.iata = ANY($4::bpchar[]) + )) +ORDER BY c.last_seen DESC, c.id DESC +LIMIT $5 +` + +type ListChannelsAfterParams struct { + CursorTs pgtype.Timestamptz `json:"cursor_ts"` + CursorID int32 `json:"cursor_id"` + ChannelHash []byte `json:"channel_hash"` + Iatas []string `json:"iatas"` + PageLimit int32 `json:"page_limit"` +} + +// Keep the non-null tuple boundary separate from the legacy optional cursor so +// generic prepared plans can seek directly into the composite ordered index. +func (q *Queries) ListChannelsAfter(ctx context.Context, arg ListChannelsAfterParams) ([]Channel, error) { + rows, err := q.db.Query(ctx, listChannelsAfter, + arg.CursorTs, + arg.CursorID, + arg.ChannelHash, + arg.Iatas, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Channel{} + for rows.Next() { + var i Channel + if err := rows.Scan( + &i.ID, + &i.ChannelHash, + &i.KeyFingerprint, + &i.Name, + &i.Hashtag, + &i.IsHashtag, + &i.IsPublic, + &i.KeyKnown, + &i.FirstSeen, + &i.LastSeen, + &i.MessageCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listIATAs = `-- name: ListIATAs :many SELECT iata, display_name, approx_lat, approx_lng, added_at, border FROM iata_codes ORDER BY iata ` diff --git a/docs/docs.go b/docs/docs.go index b9f685a5..3a97dd94 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -74,10 +74,16 @@ const docTemplate = `{ }, { "type": "integer", - "description": "last_seen epoch ms of last item for pagination", + "description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning", "name": "cursor", "in": "query" }, + { + "type": "string", + "description": "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed.", + "name": "pageCursor", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -91,7 +97,7 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage" } }, "400": { @@ -2439,6 +2445,26 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage": { + "type": "object", + "properties": { + "hasMore": { + "type": "boolean" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary" + } + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary": { "type": "object", "properties": { @@ -3414,23 +3440,6 @@ const docTemplate = `{ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary": { - "type": "object", - "properties": { - "hasMore": { - "type": "boolean" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary" - } - }, - "nextCursor": { - "type": "integer" - } - } - }, "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 7db537cc..c9f19cf7 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -72,10 +72,16 @@ }, { "type": "integer", - "description": "last_seen epoch ms of last item for pagination", + "description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning", "name": "cursor", "in": "query" }, + { + "type": "string", + "description": "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed.", + "name": "pageCursor", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -89,7 +95,7 @@ "200": { "description": "OK", "schema": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary" + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage" } }, "400": { @@ -2437,6 +2443,26 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage": { + "type": "object", + "properties": { + "hasMore": { + "type": "boolean" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary" + } + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary": { "type": "object", "properties": { @@ -3412,23 +3438,6 @@ } } }, - "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary": { - "type": "object", - "properties": { - "hasMore": { - "type": "boolean" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary" - } - }, - "nextCursor": { - "type": "integer" - } - } - }, "github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 5bb289bc..147830bc 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -81,6 +81,19 @@ definitions: description: epoch ms, from the sender's embedded timestamp type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage: + properties: + hasMore: + type: boolean + items: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary' + type: array + nextCursor: + type: integer + nextPageCursor: + type: string + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary: properties: channelHash: @@ -775,17 +788,6 @@ definitions: nextCursor: type: integer type: object - github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary: - properties: - hasMore: - type: boolean - items: - items: - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelSummary' - type: array - nextCursor: - type: integer - type: object github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_NodeSummary: properties: hasMore: @@ -1171,10 +1173,17 @@ paths: in: query name: iatas type: string - - description: last_seen epoch ms of last item for pagination + - description: last_seen epoch ms of last item for pagination; 0 starts from + the beginning in: query name: cursor type: integer + - description: Opaque nextPageCursor from a previous response; preserves timestamp + ties and precision. Cannot be combined with a positive cursor; cursor=0 + is allowed. + in: query + name: pageCursor + type: string - description: Max results (default 50); must be positive, values above 200 are clamped in: query @@ -1188,7 +1197,7 @@ paths: "200": description: OK schema: - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Page-github_com_MeshCore-Beacon_beacon-server_internal_api_ChannelSummary' + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ChannelPage' "400": description: Bad Request schema: diff --git a/internal/api/channels.go b/internal/api/channels.go index ec91009e..c609a23a 100644 --- a/internal/api/channels.go +++ b/internal/api/channels.go @@ -3,6 +3,56 @@ package api +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +// ChannelPage adds a precise cursor while retaining the numeric cursor for older clients. +type ChannelPage struct { + Page[ChannelSummary] + NextPageCursor *string `json:"nextPageCursor,omitempty"` +} + +// ChannelCursor identifies a boundary in (last_seen DESC, id DESC) order. +type ChannelCursor struct { + LastSeen time.Time + ID int32 +} + +func (c ChannelCursor) String() string { + return fmt.Sprintf("v1:%d:%d", c.LastSeen.UnixMicro(), c.ID) +} + +var errChannelCursor = errors.New("invalid channel page cursor") + +// ParseChannelCursor parses the opaque, versioned cursor returned by ChannelPage. +func ParseChannelCursor(raw string) (*ChannelCursor, error) { + if len(raw) > 64 { + return nil, errChannelCursor + } + parts := strings.Split(raw, ":") + if len(parts) != 3 || parts[0] != "v1" { + return nil, errChannelCursor + } + micros, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, errChannelCursor + } + id, err := strconv.ParseInt(parts[2], 10, 32) + if err != nil || id <= 0 { + return nil, errChannelCursor + } + at := time.UnixMicro(micros).UTC() + if at.Year() < 1 || at.Year() > 9999 { + return nil, errChannelCursor + } + return &ChannelCursor{LastSeen: at, ID: int32(id)}, nil +} + // ChannelMessage represents a single decrypted channel message. // Only messages for channels with a known key are stored and returned. type ChannelMessage struct { diff --git a/internal/api/channels_test.go b/internal/api/channels_test.go new file mode 100644 index 00000000..bf53c499 --- /dev/null +++ b/internal/api/channels_test.go @@ -0,0 +1,34 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "strings" + "testing" + "time" +) + +func TestChannelCursorRoundTrip(t *testing.T) { + for _, at := range []time.Time{ + time.Date(2026, 9, 8, 12, 0, 0, 123456000, time.FixedZone("offset", 3600)), + time.UnixMicro(0), time.UnixMicro(-1), + time.Date(9999, 12, 31, 23, 59, 59, 999999000, time.UTC), + } { + want := ChannelCursor{LastSeen: at, ID: 2147483647} + got, err := ParseChannelCursor(want.String()) + if err != nil || got.ID != want.ID || !got.LastSeen.Equal(want.LastSeen) { + t.Fatalf("round trip lost timestamp precision or ID: %v", err) + } + } +} + +func TestChannelCursorInvalid(t *testing.T) { + for _, raw := range []string{"", "1", "v2:1:1", "v1:1", "v1:1:1:1", "v1:x:1", "v1:1.5:1", + "v1:1:0", "v1:1:-1", "v1:1:2147483648", "v1:9223372036854775808:1", + "v1:-9223372036854775808:1", "v1:9223372036854775807:1", strings.Repeat("x", 65)} { + if _, err := ParseChannelCursor(raw); err == nil { + t.Errorf("accepted invalid cursor %q", raw) + } + } +} diff --git a/internal/api/handlers/channels.go b/internal/api/handlers/channels.go index d0f6f7f4..81ee792f 100644 --- a/internal/api/handlers/channels.go +++ b/internal/api/handlers/channels.go @@ -37,9 +37,10 @@ func ChannelsRouter(reader api.Reader) http.Handler { // @Param hash query string false "Single-byte channel hash (hex)" // @Param iata query string false "Filter by IATA code" // @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ" -// @Param cursor query int false "last_seen epoch ms of last item for pagination" +// @Param cursor query int false "last_seen epoch ms of last item for pagination; 0 starts from the beginning" +// @Param pageCursor query string false "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed." // @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) -// @Success 200 {object} api.Page[api.ChannelSummary] +// @Success 200 {object} api.ChannelPage // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /channels [get] @@ -60,6 +61,18 @@ func listChannels(reader api.Reader) http.HandlerFunc { } cursor = c } + var pageCursor *api.ChannelCursor + if raw := r.URL.Query().Get("pageCursor"); raw != "" { + if cursor > 0 { + respondError(w, http.StatusBadRequest, "pageCursor and a positive cursor cannot be combined") + return + } + pageCursor, err = api.ParseChannelCursor(raw) + if err != nil { + respondError(w, http.StatusBadRequest, "invalid pageCursor") + return + } + } var hashHex []byte if hash := strings.ToLower(r.URL.Query().Get("hash")); hash != "" { h, decodeErr := hex.DecodeString(hash) @@ -73,7 +86,7 @@ func listChannels(reader api.Reader) http.HandlerFunc { } hashHex = h } - channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, cursor) + channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, cursor, pageCursor) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/channels_test.go b/internal/api/handlers/channels_test.go index c014dc4e..8b964d7f 100644 --- a/internal/api/handlers/channels_test.go +++ b/internal/api/handlers/channels_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "reflect" "testing" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" @@ -36,6 +37,34 @@ func TestListChannels_InvalidCursor(t *testing.T) { } } +func TestListChannels_PageCursor(t *testing.T) { + called := false + r := ChannelsRouter(stubReader{listChannels: func(_ context.Context, limit int32, hash []byte, iatas []string, legacy int64, cursor *api.ChannelCursor) (api.ChannelPage, error) { + called = true + if limit != 2 || legacy != 0 || cursor == nil || cursor.ID != 9 || !cursor.LastSeen.Equal(time.UnixMicro(1700000000000123)) || + !reflect.DeepEqual(hash, []byte{0xaa}) || !reflect.DeepEqual(iatas, []string{"YOW"}) { + t.Error("page cursor or filters did not reach the reader intact") + } + return api.ChannelPage{}, nil + }}) + for _, legacy := range []string{"", "&cursor=0"} { + called = false + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/?pageCursor=v1:1700000000000123:9&limit=2&hash=AA&iata=yow"+legacy, nil)) + if w.Code != http.StatusOK || !called { + t.Fatalf("legacy %q: HTTP %d, reader called=%v", legacy, w.Code, called) + } + } + for _, query := range []string{"pageCursor=bad", "cursor=1&pageCursor=v1:1700000000000123:9", "cursor=bad&pageCursor=v1:1700000000000123:9"} { + called = false + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/?"+query, nil)) + if w.Code != http.StatusBadRequest || called { + t.Errorf("HTTP %d, invalid request reached reader=%v", w.Code, called) + } + } +} + func TestListChannels_InvalidHash(t *testing.T) { r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{})) @@ -116,8 +145,8 @@ func TestListChannelMessages_InvalidCursor(t *testing.T) { func TestListChannels_OK(t *testing.T) { r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{ - listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ int64) (api.Page[api.ChannelSummary], error) { - return api.Page[api.ChannelSummary]{Items: []api.ChannelSummary{{ID: 1, ChannelHash: "ab"}}}, nil + listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + return api.ChannelPage{Page: api.Page[api.ChannelSummary]{Items: []api.ChannelSummary{{ID: 1, ChannelHash: "ab"}}}}, nil }, })) req := httptest.NewRequest(http.MethodGet, "/channels", nil) @@ -143,9 +172,9 @@ func TestListChannels_IATAParsing(t *testing.T) { var got []string r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{ - listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ int64) (api.Page[api.ChannelSummary], error) { + listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { got = iatas - return api.Page[api.ChannelSummary]{}, nil + return api.ChannelPage{}, nil }, })) req := httptest.NewRequest(http.MethodGet, "/channels"+tc.query, nil) diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go index ff63a95e..991a2076 100644 --- a/internal/api/handlers/limits_test.go +++ b/internal/api/handlers/limits_test.go @@ -19,10 +19,10 @@ func TestListLimits(t *testing.T) { var got int32 var calls int reader := stubReader{ - listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) { + listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { calls++ got = limit - return api.Page[api.ChannelSummary]{}, nil + return api.ChannelPage{}, nil }, listChannelMessages: func(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) { calls++ diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index f53cb0a0..2b97ed23 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -22,7 +22,7 @@ type stubReader struct { listRegions func(ctx context.Context) ([]api.RegionSummary, error) getRegion func(ctx context.Context, regionID int32) (*api.Region, error) getRegionBySlug func(ctx context.Context, slug string) (*api.Region, error) - listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) + listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) getChannel func(ctx context.Context, channelID int32) (*api.Channel, error) listChannelMessages func(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) listChannelMessagesByHash func(ctx context.Context, hash []byte, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) @@ -106,11 +106,11 @@ func (s stubReader) GetRegionBySlug(ctx context.Context, slug string) (*api.Regi return nil, nil } -func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) { +func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { if s.listChannels != nil { - return s.listChannels(ctx, limit, hash, iatas, cursor) + return s.listChannels(ctx, limit, hash, iatas, cursor, pageCursor) } - return api.Page[api.ChannelSummary]{}, nil + return api.ChannelPage{}, nil } func (s stubReader) GetChannel(ctx context.Context, channelID int32) (*api.Channel, error) { diff --git a/internal/api/reader.go b/internal/api/reader.go index 964b04b2..b8b353a9 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -53,7 +53,9 @@ type Reader interface { // Pass nil hash to skip hash filtering. Pass empty iatas to return all channels; // IATAs must be uppercase. // cursor is last_seen epoch ms of the last item; pass 0 to start from the beginning. - ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (Page[ChannelSummary], error) + // pageCursor is the precise timestamp/ID boundary; nil preserves the legacy numeric cursor. + // When non-nil, pageCursor takes precedence over cursor. + ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *ChannelCursor) (ChannelPage, error) // GetChannel returns full detail for a single channel by its integer ID. // Returns nil, pgx.ErrNoRows if the channel is not found. diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 4531808e..eb1ad142 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -137,8 +137,8 @@ func (s *stubReader) GetCrossIATANeighbors(_ context.Context, _ uuid.UUID, _ str return nil, nil } -func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ int64) (api.Page[api.ChannelSummary], error) { - return api.Page[api.ChannelSummary]{}, nil +func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + return api.ChannelPage{}, nil } func (s *stubReader) ListChannelMessages(_ context.Context, _ *int32, _ time.Time, _ int32, _ []string, _ string, _ int64) (api.Page[api.ChannelMessage], error) { diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 6b234ccd..04dc5b75 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -378,8 +378,8 @@ func (cr *CachedReader) GetCrossIATANeighbors(ctx context.Context, nodeID uuid.U } // ListChannels implements [api.Reader]. -func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64) (api.Page[api.ChannelSummary], error) { - return cr.inner.ListChannels(ctx, limit, hash, iatas, cursor) +func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { + return cr.inner.ListChannels(ctx, limit, hash, iatas, cursor, pageCursor) } // ListChannelMessages implements [api.Reader]. From 22410cf4a6ded0d72f52a66e758fede8b34ee086 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 9 Sep 2026 20:42:35 -0400 Subject: [PATCH 024/141] fix(api): trust client IP headers only from configured proxies (#133) --- cmd/beacon/main.go | 5 +- config.yaml.example | 19 ++++++ internal/api/middleware/trusted_proxy.go | 39 +++++++++++ internal/api/middleware/trusted_proxy_test.go | 61 ++++++++++++++++++ internal/api/router/router.go | 4 +- internal/api/router/router_test.go | 64 +++++++++++++++++++ internal/config/config.go | 32 ++++++++++ internal/config/config_test.go | 31 +++++++++ 8 files changed, 252 insertions(+), 3 deletions(-) create mode 100644 internal/api/middleware/trusted_proxy.go create mode 100644 internal/api/middleware/trusted_proxy_test.go create mode 100644 internal/api/router/router_test.go diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index ef46a383..bb31b2ab 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -88,6 +88,9 @@ func main() { if err != nil { log.Fatalf("failed to load config: %v", err) } + if len(cfg.Server.TrustedProxies) == 0 { + log.Print("warning: server.trusted_proxies is empty; client IP headers are ignored and proxied clients share a WebSocket connection limit") + } resolved := config.Resolve(cfg) @@ -272,7 +275,7 @@ func main() { go scheduler.Start(ctx) // ── HTTP server ────────────────────────────────────────────────────────── - r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, cfg.CORS) + r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, cfg.CORS, cfg.Server) srv := &http.Server{ Addr: addr, diff --git a/config.yaml.example b/config.yaml.example index cc91a39f..439c51ac 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,6 +1,25 @@ # Beacon configuration file # Copy to config.yaml and adjust as needed. +server: + # Only these direct proxy peers may set the client IP through X-Real-IP. + # The proxy must overwrite that header, not pass through client input. + # Leave empty for direct access (logs a startup warning). Invalid/empty CIDR + # entries prevent startup. A single host needs /32 for IPv4 or /128 for IPv6; + # bare IP addresses are not accepted. + # For Docker, list the actual proxy/compose network CIDR (e.g. 172.20.0.0/24). + # For a local host proxy, use ["127.0.0.1/32", "::1/128"]. + # Direct-client proxy examples; X-Forwarded-For alone is not used: + # Apache 2.4.10+ (mod_headers, in the proxy virtual host): + # RequestHeader set X-Real-IP "expr=%{REMOTE_ADDR}" + # nginx (in the proxy location): + # proxy_set_header X-Real-IP $remote_addr; + # Caddy (inside the reverse_proxy block): + # header_up X-Real-IP {http.request.remote.host} + # With additional proxy hops, first configure their trust at the edge proxy + # and pass its validated client address rather than a client-supplied header. + trusted_proxies: [] + iatas: YVR: name: Vancouver International diff --git a/internal/api/middleware/trusted_proxy.go b/internal/api/middleware/trusted_proxy.go new file mode 100644 index 00000000..606fbc47 --- /dev/null +++ b/internal/api/middleware/trusted_proxy.go @@ -0,0 +1,39 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "net/http" + "net/netip" +) + +// TrustedProxyIP accepts X-Real-IP only from a configured direct proxy peer. +// Run it before the access logger and handlers that use RemoteAddr as a key. +// All other forwarding headers are ignored, as are invalid/ambiguous values. +func TrustedProxyIP(proxies []netip.Prefix) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peer, err := netip.ParseAddrPort(r.RemoteAddr) + if err == nil { + for _, proxy := range proxies { + if !proxy.Contains(peer.Addr().Unmap()) { + continue + } + if values := r.Header.Values("X-Real-IP"); len(values) == 1 { + if ip, err := netip.ParseAddr(values[0]); err == nil && ip.Zone() == "" { + r.RemoteAddr = ip.Unmap().String() + } + } + break + } + } + // Downstream client-IP keys must use the resolved RemoteAddr, not + // reinterpret forwarding headers (including those from trusted peers). + r.Header.Del("True-Client-IP") + r.Header.Del("X-Forwarded-For") + r.Header.Del("X-Real-IP") + next.ServeHTTP(w, r) + }) + } +} diff --git a/internal/api/middleware/trusted_proxy_test.go b/internal/api/middleware/trusted_proxy_test.go new file mode 100644 index 00000000..264fe93a --- /dev/null +++ b/internal/api/middleware/trusted_proxy_test.go @@ -0,0 +1,61 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "net/http" + "net/http/httptest" + "net/netip" + "testing" +) + +func TestTrustedProxyIP(t *testing.T) { + trusted := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24"), netip.MustParsePrefix("2001:db8:1::/48")} + for _, tc := range []struct { + name, peer string + proxies []netip.Prefix + realIP []string + want string + }{ + {"empty allowlist", "192.0.2.10:1234", nil, []string{"198.51.100.1"}, "192.0.2.10:1234"}, + {"untrusted peer", "192.0.3.10:1234", trusted, []string{"198.51.100.1"}, "192.0.3.10:1234"}, + {"trusted IPv4", "192.0.2.10:1234", trusted, []string{"198.51.100.1"}, "198.51.100.1"}, + {"trusted IPv6", "[2001:db8:1::10]:1234", trusted, []string{"2001:db8:2::1"}, "2001:db8:2::1"}, + {"mapped peer", "[::ffff:192.0.2.10]:1234", trusted, []string{"198.51.100.1"}, "198.51.100.1"}, + {"mapped client", "192.0.2.10:1234", trusted, []string{"::ffff:198.51.100.1"}, "198.51.100.1"}, + {"missing header", "192.0.2.10:1234", trusted, nil, "192.0.2.10:1234"}, + {"empty header", "192.0.2.10:1234", trusted, []string{""}, "192.0.2.10:1234"}, + {"invalid IP", "192.0.2.10:1234", trusted, []string{"not-an-ip"}, "192.0.2.10:1234"}, + {"IP with port", "192.0.2.10:1234", trusted, []string{"198.51.100.1:8080"}, "192.0.2.10:1234"}, + {"IP with zone", "192.0.2.10:1234", trusted, []string{"fe80::1%eth0"}, "192.0.2.10:1234"}, + {"comma list", "192.0.2.10:1234", trusted, []string{"198.51.100.1, 198.51.100.2"}, "192.0.2.10:1234"}, + {"repeated header", "192.0.2.10:1234", trusted, []string{"198.51.100.1", "198.51.100.2"}, "192.0.2.10:1234"}, + {"invalid peer", "unknown", trusted, []string{"198.51.100.1"}, "unknown"}, + } { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.RemoteAddr = tc.peer + for _, ip := range tc.realIP { + r.Header.Add("X-Real-IP", ip) + } + // These must never override the selected address, even for trusted peers. + r.Header.Set("True-Client-IP", "203.0.113.1") + r.Header.Set("X-Forwarded-For", "203.0.113.2") + r.Header.Set("X-Request-ID", "keep-me") + TrustedProxyIP(tc.proxies)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RemoteAddr != tc.want { + t.Errorf("RemoteAddr = %q, want %q", r.RemoteAddr, tc.want) + } + for _, header := range []string{"True-Client-IP", "X-Forwarded-For", "X-Real-IP"} { + if len(r.Header.Values(header)) != 0 { + t.Errorf("downstream middleware can still read %s", header) + } + } + if r.Header.Get("X-Request-ID") != "keep-me" { + t.Error("unrelated header was changed") + } + })).ServeHTTP(httptest.NewRecorder(), r) + }) + } +} diff --git a/internal/api/router/router.go b/internal/api/router/router.go index ce4ac666..5f4852a7 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -41,7 +41,7 @@ import ( // // The private group is stubbed and ready for the auth middleware drop-in // described in Future Features → Admin authentication. -func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP int, corsCfg config.CORSConfig) http.Handler { +func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP int, corsCfg config.CORSConfig, serverCfg config.ServerConfig) http.Handler { r := chi.NewRouter() // ── CORS ───────────────────────────────────────────────────────────────── @@ -71,7 +71,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP // ── Global middleware ──────────────────────────────────────────────────── r.Use(middleware.RequestID) - r.Use(middleware.RealIP) + r.Use(mw.TrustedProxyIP(serverCfg.TrustedProxies)) r.Use(middleware.Logger) r.Use(middleware.Recoverer) r.Use(middleware.CleanPath) diff --git a/internal/api/router/router_test.go b/internal/api/router/router_test.go new file mode 100644 index 00000000..9a42eb3e --- /dev/null +++ b/internal/api/router/router_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "context" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" +) + +func TestWebSocketLimitIgnoresUntrustedHeaders(t *testing.T) { + checkWebSocketLimit(t, config.ServerConfig{}, "198.51.100.2", http.StatusTooManyRequests) +} + +func TestWebSocketLimitUsesTrustedClientIP(t *testing.T) { + cfg := config.ServerConfig{TrustedProxies: []netip.Prefix{ + netip.MustParsePrefix("127.0.0.1/8"), netip.MustParsePrefix("::1/128"), + }} + t.Run("different clients", func(t *testing.T) { + checkWebSocketLimit(t, cfg, "198.51.100.2", http.StatusSwitchingProtocols) + }) + t.Run("same client", func(t *testing.T) { + checkWebSocketLimit(t, cfg, "198.51.100.1", http.StatusTooManyRequests) + }) +} + +func checkWebSocketLimit(t *testing.T, cfg config.ServerConfig, secondIP string, wantStatus int) { + t.Helper() + server := httptest.NewServer(New(hub.New(), nil, nil, 1, config.CORSConfig{}, cfg)) + defer server.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + url := "ws" + strings.TrimPrefix(server.URL, "http") + "/ws" + dial := func(ip string) (*websocket.Conn, *http.Response, error) { + return websocket.Dial(ctx, url, &websocket.DialOptions{HTTPHeader: http.Header{ + "X-Real-Ip": {ip}, "True-Client-Ip": {ip}, "X-Forwarded-For": {ip}, + }}) + } + first, _, err := dial("198.51.100.1") + if err != nil { + t.Fatal(err) + } + defer first.CloseNow() + if _, _, err := first.Read(ctx); err != nil { + t.Fatalf("read hello: %v", err) + } + second, response, err := dial(secondIP) + if second != nil { + second.CloseNow() + } + if response == nil || response.StatusCode != wantStatus || (err == nil) != (wantStatus == http.StatusSwitchingProtocols) { + t.Fatalf("second connection: response=%v err=%v, want status %d", response, err, wantStatus) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index feffb27b..4a9f4173 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,6 +7,7 @@ package config import ( "fmt" + "net/netip" "os" "path/filepath" "time" @@ -16,6 +17,7 @@ import ( // Config is the top-level structure of the Beacon config file. type Config struct { + Server ServerConfig `yaml:"server"` IATAs map[string]IATAConfig `yaml:"iatas"` Regions []RegionConfig `yaml:"regions"` ChannelKeys ChannelKeysConfig `yaml:"channel_keys"` @@ -33,6 +35,31 @@ type Config struct { Observers ObserversConfig `yaml:"observers"` } +// ServerConfig controls which direct peers may supply the client address. +type ServerConfig struct { + // TrustedProxies accepts IPv4/IPv6 CIDRs; an empty list trusts no proxy. + // netip.Prefix validates each CIDR while the configuration is loaded. + TrustedProxies []netip.Prefix `yaml:"trusted_proxies"` +} + +func (c *ServerConfig) UnmarshalYAML(node *yaml.Node) error { + // Pointers retain null list entries, which yaml would otherwise discard. + // Leave them as invalid prefixes for Load's validation below. + var raw struct { + TrustedProxies []*netip.Prefix `yaml:"trusted_proxies"` + } + if err := node.Decode(&raw); err != nil { + return err + } + c.TrustedProxies = make([]netip.Prefix, len(raw.TrustedProxies)) + for i, prefix := range raw.TrustedProxies { + if prefix != nil { + c.TrustedProxies[i] = *prefix + } + } + return nil +} + // ResolvedConfig holds all runtime configuration with defaults applied. type ResolvedConfig struct { TelemetryResolution time.Duration @@ -313,6 +340,11 @@ func Load(path string) (*Config, error) { if err := yaml.Unmarshal(data, cfg); err != nil { return nil, err } + for i, prefix := range cfg.Server.TrustedProxies { + if !prefix.IsValid() { + return nil, fmt.Errorf("server.trusted_proxies[%d] must be a valid CIDR", i) + } + } configDir := filepath.Dir(path) for iata, details := range cfg.IATAs { if details.BorderFile != "" && !filepath.IsAbs(details.BorderFile) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 75f673cf..e73e581a 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -5,11 +5,42 @@ package config import ( "os" + "path/filepath" "strings" "testing" "time" ) +func TestLoadTrustedProxies(t *testing.T) { + for _, tc := range []struct { + name, yaml string + count int + wantError bool + }{ + {"omitted", "{}", 0, false}, + {"empty", "server: {trusted_proxies: []}", 0, false}, + {"CIDRs", "server: {trusted_proxies: ['192.0.2.0/24', '2001:db8::/32']}", 2, false}, + {"bare IP", "server: {trusted_proxies: ['192.0.2.1']}", 0, true}, + {"invalid CIDR", "server: {trusted_proxies: ['192.0.2.0/99']}", 0, true}, + {"empty entry", "server: {trusted_proxies: ['', '192.0.2.0/24']}", 0, true}, + {"null entry", "server: {trusted_proxies: [null, '192.0.2.0/24']}", 0, true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if (err != nil) != tc.wantError { + t.Fatalf("Load error = %v, want error = %v", err, tc.wantError) + } + if err == nil && len(cfg.Server.TrustedProxies) != tc.count { + t.Fatalf("got %d proxy prefixes, want %d", len(cfg.Server.TrustedProxies), tc.count) + } + }) + } +} + func TestLoad_FileNotFound(t *testing.T) { cfg, err := Load("/nonexistent/path/config.yaml") if err != nil { From 6521fddbb6a32d7637ae0932513ed782eeb0cb55 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:42:45 -0400 Subject: [PATCH 025/141] recover from interrupted CREATE INDEX CONCURRENTLY in migrations (#134) --- db/migrate.go | 66 ++++++++++++++++++++++++++++++++- db/migrate_integration_test.go | 68 ++++++++++++++++++++++++++++++++++ db/migrate_test.go | 66 +++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 db/migrate_integration_test.go create mode 100644 db/migrate_test.go diff --git a/db/migrate.go b/db/migrate.go index 48b023b2..eebad5f2 100644 --- a/db/migrate.go +++ b/db/migrate.go @@ -6,17 +6,81 @@ package db import ( "context" "embed" + "errors" "fmt" "io/fs" + "regexp" "sort" "strings" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" ) //go:embed migrations/*.sql var migrationFiles embed.FS +// execQuerier is the slice of pgxpool.Pool / pgx.Conn migrations need; it lets +// integration tests drive applyMigration over a plain connection. +type execQuerier interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +var ( + lineCommentRe = regexp.MustCompile(`(?m)--[^\n]*`) + concurrentIndexRe = regexp.MustCompile(`(?is)\bCREATE\s+(?:UNIQUE\s+)?INDEX\s+CONCURRENTLY\s+(?:IF\s+NOT\s+EXISTS\s+)?("?[\w.]+"?)`) +) + +// concurrentIndexName returns the index a CREATE INDEX CONCURRENTLY migration builds. +func concurrentIndexName(sql string) (string, bool) { + m := concurrentIndexRe.FindStringSubmatch(lineCommentRe.ReplaceAllString(sql, "")) + if m == nil { + return "", false + } + return strings.Trim(m[1], `"`), true +} + +// isDuplicateRelation reports SQLSTATE 42P07 (relation already exists). +func isDuplicateRelation(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "42P07" +} + +// applyMigration runs one file outside a transaction. An interrupted CREATE INDEX +// CONCURRENTLY leaves an invalid index that trips 42P07 on retry; drop it and rebuild once. +func applyMigration(ctx context.Context, db execQuerier, sql string) error { + _, err := db.Exec(ctx, sql) + if err == nil || !isDuplicateRelation(err) { + return err + } + name, ok := concurrentIndexName(sql) + if !ok { + return err + } + ident := pgx.Identifier(strings.Split(name, ".")).Sanitize() + var valid bool + if scanErr := db.QueryRow(ctx, + "SELECT indisvalid FROM pg_index WHERE indexrelid = to_regclass($1)", ident, + ).Scan(&valid); scanErr != nil { + if errors.Is(scanErr, pgx.ErrNoRows) { + return err + } + return fmt.Errorf("%w (checking index %s: %v)", err, name, scanErr) + } + if valid { + fmt.Printf("index %s already built, recording migration\n", name) + return nil + } + if _, dropErr := db.Exec(ctx, "DROP INDEX CONCURRENTLY IF EXISTS "+ident); dropErr != nil { + return fmt.Errorf("dropping invalid index %s: %w", name, dropErr) + } + fmt.Printf("dropped invalid index %s, rebuilding\n", name) + _, err = db.Exec(ctx, sql) + return err +} + func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { _, err := pool.Exec(ctx, ` CREATE TABLE IF NOT EXISTS schema_migrations ( @@ -90,7 +154,7 @@ func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { return fmt.Errorf("failed to read migration %s: %w", entry.Name(), err) } - if _, err := pool.Exec(ctx, string(sql)); err != nil { + if err := applyMigration(ctx, pool, string(sql)); err != nil { return fmt.Errorf("failed to apply migration %s: %w", entry.Name(), err) } diff --git a/db/migrate_integration_test.go b/db/migrate_integration_test.go new file mode 100644 index 00000000..1239f5d8 --- /dev/null +++ b/db/migrate_integration_test.go @@ -0,0 +1,68 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +// CONCURRENTLY is illegal inside a transaction, so this uses a plain connection +// and a real table rather than the usual tx + TEMP table pattern. +func TestApplyMigrationRecoversInvalidIndexPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + + table := fmt.Sprintf("migrate_recovery_test_%d", time.Now().UnixNano()) + index := table + "_idx" + if _, err := conn.Exec(ctx, "CREATE TABLE "+table+" (v INT)"); err != nil { + t.Fatal(err) + } + defer func() { _, _ = conn.Exec(context.Background(), "DROP TABLE IF EXISTS "+table+" CASCADE") }() + + build := "CREATE INDEX CONCURRENTLY " + index + " ON " + table + " (v)" + if err := applyMigration(ctx, conn, build); err != nil { + t.Fatalf("initial build: %v", err) + } + + // A valid, already-built index (process died before recording) is a success. + if err := applyMigration(ctx, conn, build); err != nil { + t.Fatalf("valid existing index must be accepted: %v", err) + } + + // Simulate an interrupted build by flipping the catalog flag. + if _, err := conn.Exec(ctx, "UPDATE pg_index SET indisvalid = false WHERE indexrelid = to_regclass($1)", index); err != nil { + t.Skipf("cannot mark index invalid (needs catalog write privilege): %v", err) + } + if err := applyMigration(ctx, conn, build); err != nil { + t.Fatalf("invalid index must be dropped and rebuilt: %v", err) + } + var valid bool + if err := conn.QueryRow(ctx, "SELECT indisvalid FROM pg_index WHERE indexrelid = to_regclass($1)", index).Scan(&valid); err != nil { + t.Fatal(err) + } + if !valid { + t.Fatal("index still invalid after recovery") + } + + // Recovery is scoped to CONCURRENTLY migrations; a plain duplicate still errors. + plain := "CREATE INDEX " + index + " ON " + table + " (v)" + if err := applyMigration(ctx, conn, plain); !isDuplicateRelation(err) { + t.Fatalf("plain duplicate index must surface 42P07, got %v", err) + } +} diff --git a/db/migrate_test.go b/db/migrate_test.go new file mode 100644 index 00000000..f93f4442 --- /dev/null +++ b/db/migrate_test.go @@ -0,0 +1,66 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "errors" + "fmt" + "testing" + + "github.com/jackc/pgx/v5/pgconn" +) + +func TestConcurrentIndexName(t *testing.T) { + cases := []struct { + name string + sql string + want string + ok bool + }{ + {"bare 027 style with comments", `-- RunMigrations executes each file outside an explicit transaction. Keep this +-- as one statement so other subscribers can keep writing during the build. +-- If interrupted, check pg_index.indisvalid and remove this named index before +-- retrying; do not silently accept an invalid index with IF NOT EXISTS. +CREATE INDEX CONCURRENTLY idx_known_routes_iata_last_seen +ON known_routes (iata, last_seen DESC);`, "idx_known_routes_iata_last_seen", true}, + {"if not exists", "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_a ON t (c);", "idx_a", true}, + {"unique", "CREATE UNIQUE INDEX CONCURRENTLY idx_u ON t (c);", "idx_u", true}, + {"quoted", `CREATE INDEX CONCURRENTLY "Idx_Q" ON t (c);`, "Idx_Q", true}, + {"schema qualified", "CREATE INDEX CONCURRENTLY public.idx_s ON t (c);", "public.idx_s", true}, + {"lowercase multiline", "create index\n concurrently\n idx_l\n on t (c);", "idx_l", true}, + {"plain create index", "CREATE INDEX idx_p ON t (c);", "", false}, + {"only in comment", "-- CREATE INDEX CONCURRENTLY idx_c ON t (c);\nSELECT 1;", "", false}, + {"empty", "", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := concurrentIndexName(tc.sql) + if ok != tc.ok || got != tc.want { + t.Fatalf("got (%q, %v), want (%q, %v)", got, ok, tc.want, tc.ok) + } + }) + } +} + +func TestIsDuplicateRelation(t *testing.T) { + dup := &pgconn.PgError{Code: "42P07"} + cases := []struct { + name string + err error + want bool + }{ + {"42P07", dup, true}, + {"wrapped 42P07", fmt.Errorf("apply: %w", dup), true}, + {"other sqlstate", &pgconn.PgError{Code: "42501"}, false}, + {"plain error", errors.New("boom"), false}, + {"nil", nil, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isDuplicateRelation(tc.err); got != tc.want { + t.Fatalf("got %v, want %v", got, tc.want) + } + }) + } +} From b66573b3b7ff93829026ee26d43b45b097e42ed1 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:53:00 -0400 Subject: [PATCH 026/141] feat(observers): per-observer activity endpoint (#135) --- db/migrations/030_observation_airtime.sql | 2 + .../031_observation_airtime_backfill.sql | 30 ++ db/migrations/032_mv_observer_activity.sql | 24 ++ db/observers.go | 124 ++++++ db/observers_test.go | 367 ++++++++++++++++++ db/packets.go | 1 + db/queries/queries.sql | 58 ++- db/sqlc/mock/querier.go | 74 ++++ db/sqlc/models.go | 15 + db/sqlc/querier.go | 9 + db/sqlc/queries.sql.go | 233 ++++++++++- db/stats.go | 4 + docs/docs.go | 128 ++++++ docs/swagger.json | 128 ++++++ docs/swagger.yaml | 85 ++++ internal/api/handlers/observers.go | 76 ++++ internal/api/handlers/observers_test.go | 228 +++++++++++ internal/api/handlers/stub_reader_test.go | 8 + internal/api/observers.go | 28 ++ internal/api/reader.go | 5 + internal/background/tasks.go | 2 + internal/background/tasks_test.go | 7 +- internal/cache/cache_test.go | 4 + internal/cache/reader.go | 12 + internal/ingest/frame_length_test.go | 145 +++++++ internal/ingest/packet.go | 10 + internal/lora/airtime.go | 56 +++ internal/lora/airtime_test.go | 122 ++++++ 28 files changed, 1977 insertions(+), 8 deletions(-) create mode 100644 db/migrations/030_observation_airtime.sql create mode 100644 db/migrations/031_observation_airtime_backfill.sql create mode 100644 db/migrations/032_mv_observer_activity.sql create mode 100644 internal/ingest/frame_length_test.go create mode 100644 internal/lora/airtime.go create mode 100644 internal/lora/airtime_test.go diff --git a/db/migrations/030_observation_airtime.sql b/db/migrations/030_observation_airtime.sql new file mode 100644 index 00000000..1a7c29eb --- /dev/null +++ b/db/migrations/030_observation_airtime.sql @@ -0,0 +1,2 @@ +-- Time-on-air per observation so per-observer activity never joins back to packets. +ALTER TABLE packet_observations ADD COLUMN airtime_ms REAL; diff --git a/db/migrations/031_observation_airtime_backfill.sql b/db/migrations/031_observation_airtime_backfill.sql new file mode 100644 index 00000000..0a74f7d6 --- /dev/null +++ b/db/migrations/031_observation_airtime_backfill.sql @@ -0,0 +1,30 @@ +-- Separate from 030 so the ALTER's ACCESS EXCLUSIVE lock is released before this backfill runs. + +-- Backfill only: mirrors internal/lora.TimeOnAirMs (RadioLib getTimeOnAir, CRC on, explicit header). +CREATE FUNCTION lora_airtime_ms_backfill(frame_len int, sf int, bw real, cr int) RETURNS real +LANGUAGE sql IMMUTABLE AS $$ + SELECT ( + (CASE WHEN sf <= 8 THEN 32 ELSE 16 END) + 4.25 + + 8 + GREATEST(ceil((8*frame_len - 4*sf + 44)::double precision + / (4*(sf - 2*(CASE WHEN power(2, sf)/bw >= 16 THEN 1 ELSE 0 END)))) * cr, 0) + ) * (power(2, sf)/bw) +$$; + +-- Last 7 days like 015; parallelism off so the join spills to disk, not /dev/shm. +SET max_parallel_workers_per_gather = 0; + +UPDATE packet_observations po +SET airtime_ms = lora_airtime_ms_backfill( + 1 + CASE WHEN p.transport_codes_present THEN 4 ELSE 0 END + 1 + + COALESCE(octet_length(po.path_bytes), 0) + octet_length(p.raw_payload), + po.spread_factor, po.bandwidth_khz, po.coding_rate) +FROM packets p +WHERE p.packet_hash = po.packet_hash + AND po.heard_at > NOW() - INTERVAL '7 days' + AND po.airtime_ms IS NULL + AND po.spread_factor BETWEEN 7 AND 12 + AND po.bandwidth_khz > 0 + AND po.coding_rate BETWEEN 5 AND 8; + +RESET max_parallel_workers_per_gather; +DROP FUNCTION lora_airtime_ms_backfill(int, int, real, int); diff --git a/db/migrations/032_mv_observer_activity.sql b/db/migrations/032_mv_observer_activity.sql new file mode 100644 index 00000000..36407593 --- /dev/null +++ b/db/migrations/032_mv_observer_activity.sql @@ -0,0 +1,24 @@ +-- Hourly per-observer rollup so 7d/30d activity sums a few thousand rows instead of scanning +-- a month of observations. payload_type is a key so the breakdown comes from the same rows. +-- sqlc types the cast aggregates as non-null even though airtime_ms/snr_*/rssi_sum are NULL for +-- empty groups, so readers must gate on the matching *_n counts. +CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS +SELECT + observer_id, + payload_type, + date_trunc('hour', heard_at)::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY observer_id, payload_type, date_trunc('hour', heard_at); + +CREATE UNIQUE INDEX idx_mv_observer_activity_hourly + ON mv_observer_activity_hourly(observer_id, payload_type, bucket); diff --git a/db/observers.go b/db/observers.go index c8036ca4..1a71329a 100644 --- a/db/observers.go +++ b/db/observers.go @@ -13,6 +13,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/MeshCore-Beacon/beacon-server/internal/lora" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) @@ -226,6 +227,129 @@ func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uui return points, nil } +// GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. +// Buckets of an hour or coarser come from the hourly rollup; anything finer reads observations directly. +// Range and Interval are left empty for the handler to fill. +func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + obs, err := s.q.GetObserverByID(ctx, observerID) + if err != nil { + return nil, err + } + activity := &api.ObserverActivity{} + // radio is non-nil only when airtime is actually costable, so radio != null implies costed buckets + if obs.RadioSf != nil && obs.RadioBwKhz != nil && obs.RadioCr != nil && + *obs.RadioSf >= 7 && *obs.RadioSf <= 12 && *obs.RadioBwKhz > 0 && *obs.RadioCr > 0 { + activity.Radio = &api.ObserverActivityRadio{ + FreqMHz: obs.RadioFreqMhz, + SF: *obs.RadioSf, + BWKHz: *obs.RadioBwKhz, + CR: *obs.RadioCr, + PreambleSymbols: lora.PreambleSymbols(int(*obs.RadioSf)), + } + } + // round the window start up to a bucket boundary so the first bucket is never a partial one + start := time.Now().Add(-window).UTC() + since := start.Truncate(interval) + if since.Before(start) { + since = since.Add(interval) + } + sinceTS := pgtype.Timestamptz{Time: since, Valid: true} + binWidth := pgtype.Interval{Microseconds: interval.Microseconds(), Valid: true} + + if interval >= time.Hour { + rows, err := s.q.GetObserverActivityHourly(ctx, sqlc.GetObserverActivityHourlyParams{ + ObserverID: observerID, + Column2: sinceTS, + Column3: binWidth, + }) + if err != nil { + return nil, err + } + activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) + for _, r := range rows { + p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + if r.AirtimeN > 0 { + airtime := r.AirtimeMs + p.AirtimeMs = &airtime + } + if r.SnrN > 0 { + avg := r.SnrSum / float32(r.SnrN) + min := r.SnrMin + p.SNRAvg, p.SNRMin = &avg, &min + } + if r.RssiN > 0 { + avg := float32(r.RssiSum) / float32(r.RssiN) + p.RSSIAvg = &avg + } + activity.Points = append(activity.Points, p) + } + typeRows, err := s.q.GetObserverActivityHourlyPayloadTypes(ctx, sqlc.GetObserverActivityHourlyPayloadTypesParams{ + ObserverID: observerID, + Column2: sinceTS, + }) + if err != nil { + return nil, err + } + activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) + for _, v := range typeRows { + if v.PayloadType == nil { + continue + } + activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ + PayloadType: *v.PayloadType, + PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + Count: v.Count, + }) + } + return activity, nil + } + + rows, err := s.q.GetObserverActivityRaw(ctx, sqlc.GetObserverActivityRawParams{ + ObserverID: observerID, + Column2: sinceTS, + Column3: binWidth, + }) + if err != nil { + return nil, err + } + activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) + for _, r := range rows { + p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + if r.AirtimeN > 0 { + airtime := r.AirtimeMs + p.AirtimeMs = &airtime + } + if r.SnrN > 0 { + avg, min := r.SnrAvg, r.SnrMin + p.SNRAvg, p.SNRMin = &avg, &min + } + if r.RssiN > 0 { + avg := r.RssiAvg + p.RSSIAvg = &avg + } + activity.Points = append(activity.Points, p) + } + typeRows, err := s.q.GetObserverActivityRawPayloadTypes(ctx, sqlc.GetObserverActivityRawPayloadTypesParams{ + ObserverID: observerID, + Column2: sinceTS, + }) + if err != nil { + return nil, err + } + activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) + for _, v := range typeRows { + if v.PayloadType == nil { + continue + } + activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ + PayloadType: *v.PayloadType, + PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + Count: v.Count, + }) + } + return activity, nil +} + func (s *Store) ListObserverAdverts(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) { rows, err := s.q.ListObserverAdverts(ctx, sqlc.ListObserverAdvertsParams{ ObserverID: observerID, diff --git a/db/observers_test.go b/db/observers_test.go index f6f7572a..76ccf548 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -11,7 +11,9 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "go.uber.org/mock/gomock" ) @@ -550,3 +552,368 @@ func TestIsObserverByPubkey_NotFound(t *testing.T) { t.Error("expected false for missing observer") } } + +func activityObserver(sf, cr *int16, bw, freq *float32) sqlc.Observer { + return sqlc.Observer{ + ID: uuid.MustParse("00000000-0000-0000-0000-000000000001"), + RadioFreqMhz: freq, + RadioSf: sf, + RadioBwKhz: bw, + RadioCr: cr, + } +} + +func i16(v int16) *int16 { return &v } +func f32(v float32) *float32 { return &v } + +func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), f32(910.525)), nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyRow{{ + Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, + Observations: 12, + AirtimeMs: 0, + AirtimeN: 0, + SnrSum: 0, + SnrN: 0, + SnrMin: 0, + RssiSum: 0, + RssiN: 0, + }}, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{ + {PayloadType: i16(4), Count: 9}, + {PayloadType: nil, Count: 3}, + }, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got.Points) != 1 { + t.Fatalf("expected 1 point, got %d", len(got.Points)) + } + p := got.Points[0] + if p.T != 1700000000000 { + t.Errorf("expected T 1700000000000, got %d", p.T) + } + if p.Observations != 12 { + t.Errorf("expected 12 observations, got %d", p.Observations) + } + if p.AirtimeMs != nil { + t.Errorf("expected nil AirtimeMs, got %v", *p.AirtimeMs) + } + if p.SNRAvg != nil || p.SNRMin != nil { + t.Errorf("expected nil SNR fields, got %v %v", p.SNRAvg, p.SNRMin) + } + if p.RSSIAvg != nil { + t.Errorf("expected nil RSSIAvg, got %v", *p.RSSIAvg) + } + if len(got.PayloadTypes) != 1 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 { + t.Fatalf("expected one payload type 4 with count 9, got %+v", got.PayloadTypes) + } + if got.PayloadTypes[0].PayloadTypeName != api.PayloadTypeName(4) { + t.Errorf("expected payload type name %q, got %q", api.PayloadTypeName(4), got.PayloadTypes[0].PayloadTypeName) + } + if got.Radio == nil { + t.Fatal("expected radio") + } + if got.Radio.SF != 10 || got.Radio.BWKHz != 62.5 || got.Radio.CR != 5 || got.Radio.PreambleSymbols != 16 { + t.Errorf("unexpected radio %+v", *got.Radio) + } + if got.Radio.FreqMHz == nil || *got.Radio.FreqMHz != 910.525 { + t.Errorf("unexpected freq %v", got.Radio.FreqMHz) + } + if got.Range != "" || got.Interval != "" { + t.Errorf("expected range/interval left for the handler, got %q %q", got.Range, got.Interval) + } +} + +func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(7), i16(5), f32(250), nil), nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyRow{{ + Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, + Observations: 8, + AirtimeMs: 123.5, + AirtimeN: 3, + SnrSum: 30, + SnrN: 4, + SnrMin: -3.5, + RssiSum: -400, + RssiN: 4, + }}, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + p := got.Points[0] + if p.AirtimeMs == nil || *p.AirtimeMs != 123.5 { + t.Errorf("expected AirtimeMs 123.5, got %v", p.AirtimeMs) + } + if p.SNRAvg == nil || *p.SNRAvg != 7.5 { + t.Errorf("expected SNRAvg 7.5, got %v", p.SNRAvg) + } + if p.SNRMin == nil || *p.SNRMin != -3.5 { + t.Errorf("expected SNRMin -3.5, got %v", p.SNRMin) + } + if p.RSSIAvg == nil || *p.RSSIAvg != -100 { + t.Errorf("expected RSSIAvg -100, got %v", p.RSSIAvg) + } + if got.PayloadTypes == nil { + t.Error("expected non-nil PayloadTypes slice") + } + if got.Radio.FreqMHz != nil { + t.Errorf("expected nil freq, got %v", *got.Radio.FreqMHz) + } + if got.Radio.PreambleSymbols != 32 { + t.Errorf("expected 32 preamble symbols at SF7, got %d", got.Radio.PreambleSymbols) + } +} + +func TestGetObserverActivity_RawPath(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawRow{{ + Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, + Observations: 5, + AirtimeMs: 50, + AirtimeN: 5, + SnrAvg: 4.25, + SnrMin: 1.5, + SnrN: 2, + RssiAvg: -95.5, + RssiN: 0, + }}, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawPayloadTypesRow{{PayloadType: i16(1), Count: 5}}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + p := got.Points[0] + if p.Observations != 5 { + t.Errorf("expected 5 observations, got %d", p.Observations) + } + if p.AirtimeMs == nil || *p.AirtimeMs != 50 { + t.Errorf("expected AirtimeMs 50, got %v", p.AirtimeMs) + } + if p.SNRAvg == nil || *p.SNRAvg != 4.25 { + t.Errorf("expected SNRAvg 4.25, got %v", p.SNRAvg) + } + if p.SNRMin == nil || *p.SNRMin != 1.5 { + t.Errorf("expected SNRMin 1.5, got %v", p.SNRMin) + } + if p.RSSIAvg != nil { + t.Errorf("expected nil RSSIAvg with rssi_n 0, got %v", *p.RSSIAvg) + } + if len(got.PayloadTypes) != 1 || got.PayloadTypes[0].Count != 5 { + t.Errorf("unexpected payload types %+v", got.PayloadTypes) + } +} + +func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawRow{{ + Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, + Observations: 7, + AirtimeMs: 0, + AirtimeN: 0, + SnrAvg: 0, + SnrMin: 0, + SnrN: 0, + RssiAvg: 0, + RssiN: 0, + }}, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + p := got.Points[0] + if p.Observations != 7 { + t.Errorf("expected 7 observations, got %d", p.Observations) + } + if p.AirtimeMs != nil { + t.Errorf("expected nil AirtimeMs with airtime_n 0, got %v", *p.AirtimeMs) + } + if p.SNRAvg != nil || p.SNRMin != nil { + t.Errorf("expected nil SNR fields with snr_n 0, got %v %v", p.SNRAvg, p.SNRMin) + } + if p.RSSIAvg != nil { + t.Errorf("expected nil RSSIAvg with rssi_n 0, got %v", *p.RSSIAvg) + } +} + +func TestGetObserverActivity_RawPathEmpty(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawRow{}, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Points == nil || len(got.Points) != 0 { + t.Errorf("expected empty non-nil points, got %#v", got.Points) + } + if got.PayloadTypes == nil || len(got.PayloadTypes) != 0 { + t.Errorf("expected empty non-nil payload types, got %#v", got.PayloadTypes) + } +} + +func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyRow{}, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Points == nil || len(got.Points) != 0 { + t.Errorf("expected empty non-nil points, got %#v", got.Points) + } +} + +func TestGetObserverActivity_UnknownObserver(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000002") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(sqlc.Observer{}, pgx.ErrNoRows) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour) + if !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("expected pgx.ErrNoRows, got %v", err) + } + if got != nil { + t.Errorf("expected nil result, got %+v", got) + } +} + +func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { + cases := []struct { + name string + sf, cr *int16 + bw *float32 + }{ + {name: "sf nil", sf: nil, cr: i16(5), bw: f32(62.5)}, + {name: "sf zero", sf: i16(0), cr: i16(5), bw: f32(62.5)}, + {name: "sf below lora range", sf: i16(6), cr: i16(5), bw: f32(62.5)}, + {name: "sf above lora range", sf: i16(13), cr: i16(5), bw: f32(62.5)}, + {name: "bw zero", sf: i16(10), cr: i16(5), bw: f32(0)}, + {name: "cr nil", sf: i16(10), cr: nil, bw: f32(62.5)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(tc.sf, tc.cr, tc.bw, nil), nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyRow{}, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) + + store := &Store{q: mock} + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.Radio != nil { + t.Errorf("expected nil radio, got %+v", *got.Radio) + } + }) + } +} + +func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + window, interval := 24*time.Hour, 6*time.Hour + + var gotSince time.Time + var gotInterval pgtype.Interval + mock.EXPECT().GetObserverByID(gomock.Any(), observerID). + Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, arg sqlc.GetObserverActivityHourlyParams) ([]sqlc.GetObserverActivityHourlyRow, error) { + gotSince, gotInterval = arg.Column2.Time, arg.Column3 + return nil, nil + }) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). + Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) + + before := time.Now().Add(-window) + store := &Store{q: mock} + if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval); err != nil { + t.Fatalf("unexpected error: %v", err) + } + after := time.Now().Add(-window) + + if gotSince.UnixNano()%int64(interval) != 0 { + t.Errorf("since %s is not aligned to %s", gotSince, interval) + } + if gotSince.Before(before) { + t.Errorf("since %s is before the window start %s", gotSince, before) + } + if !gotSince.Before(after.Add(interval)) { + t.Errorf("since %s is more than one interval past the window start %s", gotSince, after) + } + if !gotInterval.Valid || gotInterval.Microseconds != interval.Microseconds() { + t.Errorf("expected interval %d us, got %+v", interval.Microseconds(), gotInterval) + } +} diff --git a/db/packets.go b/db/packets.go index b03eda68..ddf0ad22 100644 --- a/db/packets.go +++ b/db/packets.go @@ -586,6 +586,7 @@ func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservatio SourceBroker: &o.SourceBroker, PayloadType: &o.PayloadType, ResolvedEndpoints: o.ResolvedEndpoints, + AirtimeMs: o.AirtimeMs, } row, err := s.q.InsertObservation(ctx, params) if errors.Is(err, pgx.ErrNoRows) { diff --git a/db/queries/queries.sql b/db/queries/queries.sql index da3fa9bd..7c9a4021 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -260,6 +260,56 @@ WHERE observer_id = $1 GROUP BY bucket ORDER BY bucket ASC; +-- name: GetObserverActivityRaw :many +-- Sub-hour activity buckets straight off idx_observations_observer; no join to packets. +-- Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as +-- NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. +SELECT + date_bin($3::interval, heard_at, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + COALESCE(AVG(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS snr_avg, + COALESCE(MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS snr_min, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE observer_id = $1 AND heard_at >= $2::timestamptz +GROUP BY bucket +ORDER BY bucket; + +-- name: GetObserverActivityRawPayloadTypes :many +SELECT payload_type, COUNT(*)::bigint AS count +FROM packet_observations +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +GROUP BY payload_type +ORDER BY count DESC; + +-- name: GetObserverActivityHourly :many +-- Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. +SELECT + date_bin($3::interval, bucket, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + SUM(observations)::bigint AS observations, + COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, + SUM(airtime_n)::bigint AS airtime_n, + COALESCE(SUM(snr_sum), 0)::real AS snr_sum, + SUM(snr_n)::bigint AS snr_n, + COALESCE(MIN(snr_min), 0)::real AS snr_min, + COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, + SUM(rssi_n)::bigint AS rssi_n +FROM mv_observer_activity_hourly +WHERE observer_id = $1 AND bucket >= $2::timestamptz +GROUP BY 1 +ORDER BY 1; + +-- name: GetObserverActivityHourlyPayloadTypes :many +SELECT payload_type, SUM(observations)::bigint AS count +FROM mv_observer_activity_hourly +WHERE observer_id = $1 AND bucket >= $2::timestamptz +GROUP BY payload_type +ORDER BY count DESC; + -- name: ListObserverAdverts :many -- Returns advert packets (payload_type=4) heard by a specific observer. -- Pass cursor=0 to start from the beginning, or the last seen id for pagination. @@ -626,9 +676,10 @@ INSERT INTO packet_observations ( coding_rate, source_broker, payload_type, - resolved_endpoints + resolved_endpoints, + airtime_ms ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING RETURNING *; @@ -1330,6 +1381,9 @@ REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata; -- name: RefreshRadioPresets :exec REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets; +-- name: RefreshObserverActivity :exec +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_observer_activity_hourly; + -- name: ReconfirmRoutes :exec -- Checks the $1 least-recently-reconfirmed routes: deletes those with a departed -- hop node or a hop prefix now matching >1 node in that IATA (length-aware: diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index a1a5e7a3..e43434b0 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -292,6 +292,66 @@ func (mr *MockQuerierMockRecorder) GetNodesByIDs(ctx, dollar_1 any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodesByIDs", reflect.TypeOf((*MockQuerier)(nil).GetNodesByIDs), ctx, dollar_1) } +// GetObserverActivityHourly mocks base method. +func (m *MockQuerier) GetObserverActivityHourly(ctx context.Context, arg db.GetObserverActivityHourlyParams) ([]db.GetObserverActivityHourlyRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityHourly", ctx, arg) + ret0, _ := ret[0].([]db.GetObserverActivityHourlyRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityHourly indicates an expected call of GetObserverActivityHourly. +func (mr *MockQuerierMockRecorder) GetObserverActivityHourly(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityHourly", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityHourly), ctx, arg) +} + +// GetObserverActivityHourlyPayloadTypes mocks base method. +func (m *MockQuerier) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg db.GetObserverActivityHourlyPayloadTypesParams) ([]db.GetObserverActivityHourlyPayloadTypesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityHourlyPayloadTypes", ctx, arg) + ret0, _ := ret[0].([]db.GetObserverActivityHourlyPayloadTypesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityHourlyPayloadTypes indicates an expected call of GetObserverActivityHourlyPayloadTypes. +func (mr *MockQuerierMockRecorder) GetObserverActivityHourlyPayloadTypes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityHourlyPayloadTypes", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityHourlyPayloadTypes), ctx, arg) +} + +// GetObserverActivityRaw mocks base method. +func (m *MockQuerier) GetObserverActivityRaw(ctx context.Context, arg db.GetObserverActivityRawParams) ([]db.GetObserverActivityRawRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityRaw", ctx, arg) + ret0, _ := ret[0].([]db.GetObserverActivityRawRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityRaw indicates an expected call of GetObserverActivityRaw. +func (mr *MockQuerierMockRecorder) GetObserverActivityRaw(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityRaw", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityRaw), ctx, arg) +} + +// GetObserverActivityRawPayloadTypes mocks base method. +func (m *MockQuerier) GetObserverActivityRawPayloadTypes(ctx context.Context, arg db.GetObserverActivityRawPayloadTypesParams) ([]db.GetObserverActivityRawPayloadTypesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityRawPayloadTypes", ctx, arg) + ret0, _ := ret[0].([]db.GetObserverActivityRawPayloadTypesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityRawPayloadTypes indicates an expected call of GetObserverActivityRawPayloadTypes. +func (mr *MockQuerierMockRecorder) GetObserverActivityRawPayloadTypes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityRawPayloadTypes", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityRawPayloadTypes), ctx, arg) +} + // GetObserverBrokers mocks base method. func (m *MockQuerier) GetObserverBrokers(ctx context.Context, observerID uuid.UUID) ([]db.GetObserverBrokersRow, error) { m.ctrl.T.Helper() @@ -1098,6 +1158,20 @@ func (mr *MockQuerierMockRecorder) RefreshHourlyStats(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshHourlyStats", reflect.TypeOf((*MockQuerier)(nil).RefreshHourlyStats), ctx) } +// RefreshObserverActivity mocks base method. +func (m *MockQuerier) RefreshObserverActivity(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RefreshObserverActivity", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// RefreshObserverActivity indicates an expected call of RefreshObserverActivity. +func (mr *MockQuerierMockRecorder) RefreshObserverActivity(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshObserverActivity", reflect.TypeOf((*MockQuerier)(nil).RefreshObserverActivity), ctx) +} + // RefreshPayloadBreakdown mocks base method. func (m *MockQuerier) RefreshPayloadBreakdown(ctx context.Context) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 5bed2693..9f6ef210 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -77,6 +77,20 @@ type MvHourlyIataStat struct { ActiveObservers int64 `json:"active_observers"` } +type MvObserverActivityHourly struct { + ObserverID uuid.UUID `json:"observer_id"` + PayloadType *int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin float32 `json:"snr_min"` + RssiSum int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + type MvPayloadBreakdownByIatum struct { Iata string `json:"iata"` PayloadType *int16 `json:"payload_type"` @@ -293,6 +307,7 @@ type PacketObservation struct { SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` ResolvedEndpoints []byte `json:"resolved_endpoints"` + AirtimeMs *float32 `json:"airtime_ms"` } type Region struct { diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 4d1bdd19..7a64432e 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -50,6 +50,14 @@ type Querier interface { // Returns the neighbors of a node with details, ordered by most recently seen. GetNodeNeighbors(ctx context.Context, nodeID uuid.UUID) ([]GetNodeNeighborsRow, error) GetNodesByIDs(ctx context.Context, dollar_1 []uuid.UUID) ([]GetNodesByIDsRow, error) + // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. + GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) + GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) + // Sub-hour activity buckets straight off idx_observations_observer; no join to packets. + // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as + // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. + GetObserverActivityRaw(ctx context.Context, arg GetObserverActivityRawParams) ([]GetObserverActivityRawRow, error) + GetObserverActivityRawPayloadTypes(ctx context.Context, arg GetObserverActivityRawPayloadTypesParams) ([]GetObserverActivityRawPayloadTypesRow, error) GetObserverBrokers(ctx context.Context, observerID uuid.UUID) ([]GetObserverBrokersRow, error) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, error) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Observer, error) @@ -192,6 +200,7 @@ type Querier interface { // 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. ReconfirmRoutes(ctx context.Context, limit int32) error RefreshHourlyStats(ctx context.Context) error + RefreshObserverActivity(ctx context.Context) error RefreshPayloadBreakdown(ctx context.Context) error RefreshRadioPresets(ctx context.Context) error RefreshTopAdvertisers(ctx context.Context) error diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 0dce6ba9..45641d45 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -515,6 +515,216 @@ func (q *Queries) GetNodesByIDs(ctx context.Context, dollar_1 []uuid.UUID) ([]Ge return items, nil } +const getObserverActivityHourly = `-- name: GetObserverActivityHourly :many +SELECT + date_bin($3::interval, bucket, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + SUM(observations)::bigint AS observations, + COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, + SUM(airtime_n)::bigint AS airtime_n, + COALESCE(SUM(snr_sum), 0)::real AS snr_sum, + SUM(snr_n)::bigint AS snr_n, + COALESCE(MIN(snr_min), 0)::real AS snr_min, + COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, + SUM(rssi_n)::bigint AS rssi_n +FROM mv_observer_activity_hourly +WHERE observer_id = $1 AND bucket >= $2::timestamptz +GROUP BY 1 +ORDER BY 1 +` + +type GetObserverActivityHourlyParams struct { + ObserverID uuid.UUID `json:"observer_id"` + Column2 pgtype.Timestamptz `json:"column_2"` + Column3 pgtype.Interval `json:"column_3"` +} + +type GetObserverActivityHourlyRow struct { + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin float32 `json:"snr_min"` + RssiSum int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + +// Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. +func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) { + rows, err := q.db.Query(ctx, getObserverActivityHourly, arg.ObserverID, arg.Column2, arg.Column3) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetObserverActivityHourlyRow{} + for rows.Next() { + var i GetObserverActivityHourlyRow + if err := rows.Scan( + &i.Bucket, + &i.Observations, + &i.AirtimeMs, + &i.AirtimeN, + &i.SnrSum, + &i.SnrN, + &i.SnrMin, + &i.RssiSum, + &i.RssiN, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many +SELECT payload_type, SUM(observations)::bigint AS count +FROM mv_observer_activity_hourly +WHERE observer_id = $1 AND bucket >= $2::timestamptz +GROUP BY payload_type +ORDER BY count DESC +` + +type GetObserverActivityHourlyPayloadTypesParams struct { + ObserverID uuid.UUID `json:"observer_id"` + Column2 pgtype.Timestamptz `json:"column_2"` +} + +type GetObserverActivityHourlyPayloadTypesRow struct { + PayloadType *int16 `json:"payload_type"` + Count int64 `json:"count"` +} + +func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) { + rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetObserverActivityHourlyPayloadTypesRow{} + for rows.Next() { + var i GetObserverActivityHourlyPayloadTypesRow + if err := rows.Scan(&i.PayloadType, &i.Count); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getObserverActivityRaw = `-- name: GetObserverActivityRaw :many +SELECT + date_bin($3::interval, heard_at, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + COALESCE(AVG(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS snr_avg, + COALESCE(MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS snr_min, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE observer_id = $1 AND heard_at >= $2::timestamptz +GROUP BY bucket +ORDER BY bucket +` + +type GetObserverActivityRawParams struct { + ObserverID uuid.UUID `json:"observer_id"` + Column2 pgtype.Timestamptz `json:"column_2"` + Column3 pgtype.Interval `json:"column_3"` +} + +type GetObserverActivityRawRow struct { + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrAvg float32 `json:"snr_avg"` + SnrMin float32 `json:"snr_min"` + SnrN int64 `json:"snr_n"` + RssiAvg float32 `json:"rssi_avg"` + RssiN int64 `json:"rssi_n"` +} + +// Sub-hour activity buckets straight off idx_observations_observer; no join to packets. +// Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as +// NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. +func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverActivityRawParams) ([]GetObserverActivityRawRow, error) { + rows, err := q.db.Query(ctx, getObserverActivityRaw, arg.ObserverID, arg.Column2, arg.Column3) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetObserverActivityRawRow{} + for rows.Next() { + var i GetObserverActivityRawRow + if err := rows.Scan( + &i.Bucket, + &i.Observations, + &i.AirtimeMs, + &i.AirtimeN, + &i.SnrAvg, + &i.SnrMin, + &i.SnrN, + &i.RssiAvg, + &i.RssiN, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const getObserverActivityRawPayloadTypes = `-- name: GetObserverActivityRawPayloadTypes :many +SELECT payload_type, COUNT(*)::bigint AS count +FROM packet_observations +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +GROUP BY payload_type +ORDER BY count DESC +` + +type GetObserverActivityRawPayloadTypesParams struct { + ObserverID uuid.UUID `json:"observer_id"` + Column2 pgtype.Timestamptz `json:"column_2"` +} + +type GetObserverActivityRawPayloadTypesRow struct { + PayloadType *int16 `json:"payload_type"` + Count int64 `json:"count"` +} + +func (q *Queries) GetObserverActivityRawPayloadTypes(ctx context.Context, arg GetObserverActivityRawPayloadTypesParams) ([]GetObserverActivityRawPayloadTypesRow, error) { + rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetObserverActivityRawPayloadTypesRow{} + for rows.Next() { + var i GetObserverActivityRawPayloadTypesRow + if err := rows.Scan(&i.PayloadType, &i.Count); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getObserverBrokers = `-- name: GetObserverBrokers :many SELECT broker_name, last_seen, last_packet_at FROM observer_brokers @@ -1722,12 +1932,13 @@ INSERT INTO packet_observations ( coding_rate, source_broker, payload_type, - resolved_endpoints + resolved_endpoints, + airtime_ms ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING -RETURNING id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type, resolved_endpoints +RETURNING id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type, resolved_endpoints, airtime_ms ` type InsertObservationParams struct { @@ -1749,6 +1960,7 @@ type InsertObservationParams struct { SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` ResolvedEndpoints []byte `json:"resolved_endpoints"` + AirtimeMs *float32 `json:"airtime_ms"` } // ============================================================ @@ -1774,6 +1986,7 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa arg.SourceBroker, arg.PayloadType, arg.ResolvedEndpoints, + arg.AirtimeMs, ) var i PacketObservation err := row.Scan( @@ -1796,6 +2009,7 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa &i.SourceBroker, &i.PayloadType, &i.ResolvedEndpoints, + &i.AirtimeMs, ) return i, err } @@ -2562,7 +2776,7 @@ func (q *Queries) ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNod } const listObservationsForPacket = `-- name: ListObservationsForPacket :many -SELECT po.id, po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, po.path_bytes, po.rssi, po.snr, po.propagation_time_ms, po.radio_freq_mhz, po.spread_factor, po.bandwidth_khz, po.coding_rate, po.source_broker, po.payload_type, po.resolved_endpoints, o.display_name AS observer_name +SELECT po.id, po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, po.path_bytes, po.rssi, po.snr, po.propagation_time_ms, po.radio_freq_mhz, po.spread_factor, po.bandwidth_khz, po.coding_rate, po.source_broker, po.payload_type, po.resolved_endpoints, po.airtime_ms, o.display_name AS observer_name FROM packet_observations po LEFT JOIN observers o ON o.id = po.observer_id WHERE po.packet_hash = $1 @@ -2589,6 +2803,7 @@ type ListObservationsForPacketRow struct { SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` ResolvedEndpoints []byte `json:"resolved_endpoints"` + AirtimeMs *float32 `json:"airtime_ms"` ObserverName *string `json:"observer_name"` } @@ -2621,6 +2836,7 @@ func (q *Queries) ListObservationsForPacket(ctx context.Context, packetHash []by &i.SourceBroker, &i.PayloadType, &i.ResolvedEndpoints, + &i.AirtimeMs, &i.ObserverName, ); err != nil { return nil, err @@ -3482,6 +3698,15 @@ func (q *Queries) RefreshHourlyStats(ctx context.Context) error { return err } +const refreshObserverActivity = `-- name: RefreshObserverActivity :exec +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_observer_activity_hourly +` + +func (q *Queries) RefreshObserverActivity(ctx context.Context) error { + _, err := q.db.Exec(ctx, refreshObserverActivity) + return err +} + const refreshPayloadBreakdown = `-- name: RefreshPayloadBreakdown :exec REFRESH MATERIALIZED VIEW CONCURRENTLY mv_payload_breakdown_by_iata ` diff --git a/db/stats.go b/db/stats.go index ffbcabb6..b8dc915c 100644 --- a/db/stats.go +++ b/db/stats.go @@ -303,3 +303,7 @@ func (s *Store) RefreshTopObservers(ctx context.Context) error { func (s *Store) RefreshRadioPresets(ctx context.Context) error { return s.q.RefreshRadioPresets(ctx) } + +func (s *Store) RefreshObserverActivity(ctx context.Context) error { + return s.q.RefreshObserverActivity(ctx) +} diff --git a/docs/docs.go b/docs/docs.go index 3a97dd94..9d560071 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -905,6 +905,64 @@ const docTemplate = `{ } } }, + "/observers/{observerId}/activity": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Observers" + ], + "summary": "Get observer heard-activity history", + "parameters": [ + { + "type": "string", + "description": "Observer UUID", + "name": "observerId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h", + "name": "range", + "in": "query" + }, + { + "type": "string", + "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", + "name": "interval", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/observers/{observerId}/adverts": { "get": { "produces": [ @@ -3004,6 +3062,76 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { + "type": "object", + "properties": { + "interval": { + "type": "string" + }, + "payloadTypes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem" + } + }, + "points": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint" + } + }, + "radio": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio" + }, + "range": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint": { + "type": "object", + "properties": { + "airtimeMs": { + "type": "number" + }, + "observations": { + "type": "integer" + }, + "rssiAvg": { + "type": "number" + }, + "snrAvg": { + "type": "number" + }, + "snrMin": { + "type": "number" + }, + "t": { + "description": "bucket start, epoch ms", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio": { + "type": "object", + "properties": { + "bwKhz": { + "type": "number" + }, + "cr": { + "type": "integer" + }, + "freqMhz": { + "type": "number" + }, + "preambleSymbols": { + "type": "integer" + }, + "sf": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index c9f19cf7..ede57845 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -903,6 +903,64 @@ } } }, + "/observers/{observerId}/activity": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "Observers" + ], + "summary": "Get observer heard-activity history", + "parameters": [ + { + "type": "string", + "description": "Observer UUID", + "name": "observerId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h", + "name": "range", + "in": "query" + }, + { + "type": "string", + "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", + "name": "interval", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/observers/{observerId}/adverts": { "get": { "produces": [ @@ -3002,6 +3060,76 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { + "type": "object", + "properties": { + "interval": { + "type": "string" + }, + "payloadTypes": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem" + } + }, + "points": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint" + } + }, + "radio": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio" + }, + "range": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint": { + "type": "object", + "properties": { + "airtimeMs": { + "type": "number" + }, + "observations": { + "type": "integer" + }, + "rssiAvg": { + "type": "number" + }, + "snrAvg": { + "type": "number" + }, + "snrMin": { + "type": "number" + }, + "t": { + "description": "bucket start, epoch ms", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio": { + "type": "object", + "properties": { + "bwKhz": { + "type": "number" + }, + "cr": { + "type": "integer" + }, + "freqMhz": { + "type": "number" + }, + "preambleSymbols": { + "type": "integer" + }, + "sf": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 147830bc..7efd8ad1 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -480,6 +480,52 @@ definitions: uptimeSeconds: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity: + properties: + interval: + type: string + payloadTypes: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem' + type: array + points: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint' + type: array + radio: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio' + range: + type: string + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint: + properties: + airtimeMs: + type: number + observations: + type: integer + rssiAvg: + type: number + snrAvg: + type: number + snrMin: + type: number + t: + description: bucket start, epoch ms + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio: + properties: + bwKhz: + type: number + cr: + type: integer + freqMhz: + type: number + preambleSymbols: + type: integer + sf: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker: properties: lastPacketAt: @@ -1739,6 +1785,45 @@ paths: summary: Get observer detail tags: - Observers + /observers/{observerId}/activity: + get: + parameters: + - description: Observer UUID + in: path + name: observerId + required: true + type: string + - description: Trailing window as a Go duration, max 720h (default 24h); max + 48h when interval is under 1h + in: query + name: range + type: string + - description: 'Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)' + in: query + name: interval + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Get observer heard-activity history + tags: + - Observers /observers/{observerId}/adverts: get: parameters: diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index d559cbdb..01491b5f 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -4,6 +4,7 @@ package handlers import ( + "errors" "net/http" "strconv" "time" @@ -11,6 +12,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" "github.com/google/uuid" + "github.com/jackc/pgx/v5" ) // ObserversRouter mounts all /observers routes onto a subrouter. @@ -18,6 +20,7 @@ import ( // GET /observers → listObservers // GET /observers/{observerId} → getObserver // GET /observers/{observerId}/telemetry → getObserverTelemetry +// GET /observers/{observerId}/activity → getObserverActivity // GET /observers/{observerId}/adverts → listObserverAdverts func ObserversRouter(reader api.Reader) http.Handler { r := chi.NewRouter() @@ -26,6 +29,7 @@ func ObserversRouter(reader api.Reader) http.Handler { r.Get("/", getObserver(reader)) r.Get("/adverts", listObserverAdverts(reader)) r.Get("/telemetry", getObserverTelemetry(reader)) + r.Get("/activity", getObserverActivity(reader)) }) return r } @@ -232,3 +236,75 @@ func getObserverTelemetry(reader api.Reader) http.HandlerFunc { respond(w, http.StatusOK, telemetry) } } + +// activityIntervals is the fixed bucket set; every value divides 24h so bucket +// starts stay aligned to the clock regardless of when the window began. +var activityIntervals = map[string]time.Duration{ + "5m": 5 * time.Minute, + "15m": 15 * time.Minute, + "1h": time.Hour, + "6h": 6 * time.Hour, + "24h": 24 * time.Hour, +} + +// getObserverActivity godoc +// +// @Summary Get observer heard-activity history +// @Tags Observers +// @Produce json +// @Param observerId path string true "Observer UUID" +// @Param range query string false "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h" +// @Param interval query string false "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)" +// @Success 200 {object} api.ObserverActivity +// @Failure 400 {object} handlers.APIError +// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Router /observers/{observerId}/activity [get] +func getObserverActivity(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + observerID, err := uuid.Parse(chi.URLParam(r, "observerId")) + if err != nil { + respondError(w, http.StatusBadRequest, "invalid observer ID") + return + } + rangeParam := r.URL.Query().Get("range") + if rangeParam == "" { + rangeParam = "24h" + } + window, err := time.ParseDuration(rangeParam) + if err != nil || window <= 0 || window > 720*time.Hour { + respondError(w, http.StatusBadRequest, "invalid range, use a duration up to 720h e.g. 24h, 168h, 720h") + return + } + intervalParam := r.URL.Query().Get("interval") + if intervalParam == "" { + intervalParam = "15m" + } + interval, ok := activityIntervals[intervalParam] + if !ok { + respondError(w, http.StatusBadRequest, "invalid interval, use 5m, 15m, 1h, 6h or 24h") + return + } + // the raw path scans live observations, so keep sub-hour windows short + if interval < time.Hour && window > 48*time.Hour { + respondError(w, http.StatusBadRequest, "range must be 48h or less for intervals under 1h") + return + } + if window/interval > 1000 { + respondError(w, http.StatusBadRequest, "range/interval exceeds 1000 buckets") + return + } + activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + respondError(w, http.StatusNotFound, "observer not found") + return + } + respondError(w, http.StatusInternalServerError, "internal server error") + return + } + activity.Range = rangeParam + activity.Interval = intervalParam + respond(w, http.StatusOK, activity) + } +} diff --git a/internal/api/handlers/observers_test.go b/internal/api/handlers/observers_test.go index 148a06f8..0d4a028c 100644 --- a/internal/api/handlers/observers_test.go +++ b/internal/api/handlers/observers_test.go @@ -5,6 +5,8 @@ package handlers import ( "context" + "encoding/json" + "errors" "net/http" "net/http/httptest" "testing" @@ -13,6 +15,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" "github.com/google/uuid" + "github.com/jackc/pgx/v5" ) func TestGetObserverTelemetry_InvalidUUID(t *testing.T) { @@ -168,3 +171,228 @@ func TestListObserverAdverts_InvalidUUID(t *testing.T) { t.Errorf("expected 400, got %d", w.Code) } } + +func TestGetObserverActivity_Defaults(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + var gotWindow, gotInterval time.Duration + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + gotWindow, gotInterval = window, interval + return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", w.Code) + } + if gotWindow != 24*time.Hour { + t.Errorf("expected window 24h, got %s", gotWindow) + } + if gotInterval != 15*time.Minute { + t.Errorf("expected interval 15m, got %s", gotInterval) + } + var body api.ObserverActivity + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Range != "24h" || body.Interval != "15m" { + t.Errorf("expected range 24h interval 15m, got %q/%q", body.Range, body.Interval) + } +} + +func TestGetObserverActivity_CustomRangeAndInterval(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + var gotWindow, gotInterval time.Duration + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + gotWindow, gotInterval = window, interval + return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity?range=168h&interval=1h", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", w.Code) + } + if gotWindow != 168*time.Hour { + t.Errorf("expected window 168h, got %s", gotWindow) + } + if gotInterval != time.Hour { + t.Errorf("expected interval 1h, got %s", gotInterval) + } + var body api.ObserverActivity + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Range != "168h" || body.Interval != "1h" { + t.Errorf("expected range 168h interval 1h, got %q/%q", body.Range, body.Interval) + } +} + +func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + t.Fatal("reader should not be called") + return nil, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity?range=168h&interval=15m", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Fatalf("expected 400, got %d (%s)", w.Code, w.Body.String()) + } + var errBody map[string]APIError + if err := json.Unmarshal(w.Body.Bytes(), &errBody); err != nil { + t.Fatalf("decode body: %v", err) + } + if errBody["error"].Message != "range must be 48h or less for intervals under 1h" { + t.Errorf("unexpected error message %q", errBody["error"].Message) + } + + var gotWindow, gotInterval time.Duration + ok := chi.NewRouter() + ok.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + gotWindow, gotInterval = window, interval + return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil + }, + })) + req = httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity?range=48h&interval=15m", nil) + w = httptest.NewRecorder() + ok.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 at 48h/15m, got %d (%s)", w.Code, w.Body.String()) + } + if gotWindow != 48*time.Hour || gotInterval != 15*time.Minute { + t.Errorf("expected 48h/15m, got %s/%s", gotWindow, gotInterval) + } +} + +func TestGetObserverActivity_BadRequests(t *testing.T) { + tests := []struct { + name string + path string + query string + }{ + {"invalid uuid", "not-a-uuid", ""}, + {"unparseable range", "00000000-0000-0000-0000-000000000001", "?range=banana"}, + {"range too long", "00000000-0000-0000-0000-000000000001", "?range=721h"}, + {"zero range", "00000000-0000-0000-0000-000000000001", "?range=0"}, + {"invalid interval", "00000000-0000-0000-0000-000000000001", "?interval=2h"}, + {"too many buckets", "00000000-0000-0000-0000-000000000001", "?range=720h&interval=5m"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + t.Fatal("reader should not be called") + return nil, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+tt.path+"/activity"+tt.query, nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Errorf("expected 400, got %d (%s)", w.Code, w.Body.String()) + } + }) + } +} + +func TestGetObserverActivity_NotFound(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + return nil, pgx.ErrNoRows + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusNotFound { + t.Fatalf("expected 404, got %d", w.Code) + } + var body struct { + Error APIError `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Error.Code != "not_found" { + t.Errorf("expected code not_found, got %q", body.Error.Code) + } +} + +func TestGetObserverActivity_ReaderError(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + return nil, errors.New("boom") + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusInternalServerError { + t.Errorf("expected 500, got %d", w.Code) + } +} + +func TestGetObserverActivity_NullsSerialise(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + r := chi.NewRouter() + r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + return &api.ObserverActivity{ + PayloadTypes: []api.PayloadBreakdownItem{}, + Points: []api.ObserverActivityPoint{{T: 1757376000000, Observations: 3}}, + }, nil + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/activity", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", w.Code) + } + var body map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("decode body: %v", err) + } + radio, ok := body["radio"] + if !ok { + t.Fatal("expected radio key in body") + } + if radio != nil { + t.Errorf("expected radio null, got %v", radio) + } + points, ok := body["points"].([]any) + if !ok || len(points) != 1 { + t.Fatalf("expected 1 point, got %v", body["points"]) + } + point := points[0].(map[string]any) + airtime, ok := point["airtimeMs"] + if !ok { + t.Fatal("expected airtimeMs key in point") + } + if airtime != nil { + t.Errorf("expected airtimeMs null, got %v", airtime) + } +} diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 2b97ed23..4cd3d87a 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -31,6 +31,7 @@ type stubReader struct { getObserver func(ctx context.Context, observerID uuid.UUID) (*api.Observer, error) getObserverTelemetry func(ctx context.Context, observerID uuid.UUID, since, until time.Time, afterID int64) (*api.ObserverTelemetry, error) getObserverTelemetryBucketed func(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]api.ObserverTelemetryPoint, error) + getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) getObserverScopes func(ctx context.Context, observerID uuid.UUID) ([]string, error) listObserverAdverts func(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) listNodes func(ctx context.Context, nodeType int16, iatas []string, supportsMultibytePaths, supportsMultibyteTraces *bool, pubkey []byte, pubkeyPrefix, name, scope string, cursor int64, limit int32, includeNeighbors bool) (api.Page[api.NodeSummary], error) @@ -169,6 +170,13 @@ func (s stubReader) GetObserverTelemetryBucketed(ctx context.Context, observerID return nil, nil } +func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + if s.getObserverActivity != nil { + return s.getObserverActivity(ctx, observerID, window, interval) + } + return nil, nil +} + func (s stubReader) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) { if s.getObserverScopes != nil { return s.getObserverScopes(ctx, observerID) diff --git a/internal/api/observers.go b/internal/api/observers.go index 94d10789..afe800f1 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -67,3 +67,31 @@ type ObserverTelemetry struct { Interval string `json:"interval"` Points []ObserverTelemetryPoint `json:"points"` } + +// ObserverActivityRadio echoes the observer's current radio params plus the preamble the airtime math assumes. +type ObserverActivityRadio struct { + FreqMHz *float32 `json:"freqMhz"` + SF int16 `json:"sf"` + BWKHz float32 `json:"bwKhz"` + CR int16 `json:"cr"` + PreambleSymbols int `json:"preambleSymbols"` +} + +// ObserverActivityPoint is one bucket of what an observer heard. +type ObserverActivityPoint struct { + T int64 `json:"t"` // bucket start, epoch ms + Observations int64 `json:"observations"` + AirtimeMs *float32 `json:"airtimeMs"` + SNRAvg *float32 `json:"snrAvg"` + SNRMin *float32 `json:"snrMin"` + RSSIAvg *float32 `json:"rssiAvg"` +} + +// ObserverActivity is the per-observer heard-activity response. +type ObserverActivity struct { + Range string `json:"range"` + Interval string `json:"interval"` + Radio *ObserverActivityRadio `json:"radio"` + PayloadTypes []PayloadBreakdownItem `json:"payloadTypes"` + Points []ObserverActivityPoint `json:"points"` +} diff --git a/internal/api/reader.go b/internal/api/reader.go index b8b353a9..dc3e1dc5 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -97,6 +97,11 @@ type Reader interface { // GetObserverTelemetryBucketed returns telemetry points for an observer bucketed into N-hour intervals. GetObserverTelemetryBucketed(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]ObserverTelemetryPoint, error) + // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. + // interval >= 1h is served from the hourly rollup. Returns pgx.ErrNoRows for an unknown observer. + // Range and Interval on the result are left empty for the handler to fill. + GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*ObserverActivity, error) + // GetObserverScopes returns the names of all transport scopes an observer has // been seen forwarding packets for, ordered alphabetically. GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) diff --git a/internal/background/tasks.go b/internal/background/tasks.go index a19b6e4b..b29760ad 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -21,6 +21,7 @@ type viewRefresher interface { RefreshTopTalkers(context.Context) error RefreshTopAdvertisers(context.Context) error RefreshRadioPresets(context.Context) error + RefreshObserverActivity(context.Context) error } // ViewRefreshTask returns a Task that refreshes all materialized views. @@ -41,6 +42,7 @@ func ViewRefreshTask(store viewRefresher, interval time.Duration) Task { {"top talkers", store.RefreshTopTalkers}, {"top advertisers", store.RefreshTopAdvertisers}, {"radio presets", store.RefreshRadioPresets}, + {"observer activity", store.RefreshObserverActivity}, } { if err := view.refresh(ctx); err != nil { errs = append(errs, fmt.Errorf("%s: %w", view.name, err)) diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go index b84973a4..ca9fb999 100644 --- a/internal/background/tasks_test.go +++ b/internal/background/tasks_test.go @@ -111,6 +111,9 @@ func (s *refreshStub) RefreshTopAdvertisers(ctx context.Context) error { func (s *refreshStub) RefreshRadioPresets(ctx context.Context) error { return s.refresh(ctx, "radio presets") } +func (s *refreshStub) RefreshObserverActivity(ctx context.Context) error { + return s.refresh(ctx, "observer activity") +} func TestViewRefreshTask(t *testing.T) { first, second := errors.New("first failure"), errors.New("second failure") @@ -125,8 +128,8 @@ func TestViewRefreshTask(t *testing.T) { store := &refreshStub{errs: tc.errs} task := ViewRefreshTask(store, time.Minute) err := task.Run(context.Background()) - if len(store.calls) != 7 { - t.Fatalf("refreshed %d views, want 7", len(store.calls)) + if len(store.calls) != 8 { + t.Fatalf("refreshed %d views, want 8", len(store.calls)) } if len(tc.errs) == 0 && err != nil { t.Fatal(err) diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index eb1ad142..68c968b7 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -121,6 +121,10 @@ func (s *stubReader) GetObserverTelemetryBucketed(_ context.Context, _ uuid.UUID return nil, nil } +func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + return nil, nil +} + func (s *stubReader) GetPacket(_ context.Context, _ []byte) (*api.Packet, error) { return nil, nil } func (s *stubReader) GetChannel(_ context.Context, _ int32) (*api.Channel, error) { return nil, nil } diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 04dc5b75..4c69bd27 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -41,8 +41,12 @@ const ( keyNodesByIDsPrefix = "beacon:nodes:ids:" keyObserverPrefix = "beacon:observer:" keyObserverScopesPrefix = "beacon:observer:scopes:" + keyObserverActivityPrefix = "beacon:observer:activity:" ) +// observerActivityTTL is deliberately short: activity is a live chart, not reference data. +const observerActivityTTL = 60 * time.Second + // CachedReader wraps an api.Reader with a Redis caching layer. // It implements api.Reader and is a drop-in replacement for db.Store // at the wiring point in main.go. @@ -352,6 +356,14 @@ func (cr *CachedReader) GetObserverTelemetryBucketed(ctx context.Context, observ return cr.inner.GetObserverTelemetryBucketed(ctx, observerID, since, until, bucketHours) } +// GetObserverActivity implements [api.Reader]. +func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) { + return cr.inner.GetObserverActivity(ctx, observerID, window, interval) + }) +} + // GetPacket implements [api.Reader]. func (cr *CachedReader) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, error) { return cr.inner.GetPacket(ctx, packetHash) diff --git a/internal/ingest/frame_length_test.go b/internal/ingest/frame_length_test.go new file mode 100644 index 00000000..d69601d8 --- /dev/null +++ b/internal/ingest/frame_length_test.go @@ -0,0 +1,145 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/lora" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +// frameCaptureDB records what handlePacket stored so a test can rebuild the +// frame from the persisted columns and compare it to the bytes off the wire. +type frameCaptureDB struct { + *stubDB + radio RadioSettings + packets []UpsertPacketParams + observed []InsertObservationParams +} + +func (s *frameCaptureDB) UpsertPacket(_ context.Context, p UpsertPacketParams) (bool, error) { + s.packets = append(s.packets, p) + return true, nil +} + +func (s *frameCaptureDB) InsertObservation(_ context.Context, o InsertObservationParams) (bool, error) { + s.observed = append(s.observed, o) + return true, nil +} + +func (s *frameCaptureDB) GetObserverRadio(context.Context, uuid.UUID) (RadioSettings, error) { + return s.radio, nil +} + +// grpTxtPayload is a stand-in encrypted payload; its bytes never need decoding here. +func grpTxtPayload(t *testing.T) []byte { + t.Helper() + return buildGrpTxtPacket(t, 0x1a, make([]byte, 16)).Payload +} + +// pathBytes returns n distinct hashes of hashSize bytes each. +func pathBytes(hashCount, hashSize int) []byte { + path := make([]byte, 0, hashCount*hashSize) + for i := range hashCount { + for j := range hashSize { + path = append(path, byte(0x10+i*hashSize+j)) + } + } + return path +} + +// pathLengthByte packs hash size and hop count the way MeshCore does. +func pathLengthByte(hashCount, hashSize int) uint8 { + return uint8(hashSize-1)<<6 | uint8(hashCount) +} + +// TestFrameLengthReconstruction proves lora.FrameLength rebuilds the on-air byte +// count from the stored columns alone, across the path/transport-code shapes. +func TestFrameLengthReconstruction(t *testing.T) { + for _, tc := range []struct { + name string + packet *meshcore.Packet + wantTransportCodes bool + }{ + { + name: "flood with 1-byte hash path", + packet: &meshcore.Packet{ + Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeGrpTxt, 0), + PathLength: pathLengthByte(2, 1), + Path: pathBytes(2, 1), + Payload: grpTxtPayload(t), + }, + }, + { + name: "transport flood with transport codes", + packet: &meshcore.Packet{ + Header: meshcore.MakeHeader(meshcore.RouteTypeTransportFlood, meshcore.PayloadTypeGrpTxt, 0), + PathLength: pathLengthByte(1, 1), + Path: pathBytes(1, 1), + Payload: grpTxtPayload(t), + TransportCode1: 0xbeef, + TransportCode2: 0xf00d, + }, + wantTransportCodes: true, + }, + { + name: "flood with 2-byte hash path", + packet: &meshcore.Packet{ + Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeGrpTxt, 0), + PathLength: pathLengthByte(3, 2), + Path: pathBytes(3, 2), + Payload: grpTxtPayload(t), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + raw, err := tc.packet.ToBytes() + if err != nil { + t.Fatalf("packet to bytes: %v", err) + } + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base, radio: RadioSettings{FreqMHz: 910.525, SF: 7, BWKHz: 62.5, CR: 5}} + w.db = db + + w.handlePacket(context.Background(), "YOW", "0102", packetEnvelope(t, tc.packet)) + + if len(db.packets) != 1 || len(db.observed) != 1 { + t.Fatalf("stored %d packets and %d observations, want 1 of each", len(db.packets), len(db.observed)) + } + up, obs := db.packets[0], db.observed[0] + // Pin the captured shape: without this a lost transport-code capture would + // quietly turn this case into the plain-flood one and still pass. + if gotCodes := len(up.TransportCodes) == 4; gotCodes != tc.wantTransportCodes { + t.Fatalf("captured %d transport-code bytes, want transport codes = %v", len(up.TransportCodes), tc.wantTransportCodes) + } + got := lora.FrameLength(len(up.TransportCodes) == 4, len(obs.PathBytes), len(up.RawPayload)) + if got != len(raw) { + t.Errorf("FrameLength = %d, want %d (wire bytes)", got, len(raw)) + } + if obs.AirtimeMs == nil { + t.Error("AirtimeMs is nil, want a cost for SF7/62.5/CR5") + } + }) + } +} + +// TestObservationAirtimeUnknownRadio covers the observer that never reported its +// radio: the columns are zero, so there is no airtime to store. +func TestObservationAirtimeUnknownRadio(t *testing.T) { + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} // zero RadioSettings + w.db = db + + w.handlePacket(context.Background(), "YOW", "0102", packetEnvelope(t, buildGrpTxtPacket(t, 0x1a, make([]byte, 16)))) + + if len(db.observed) != 1 { + t.Fatalf("stored %d observations, want 1", len(db.observed)) + } + if got := db.observed[0].AirtimeMs; got != nil { + t.Errorf("AirtimeMs = %v, want nil when the radio settings are unknown", *got) + } +} diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 4a16a04c..be8d007c 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -16,6 +16,7 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/lora" "github.com/google/uuid" "github.com/meshcore-go/meshcore-go" ) @@ -56,6 +57,7 @@ type InsertObservationParams struct { SourceBroker string PayloadType int16 ResolvedEndpoints json.RawMessage + AirtimeMs *float32 // nil when the observer never reported costable radio settings } // RadioSettings holds the radio configuration for an observer, populated from @@ -787,6 +789,13 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ resolvedEndpoints = nil // optional enrichment must not discard the observation } } + // Airtime is costed from the frame as received; zero radio columns mean the + // observer never reported its settings, so there is nothing to cost. + var airtimeMs *float32 + if ms, ok := lora.TimeOnAirMs(len(hexBytes), int(radio.SF), float64(radio.BWKHz), int(radio.CR)); ok { + cost := float32(ms) + airtimeMs = &cost + } oParams := InsertObservationParams{ PacketHash: packetHash[:], ObserverID: id, @@ -806,6 +815,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ SourceBroker: w.cfg.BrokerName, PayloadType: int16(packet.PayloadType()), ResolvedEndpoints: resolvedEndpoints, + AirtimeMs: airtimeMs, } inserted, err := w.db.InsertObservation(ctx, oParams) if err != nil { diff --git a/internal/lora/airtime.go b/internal/lora/airtime.go new file mode 100644 index 00000000..1bd8a159 --- /dev/null +++ b/internal/lora/airtime.go @@ -0,0 +1,56 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package lora computes LoRa airtime for MeshCore frames, so an observation +// can be costed in milliseconds of channel occupancy. +package lora + +import "math" + +const ( + minSF = 7 + maxSF = 12 + minCR = 5 + maxCR = 8 +) + +// PreambleSymbols mirrors MeshCore's preambleLengthForSF: 32 symbols at SF<=8, 16 above. +func PreambleSymbols(sf int) int { + if sf <= 8 { + return 32 + } + return 16 +} + +// FrameLength is the on-air byte count of a MeshCore frame rebuilt from its stored parts. +func FrameLength(transportCodes bool, pathLen, payloadLen int) int { + n := 1 + 1 + pathLen + payloadLen // header byte + path length byte + if transportCodes { + n += 4 + } + return n +} + +// TimeOnAirMs is the Semtech LoRa time-on-air (RadioLib getTimeOnAir: CRC on, explicit header, +// LDRO auto at t_sym >= 16 ms). ok is false when any parameter is outside the costable range. +func TimeOnAirMs(frameLen, sf int, bwKHz float64, cr int) (ms float64, ok bool) { + if frameLen <= 0 || sf < minSF || sf > maxSF || bwKHz <= 0 || cr < minCR || cr > maxCR { + return 0, false + } + tSym := math.Exp2(float64(sf)) / bwKHz + de := 0 + if tSym >= 16 { + de = 1 + } + n := float64(PreambleSymbols(sf)) + 4.25 + float64(symbols(frameLen, sf, cr, de)) + return n * tSym, true +} + +// symbols is the Semtech payload symbol count with explicit header and CRC on. +func symbols(frameLen, sf, cr, de int) int { + n := math.Ceil(float64(8*frameLen-4*sf+44)/float64(4*(sf-2*de))) * float64(cr) + if n < 0 { + n = 0 + } + return 8 + int(n) +} diff --git a/internal/lora/airtime_test.go b/internal/lora/airtime_test.go new file mode 100644 index 00000000..795357bd --- /dev/null +++ b/internal/lora/airtime_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package lora + +import ( + "math" + "testing" +) + +func TestPreambleSymbols(t *testing.T) { + cases := []struct { + sf int + want int + }{ + {7, 32}, + {8, 32}, + {9, 16}, + {12, 16}, + } + for _, tc := range cases { + if got := PreambleSymbols(tc.sf); got != tc.want { + t.Errorf("PreambleSymbols(%d) = %d, want %d", tc.sf, got, tc.want) + } + } +} + +func TestFrameLength(t *testing.T) { + cases := []struct { + name string + transportCodes bool + pathLen int + payloadLen int + want int + }{ + {"no transport codes, no path", false, 0, 10, 12}, + {"transport codes with path", true, 3, 10, 19}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := FrameLength(tc.transportCodes, tc.pathLen, tc.payloadLen); got != tc.want { + t.Errorf("FrameLength(%v, %d, %d) = %d, want %d", tc.transportCodes, tc.pathLen, tc.payloadLen, got, tc.want) + } + }) + } +} + +func TestSymbols(t *testing.T) { + cases := []struct { + name string + frameLen int + sf int + cr int + de int + want int + }{ + {"sf7 no ldro", 50, 7, 5, 0, 83}, + {"sf10 ldro", 50, 10, 5, 1, 73}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := symbols(tc.frameLen, tc.sf, tc.cr, tc.de); got != tc.want { + t.Errorf("symbols(%d, %d, %d, %d) = %d, want %d", tc.frameLen, tc.sf, tc.cr, tc.de, got, tc.want) + } + }) + } +} + +func TestTimeOnAirMs(t *testing.T) { + cases := []struct { + name string + frameLen int + sf int + bwKHz float64 + cr int + want float64 + }{ + {"sf7 bw62.5", 50, 7, 62.5, 5, 244.224}, + {"sf10 bw62.5 ldro", 50, 10, 62.5, 5, 1527.808}, + {"sf7 bw250", 50, 7, 250, 5, 61.056}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := TimeOnAirMs(tc.frameLen, tc.sf, tc.bwKHz, tc.cr) + if !ok { + t.Fatalf("TimeOnAirMs(%d, %d, %v, %d) not costable, want ok", tc.frameLen, tc.sf, tc.bwKHz, tc.cr) + } + if math.Abs(got-tc.want) > 0.001 { + t.Errorf("TimeOnAirMs(%d, %d, %v, %d) = %v, want %v", tc.frameLen, tc.sf, tc.bwKHz, tc.cr, got, tc.want) + } + }) + } +} + +func TestTimeOnAirMsNotCostable(t *testing.T) { + cases := []struct { + name string + frameLen int + sf int + bwKHz float64 + cr int + }{ + {"sf zero", 50, 0, 62.5, 5}, + {"sf below range", 50, 6, 62.5, 5}, + {"sf above range", 50, 13, 62.5, 5}, + {"bw zero", 50, 7, 0, 5}, + {"cr below range", 50, 7, 62.5, 4}, + {"cr above range", 50, 7, 62.5, 9}, + {"empty frame", 0, 7, 62.5, 5}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := TimeOnAirMs(tc.frameLen, tc.sf, tc.bwKHz, tc.cr) + if ok { + t.Errorf("TimeOnAirMs(%d, %d, %v, %d) = %v, ok; want not costable", tc.frameLen, tc.sf, tc.bwKHz, tc.cr, got) + } + if got != 0 { + t.Errorf("TimeOnAirMs(%d, %d, %v, %d) = %v, want 0 ms when not costable", tc.frameLen, tc.sf, tc.bwKHz, tc.cr, got) + } + }) + } +} From d202205659975bd63e2cb76cdc971f65704dc73f Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:17:29 -0400 Subject: [PATCH 027/141] fix(observers): rename telemetry airtime fields to seconds (#136) --- db/migrations/033_telemetry_airtime_secs.sql | 4 +++ db/observers.go | 12 +++---- db/queries/queries.sql | 10 +++--- db/sqlc/models.go | 4 +-- db/sqlc/queries.sql.go | 34 ++++++++++---------- docs/docs.go | 4 +-- docs/swagger.json | 4 +-- docs/swagger.yaml | 4 +-- internal/api/observers.go | 5 +-- 9 files changed, 43 insertions(+), 38 deletions(-) create mode 100644 db/migrations/033_telemetry_airtime_secs.sql diff --git a/db/migrations/033_telemetry_airtime_secs.sql b/db/migrations/033_telemetry_airtime_secs.sql new file mode 100644 index 00000000..3ca4d364 --- /dev/null +++ b/db/migrations/033_telemetry_airtime_secs.sql @@ -0,0 +1,4 @@ +-- These always held seconds of radio time from the observer's tx_air_secs/rx_air_secs +-- status fields, never a percentage; name them for what they are. +ALTER TABLE observer_telemetry RENAME COLUMN airtime_tx_pct TO airtime_tx_secs; +ALTER TABLE observer_telemetry RENAME COLUMN airtime_rx_pct TO airtime_rx_secs; diff --git a/db/observers.go b/db/observers.go index 1a71329a..43948c1d 100644 --- a/db/observers.go +++ b/db/observers.go @@ -158,8 +158,8 @@ func (s *Store) InsertObserverTelemetry(ctx context.Context, observerID uuid.UUI ObserverID: observerID, ReportedAt: pgtype.Timestamptz{Time: reportedAt, Valid: true}, BatteryVoltageMv: batteryMV, - AirtimeTxPct: txAirSecs, - AirtimeRxPct: rxAirSecs, + AirtimeTxSecs: txAirSecs, + AirtimeRxSecs: rxAirSecs, NoiseFloorDb: &noiseFloor, UptimeSeconds: &uptimeSeconds, QueueLength: queueLen, @@ -183,8 +183,8 @@ func (s *Store) GetObserverTelemetry(ctx context.Context, observerID uuid.UUID, points = append(points, api.ObserverTelemetryPoint{ T: v.ReportedAt.Time.UnixMilli(), BatteryMV: v.BatteryVoltageMv, - AirtimeTxPct: v.AirtimeTxPct, - AirtimeRxPct: v.AirtimeRxPct, + AirtimeTxSecs: v.AirtimeTxSecs, + AirtimeRxSecs: v.AirtimeRxSecs, NoiseFloorDB: v.NoiseFloorDb, UptimeSeconds: v.UptimeSeconds, QueueLength: v.QueueLength, @@ -216,8 +216,8 @@ func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uui points = append(points, api.ObserverTelemetryPoint{ T: r.Bucket.Time.UnixMilli(), BatteryMV: &r.BatteryVoltageMv, - AirtimeTxPct: &r.AirtimeTxPct, - AirtimeRxPct: &r.AirtimeRxPct, + AirtimeTxSecs: &r.AirtimeTxSecs, + AirtimeRxSecs: &r.AirtimeRxSecs, NoiseFloorDB: &r.NoiseFloorDb, UptimeSeconds: &r.UptimeSeconds, QueueLength: &r.QueueLength, diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 7c9a4021..2e72472e 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -226,14 +226,14 @@ WHERE id = $1; -- Inserts a telemetry snapshot for an observer. The reported_at timestamp should -- be truncated to the configured resolution before calling to ensure deduplication. INSERT INTO observer_telemetry ( - observer_id, reported_at, battery_voltage_mv, airtime_tx_pct, - airtime_rx_pct, noise_floor_db, uptime_seconds, queue_length, + observer_id, reported_at, battery_voltage_mv, airtime_tx_secs, + airtime_rx_secs, noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (observer_id, reported_at) DO NOTHING; -- name: GetObserverTelemetry :many -SELECT id, reported_at, battery_voltage_mv, airtime_tx_pct, airtime_rx_pct, +SELECT id, reported_at, battery_voltage_mv, airtime_tx_secs, airtime_rx_secs, noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors FROM observer_telemetry WHERE observer_id = $1 @@ -247,8 +247,8 @@ SELECT (date_trunc('day', reported_at) + (EXTRACT(HOUR FROM reported_at)::int / $4::int) * ($4::int * interval '1 hour'))::timestamptz AS bucket, AVG(battery_voltage_mv)::int AS battery_voltage_mv, - GREATEST(MAX(airtime_tx_pct) - MIN(airtime_tx_pct), 0)::real AS airtime_tx_pct, - GREATEST(MAX(airtime_rx_pct) - MIN(airtime_rx_pct), 0)::real AS airtime_rx_pct, + GREATEST(MAX(airtime_tx_secs) - MIN(airtime_tx_secs), 0)::real AS airtime_tx_secs, + GREATEST(MAX(airtime_rx_secs) - MIN(airtime_rx_secs), 0)::real AS airtime_rx_secs, AVG(noise_floor_db)::real AS noise_floor_db, MAX(uptime_seconds)::bigint AS uptime_seconds, AVG(queue_length)::int AS queue_length, diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 9f6ef210..a2a5432d 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -258,8 +258,8 @@ type ObserverTelemetry struct { ObserverID uuid.UUID `json:"observer_id"` ReportedAt pgtype.Timestamptz `json:"reported_at"` BatteryVoltageMv *int32 `json:"battery_voltage_mv"` - AirtimeTxPct *float32 `json:"airtime_tx_pct"` - AirtimeRxPct *float32 `json:"airtime_rx_pct"` + AirtimeTxSecs *float32 `json:"airtime_tx_secs"` + AirtimeRxSecs *float32 `json:"airtime_rx_secs"` NoiseFloorDb *float32 `json:"noise_floor_db"` UptimeSeconds *int64 `json:"uptime_seconds"` QueueLength *int32 `json:"queue_length"` diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 45641d45..998889f2 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -891,7 +891,7 @@ func (q *Queries) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ( } const getObserverTelemetry = `-- name: GetObserverTelemetry :many -SELECT id, reported_at, battery_voltage_mv, airtime_tx_pct, airtime_rx_pct, +SELECT id, reported_at, battery_voltage_mv, airtime_tx_secs, airtime_rx_secs, noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors FROM observer_telemetry WHERE observer_id = $1 @@ -912,8 +912,8 @@ type GetObserverTelemetryRow struct { ID int64 `json:"id"` ReportedAt pgtype.Timestamptz `json:"reported_at"` BatteryVoltageMv *int32 `json:"battery_voltage_mv"` - AirtimeTxPct *float32 `json:"airtime_tx_pct"` - AirtimeRxPct *float32 `json:"airtime_rx_pct"` + AirtimeTxSecs *float32 `json:"airtime_tx_secs"` + AirtimeRxSecs *float32 `json:"airtime_rx_secs"` NoiseFloorDb *float32 `json:"noise_floor_db"` UptimeSeconds *int64 `json:"uptime_seconds"` QueueLength *int32 `json:"queue_length"` @@ -939,8 +939,8 @@ func (q *Queries) GetObserverTelemetry(ctx context.Context, arg GetObserverTelem &i.ID, &i.ReportedAt, &i.BatteryVoltageMv, - &i.AirtimeTxPct, - &i.AirtimeRxPct, + &i.AirtimeTxSecs, + &i.AirtimeRxSecs, &i.NoiseFloorDb, &i.UptimeSeconds, &i.QueueLength, @@ -962,8 +962,8 @@ SELECT (date_trunc('day', reported_at) + (EXTRACT(HOUR FROM reported_at)::int / $4::int) * ($4::int * interval '1 hour'))::timestamptz AS bucket, AVG(battery_voltage_mv)::int AS battery_voltage_mv, - GREATEST(MAX(airtime_tx_pct) - MIN(airtime_tx_pct), 0)::real AS airtime_tx_pct, - GREATEST(MAX(airtime_rx_pct) - MIN(airtime_rx_pct), 0)::real AS airtime_rx_pct, + GREATEST(MAX(airtime_tx_secs) - MIN(airtime_tx_secs), 0)::real AS airtime_tx_secs, + GREATEST(MAX(airtime_rx_secs) - MIN(airtime_rx_secs), 0)::real AS airtime_rx_secs, AVG(noise_floor_db)::real AS noise_floor_db, MAX(uptime_seconds)::bigint AS uptime_seconds, AVG(queue_length)::int AS queue_length, @@ -986,8 +986,8 @@ type GetObserverTelemetryBucketedParams struct { type GetObserverTelemetryBucketedRow struct { Bucket pgtype.Timestamptz `json:"bucket"` BatteryVoltageMv int32 `json:"battery_voltage_mv"` - AirtimeTxPct float32 `json:"airtime_tx_pct"` - AirtimeRxPct float32 `json:"airtime_rx_pct"` + AirtimeTxSecs float32 `json:"airtime_tx_secs"` + AirtimeRxSecs float32 `json:"airtime_rx_secs"` NoiseFloorDb float32 `json:"noise_floor_db"` UptimeSeconds int64 `json:"uptime_seconds"` QueueLength int32 `json:"queue_length"` @@ -1011,8 +1011,8 @@ func (q *Queries) GetObserverTelemetryBucketed(ctx context.Context, arg GetObser if err := rows.Scan( &i.Bucket, &i.BatteryVoltageMv, - &i.AirtimeTxPct, - &i.AirtimeRxPct, + &i.AirtimeTxSecs, + &i.AirtimeRxSecs, &i.NoiseFloorDb, &i.UptimeSeconds, &i.QueueLength, @@ -2016,8 +2016,8 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa const insertObserverTelemetry = `-- name: InsertObserverTelemetry :exec INSERT INTO observer_telemetry ( - observer_id, reported_at, battery_voltage_mv, airtime_tx_pct, - airtime_rx_pct, noise_floor_db, uptime_seconds, queue_length, + observer_id, reported_at, battery_voltage_mv, airtime_tx_secs, + airtime_rx_secs, noise_floor_db, uptime_seconds, queue_length, debug_flags, receive_errors ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (observer_id, reported_at) DO NOTHING @@ -2027,8 +2027,8 @@ type InsertObserverTelemetryParams struct { ObserverID uuid.UUID `json:"observer_id"` ReportedAt pgtype.Timestamptz `json:"reported_at"` BatteryVoltageMv *int32 `json:"battery_voltage_mv"` - AirtimeTxPct *float32 `json:"airtime_tx_pct"` - AirtimeRxPct *float32 `json:"airtime_rx_pct"` + AirtimeTxSecs *float32 `json:"airtime_tx_secs"` + AirtimeRxSecs *float32 `json:"airtime_rx_secs"` NoiseFloorDb *float32 `json:"noise_floor_db"` UptimeSeconds *int64 `json:"uptime_seconds"` QueueLength *int32 `json:"queue_length"` @@ -2043,8 +2043,8 @@ func (q *Queries) InsertObserverTelemetry(ctx context.Context, arg InsertObserve arg.ObserverID, arg.ReportedAt, arg.BatteryVoltageMv, - arg.AirtimeTxPct, - arg.AirtimeRxPct, + arg.AirtimeTxSecs, + arg.AirtimeRxSecs, arg.NoiseFloorDb, arg.UptimeSeconds, arg.QueueLength, diff --git a/docs/docs.go b/docs/docs.go index 9d560071..b7cf2acd 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -3204,10 +3204,10 @@ const docTemplate = `{ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetryPoint": { "type": "object", "properties": { - "airtimeRxPct": { + "airtimeRxSecs": { "type": "number" }, - "airtimeTxPct": { + "airtimeTxSecs": { "type": "number" }, "batteryMv": { diff --git a/docs/swagger.json b/docs/swagger.json index ede57845..5182f729 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -3202,10 +3202,10 @@ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetryPoint": { "type": "object", "properties": { - "airtimeRxPct": { + "airtimeRxSecs": { "type": "number" }, - "airtimeTxPct": { + "airtimeTxSecs": { "type": "number" }, "batteryMv": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 7efd8ad1..64962a77 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -576,9 +576,9 @@ definitions: type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverTelemetryPoint: properties: - airtimeRxPct: + airtimeRxSecs: type: number - airtimeTxPct: + airtimeTxSecs: type: number batteryMv: type: integer diff --git a/internal/api/observers.go b/internal/api/observers.go index afe800f1..206387b7 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -49,11 +49,12 @@ type Observer struct { } // ObserverTelemetryPoint is a single telemetry snapshot for an observer. +// Airtime is radio seconds: cumulative since boot on 1h points, per-bucket delta on 6h/24h. type ObserverTelemetryPoint struct { T int64 `json:"t"` // epoch ms BatteryMV *int32 `json:"batteryMv,omitempty"` - AirtimeTxPct *float32 `json:"airtimeTxPct,omitempty"` - AirtimeRxPct *float32 `json:"airtimeRxPct,omitempty"` + AirtimeTxSecs *float32 `json:"airtimeTxSecs,omitempty"` + AirtimeRxSecs *float32 `json:"airtimeRxSecs,omitempty"` NoiseFloorDB *float32 `json:"noiseFloorDb,omitempty"` UptimeSeconds *int64 `json:"uptimeSeconds,omitempty"` QueueLength *int32 `json:"queueLength,omitempty"` From e15f8735978d18066c24cea93f818df0159a9c78 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:55:52 -0400 Subject: [PATCH 028/141] fix(api): return JSON 500 when bucketed telemetry query fails (#137) --- internal/api/handlers/observers.go | 3 ++- internal/api/handlers/observers_test.go | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 01491b5f..43147b5a 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -222,7 +222,8 @@ func getObserverTelemetry(reader api.Reader) http.HandlerFunc { if bucketHours == 0 { telemetry, err = reader.GetObserverTelemetry(r.Context(), observerID, since, until, afterID) } else { - points, err := reader.GetObserverTelemetryBucketed(r.Context(), observerID, since, until, bucketHours) + var points []api.ObserverTelemetryPoint + points, err = reader.GetObserverTelemetryBucketed(r.Context(), observerID, since, until, bucketHours) if err == nil { telemetry = &api.ObserverTelemetry{Points: points} } diff --git a/internal/api/handlers/observers_test.go b/internal/api/handlers/observers_test.go index 0d4a028c..1bf865fa 100644 --- a/internal/api/handlers/observers_test.go +++ b/internal/api/handlers/observers_test.go @@ -145,6 +145,29 @@ func TestGetObserverTelemetry_Bucketed_OK(t *testing.T) { } } +func TestGetObserverTelemetry_Bucketed_StoreError(t *testing.T) { + observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + r := chi.NewRouter() + r.Get("/observers/{observerId}/telemetry", getObserverTelemetry(stubReader{ + getObserverTelemetryBucketed: func(_ context.Context, _ uuid.UUID, _, _ time.Time, _ int32) ([]api.ObserverTelemetryPoint, error) { + return nil, errors.New("boom") + }, + })) + req := httptest.NewRequest(http.MethodGet, "/observers/"+observerID.String()+"/telemetry?interval=6h", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d", w.Code) + } + var body map[string]APIError + if err := json.NewDecoder(w.Body).Decode(&body); err != nil { + t.Fatalf("expected JSON error body, got %q: %v", w.Body.String(), err) + } + if body["error"].Code != "internal_server_error" { + t.Errorf("expected internal_server_error, got %q", body["error"].Code) + } +} + func TestListObserverAdverts_OK(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() From b44769bef84eceabe723f213d2c99ebcb15b4a2c Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 18 Sep 2026 06:17:33 -0400 Subject: [PATCH 029/141] fix(observers): preserve advert pages with missing origin keys (#138) * test(observers): cover adverts with a missing origin key * fix(observers): preserve advert pages with missing origin keys --- db/observer_adverts_integration_test.go | 85 +++++++++++++++++++++++++ db/queries/queries.sql | 3 +- db/sqlc/querier.go | 1 + db/sqlc/queries.sql.go | 3 +- 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 db/observer_adverts_integration_test.go diff --git a/db/observer_adverts_integration_test.go b/db/observer_adverts_integration_test.go new file mode 100644 index 00000000..2531e37d --- /dev/null +++ b/db/observer_adverts_integration_test.go @@ -0,0 +1,85 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/jackc/pgx/v5" +) + +func TestObserverAdvertsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + _, err = tx.Exec(ctx, ` +CREATE TEMP TABLE packets (LIKE public.packets INCLUDING ALL) ON COMMIT DROP; +CREATE TEMP TABLE packet_observations (LIKE public.packet_observations INCLUDING ALL) ON COMMIT DROP; +CREATE TEMP TABLE nodes (LIKE public.nodes INCLUDING ALL) ON COMMIT DROP; +INSERT INTO nodes(public_key,node_type,name) VALUES (decode(repeat('01',32),'hex'),1,'Known fixture node'); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,origin_pubkey,raw_payload,raw_header,first_heard_at,last_heard_at) VALUES + (decode(repeat('a1',32),'hex'),4,0,1,NULL,'\x00','\x00',now(),now()), + (decode(repeat('b2',32),'hex'),4,0,1,decode(repeat('01',32),'hex'),'\x00','\x00',now(),now()); +INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES + (1,decode(repeat('a1',32),'hex'),'00000000-0000-0000-0000-000000000106','YOW',now(),0,1,0), + (2,decode(repeat('b2',32),'hex'),'00000000-0000-0000-0000-000000000106','YOW',now(),0,1,0);`) + if err != nil { + t.Fatal(err) + } + router := handlers.ObserversRouter(&Store{q: sqlc.New(tx)}) + request := func(query string) api.Page[api.AdvertObservation] { + t.Helper() + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/00000000-0000-0000-0000-000000000106/adverts?"+query, nil).WithContext(ctx)) + if response.Code != http.StatusOK { + t.Fatalf("advert page HTTP %d", response.Code) + } + var page api.Page[api.AdvertObservation] + if err := json.Unmarshal(response.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + return page + } + page := request("limit=50") + if len(page.Items) != 2 || page.Items[0].ID != 1 || page.Items[1].ID != 2 { + t.Fatal("missing-origin advert was not preserved in the ordered page") + } + if key := page.Items[0].NodePublicKey; key == nil || *key != "" { + t.Errorf("missing origin key = %v, want an empty string", key) + } + if key := page.Items[1].NodePublicKey; key == nil || *key != strings.Repeat("01", 32) { + t.Error("known origin key changed") + } + first := request("limit=1") + if !first.HasMore || first.NextCursor == nil || *first.NextCursor != 1 { + t.Fatal("missing-origin advert broke the cursor") + } + last := request("limit=1&cursor=1") + if len(last.Items) != 1 || last.Items[0].ID != 2 || last.HasMore { + t.Fatal("continuing after the missing-origin advert lost the known advert") + } +} diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 2e72472e..3bd18618 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -313,6 +313,7 @@ ORDER BY count DESC; -- name: ListObserverAdverts :many -- Returns advert packets (payload_type=4) heard by a specific observer. -- Pass cursor=0 to start from the beginning, or the last seen id for pagination. +-- Keep missing-origin adverts; the generated key field expects a string, not NULL. SELECT po.id, encode(po.packet_hash, 'hex') AS packet_hash_hex, @@ -323,7 +324,7 @@ SELECT po.snr, po.hop_count, n.name AS node_name, - encode(p.origin_pubkey, 'hex') AS node_public_key + COALESCE(encode(p.origin_pubkey, 'hex'), '')::text AS node_public_key FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash LEFT JOIN nodes n ON n.public_key = p.origin_pubkey diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 7a64432e..ec502af7 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -151,6 +151,7 @@ type Querier interface { ListObservationsForPacket(ctx context.Context, packetHash []byte) ([]ListObservationsForPacketRow, error) // Returns advert packets (payload_type=4) heard by a specific observer. // Pass cursor=0 to start from the beginning, or the last seen id for pagination. + // Keep missing-origin adverts; the generated key field expects a string, not NULL. ListObserverAdverts(ctx context.Context, arg ListObserverAdvertsParams) ([]ListObserverAdvertsRow, error) // Pass cursor=0 to start from the beginning, or the last seen observer's rownum for pagination. // Note: observers use UUID PKs so we order by last_seen and use a keyset on last_seen+id. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 998889f2..fec385d6 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2860,7 +2860,7 @@ SELECT po.snr, po.hop_count, n.name AS node_name, - encode(p.origin_pubkey, 'hex') AS node_public_key + COALESCE(encode(p.origin_pubkey, 'hex'), '')::text AS node_public_key FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash LEFT JOIN nodes n ON n.public_key = p.origin_pubkey @@ -2892,6 +2892,7 @@ type ListObserverAdvertsRow struct { // Returns advert packets (payload_type=4) heard by a specific observer. // Pass cursor=0 to start from the beginning, or the last seen id for pagination. +// Keep missing-origin adverts; the generated key field expects a string, not NULL. func (q *Queries) ListObserverAdverts(ctx context.Context, arg ListObserverAdvertsParams) ([]ListObserverAdvertsRow, error) { rows, err := q.db.Query(ctx, listObserverAdverts, arg.ObserverID, arg.Column2, arg.Limit) if err != nil { From ce8c6cdf8bd6481afefa50f021fcb04ee3a76e69 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 18 Sep 2026 06:17:44 -0400 Subject: [PATCH 030/141] fix(ingest): honor explicit zero locations in adverts (#139) --- db/node_location_integration_test.go | 71 ++++++++++++++++++++++++++++ internal/ingest/ingest_test.go | 4 +- internal/ingest/side_effects.go | 3 +- internal/ingest/side_effects_test.go | 55 ++++++++++++++++++++- 4 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 db/node_location_integration_test.go diff --git a/db/node_location_integration_test.go b/db/node_location_integration_test.go new file mode 100644 index 00000000..dd060011 --- /dev/null +++ b/db/node_location_integration_test.go @@ -0,0 +1,71 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" +) + +func TestNodeLocationResetPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE nodes (LIKE public.nodes INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + lat, lon := 45.0, -75.0 + params := ingest.UpsertNodeParams{PublicKey: []byte{0x97}, NodeType: 2, Name: "location fixture", Latitude: &lat, Longitude: &lon} + id, err := store.UpsertNode(ctx, params, ingest.RadioSettings{}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + present bool + lat, lon float64 + }{ + {"omission preserves location", false, 45, -75}, + {"explicit reset replaces location", true, 0, 0}, + {"omission preserves reset", false, 0, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + params.Latitude, params.Longitude = nil, nil + if tc.present { + params.Latitude, params.Longitude = &tc.lat, &tc.lon + } + updatedID, err := store.UpsertNode(ctx, params, ingest.RadioSettings{}) + if err != nil || updatedID != id { + t.Fatalf("node identity changed: %v", err) + } + var gotLat, gotLon float64 + if err := tx.QueryRow(ctx, "SELECT latitude,longitude FROM nodes WHERE id=$1", id).Scan(&gotLat, &gotLon); err != nil { + t.Fatal(err) + } + if gotLat != tc.lat || gotLon != tc.lon { + t.Fatalf("location = (%g,%g), want (%g,%g)", gotLat, gotLon, tc.lat, tc.lon) + } + }) + } +} diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 94bd6fdf..47830116 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -175,6 +175,7 @@ type stubDB struct { setCapabilityCalls []setCapabilityCall upsertNodeCalls int + upsertNodeParams UpsertNodeParams upsertChannelCalls int upsertChannelHashOnlyCalls int upsertChannelIATACalls int @@ -209,8 +210,9 @@ func (s *stubDB) InsertObservation(_ context.Context, _ InsertObservationParams) return s.observationInserted, nil } func (s *stubDB) SetNodeDefaultScope(_ context.Context, _ uuid.UUID, _ int32) error { return nil } -func (s *stubDB) UpsertNode(_ context.Context, _ UpsertNodeParams, _ RadioSettings) (uuid.UUID, error) { +func (s *stubDB) UpsertNode(_ context.Context, params UpsertNodeParams, _ RadioSettings) (uuid.UUID, error) { s.upsertNodeCalls++ + s.upsertNodeParams = params return uuid.Nil, nil } func (s *stubDB) UpsertNodeIATA(_ context.Context, _ uuid.UUID, _ string) error { return nil } diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index e802f0c2..f465c99d 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -80,7 +80,8 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc return } var lat, lon *float64 - if advert.AppData().Lat != 0 || advert.AppData().Lon != 0 { + // Presence, not value: an explicit 0/0 advert resets the stored location. + if advert.Flags()&meshcore.AdvertLatLonMask != 0 { la := float64(advert.AppData().Lat) / 1e6 lo := float64(advert.AppData().Lon) / 1e6 lat = &la diff --git a/internal/ingest/side_effects_test.go b/internal/ingest/side_effects_test.go index 25bdb2d3..782b79f8 100644 --- a/internal/ingest/side_effects_test.go +++ b/internal/ingest/side_effects_test.go @@ -6,6 +6,7 @@ package ingest import ( "context" "crypto/ed25519" + "encoding/binary" "encoding/hex" "encoding/json" "testing" @@ -31,6 +32,10 @@ func (k *mapKeys) GetKey(hash []byte) []keystore.Entry { // buildAdvertPacket signs (or, if tamper is true, signs then mutates) an // advert payload and wraps it in a minimal Packet with no path (zero-hop). func buildAdvertPacket(t *testing.T, tamper bool) *meshcore.Packet { + return buildAdvertPacketWithData(t, []byte{meshcore.AdvertTypeRepeater}, tamper) +} + +func buildAdvertPacketWithData(t *testing.T, data []byte, tamper bool) *meshcore.Packet { t.Helper() pub, priv, err := ed25519.GenerateKey(nil) if err != nil { @@ -43,7 +48,7 @@ func buildAdvertPacket(t *testing.T, tamper bool) *meshcore.Packet { advert := &meshcore.Advert{ PublicKey: id, Timestamp: 12345, - RawAppData: []byte{meshcore.AdvertTypeRepeater}, // flags byte only, no optional fields + RawAppData: data, } advert.Sign(priv) if tamper { @@ -61,6 +66,54 @@ func buildAdvertPacket(t *testing.T, tamper bool) *meshcore.Packet { } } +func TestAdvertLocationPresence(t *testing.T) { + for _, tc := range []struct { + name string + present, tamper bool + lat, lon int32 + }{ + {"ordinary location", true, false, 45000000, -75000000}, + {"explicit reset", true, false, 0, 0}, + {"zero latitude", true, false, 0, -75000000}, + {"zero longitude", true, false, 45000000, 0}, + {"no location", false, false, 0, 0}, + {"tampered reset", true, true, 0, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + data := []byte{meshcore.AdvertTypeRepeater} + if tc.present { + // Preserve the wire presence bit even for zero coordinates; the + // library's AppData encoder omits location when both values are zero. + data[0] |= meshcore.AdvertLatLonMask + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lat)) + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lon)) + } + worker, store := newTestWorker() + packet := buildAdvertPacketWithData(t, data, tc.tamper) + worker.handlePayloadTypeSideEffects(context.Background(), packet, "YOW", []byte{1}, RadioSettings{}, nil, nil, nil, 0) + if tc.tamper { + if store.upsertNodeCalls != 0 { + t.Fatal("invalid signature updated the node") + } + return + } + if store.upsertNodeCalls != 1 { + t.Fatal("signed advert did not update the node") + } + got := store.upsertNodeParams + if !tc.present { + if got.Latitude != nil || got.Longitude != nil { + t.Fatal("absent location became an update") + } + return + } + if got.Latitude == nil || got.Longitude == nil || *got.Latitude != float64(tc.lat)/1e6 || *got.Longitude != float64(tc.lon)/1e6 { + t.Fatal("advertised coordinates did not reach the node update") + } + }) + } +} + func TestHandlePayloadTypeSideEffects_Advert_ValidSignature_UpsertsNode(t *testing.T) { w, db := newTestWorker() packet := buildAdvertPacket(t, false) From 9164659544260fb873a45749f0a2a99336936e58 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 18 Sep 2026 06:17:52 -0400 Subject: [PATCH 031/141] feat(backup): export private database and saved-config bundles (#151) * feat(backup): export bounded database and saved-config bundles * test(backup): preserve synthetic channel keys in exports * fix(backup): check writable file closure before publication --- .github/workflows/ci.yml | 25 ++- README.md | 4 + cmd/beacon-backup/main.go | 39 ++++ docs/backup-export.md | 89 ++++++++ internal/backup/export.go | 242 +++++++++++++++++++++ internal/backup/export_integration_test.go | 216 ++++++++++++++++++ internal/backup/export_test.go | 232 ++++++++++++++++++++ 7 files changed, 846 insertions(+), 1 deletion(-) create mode 100644 cmd/beacon-backup/main.go create mode 100644 docs/backup-export.md create mode 100644 internal/backup/export.go create mode 100644 internal/backup/export_integration_test.go create mode 100644 internal/backup/export_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdb2a3b7..0972374f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,18 @@ on: jobs: build: runs-on: ubuntu-latest + services: + backup-postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: backup-ci-only + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 steps: - uses: actions/checkout@v4 @@ -35,6 +47,18 @@ jobs: - name: Test run: go test ./... + - name: Verify backup command against PostgreSQL 16 + run: | + CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup + CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup.test" ./internal/backup + docker run --rm --network host \ + -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ + -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable \ + -e BEACON_BACKUP_TEST_POSTGRES=1 -e BEACON_BACKUP_TEST_BINARY=/beacon-backup \ + -v "$RUNNER_TEMP/beacon-backup:/beacon-backup:ro" \ + -v "$RUNNER_TEMP/backup.test:/backup.test:ro" \ + postgres:16-alpine /backup.test -test.run '^TestExportPostgres$' -test.v + - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest @@ -54,4 +78,3 @@ jobs: exit 1 fi rm docs/swagger.json.bak - diff --git a/README.md b/README.md index ef3026db..7fd773f8 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ in PostgreSQL, and streams live events to WebSocket clients. For deployment instructions including the frontend app, see the deployment docs. +For a bounded private database and saved-config bundle, see +[backup export](docs/backup-export.md). This is a standalone export tool; the +backup web interface and import workflow are separate follow-ups. + --- ## Stack diff --git a/cmd/beacon-backup/main.go b/cmd/beacon-backup/main.go new file mode 100644 index 00000000..7e593e37 --- /dev/null +++ b/cmd/beacon-backup/main.go @@ -0,0 +1,39 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// beacon-backup creates a private database and saved-config export. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" +) + +var version = "dev" + +func main() { + var opts backup.Options + flag.StringVar(&opts.ConfigPath, "config", "config.yaml", "saved YAML file to include verbatim (may contain secrets)") + flag.StringVar(&opts.OutputPath, "output", "", "new private .tar.gz destination (required; never overwritten)") + flag.Int64Var(&opts.MaxBytes, "max-bytes", backup.DefaultMaxBytes, "maximum uncompressed database dump size") + flag.DurationVar(&opts.Timeout, "timeout", backup.DefaultTimeout, "maximum export duration") + flag.Parse() + if flag.NArg() != 0 { + fmt.Fprintln(os.Stderr, "unexpected positional arguments; connection settings use PG* environment variables") + os.Exit(2) + } + opts.Version = version + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := backup.Export(ctx, opts); err != nil { + fmt.Fprintln(os.Stderr, "backup failed:", err) + os.Exit(1) + } + fmt.Println("Backup complete. Store this bundle privately; it may contain keys and message data.") +} diff --git a/docs/backup-export.md b/docs/backup-export.md new file mode 100644 index 00000000..09afb501 --- /dev/null +++ b/docs/backup-export.md @@ -0,0 +1,89 @@ +# Database and saved-config export + +`beacon-backup` is the export foundation for issue #72. It produces a private, +versioned `.tar.gz` using PostgreSQL's `pg_dump`. It does not yet provide a web +interface, account login, scheduled/remote storage or automatic import. + +Build it with `go build ./cmd/beacon-backup`. Install `pg_dump` in the same runtime +as this command; an installation on the Docker host does not install it inside an +app container. Use a client of the same major version as the source PostgreSQL +server; an older client cannot dump a newer server. + +Set libpq's standard `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER` and TLS settings. +`PGDATABASE` is required and must be a plain database name. Prefer a private +`PGPASSFILE` (0600 on Unix) or an existing libpq service configuration for secrets. +The command does not load `.env`, read `POSTGRES_DSN`, start Beacon, run migrations, +subscribe to MQTT or connect to Redis. Connection settings are not command-line +arguments and client stderr is not printed because it can contain private data. + +For example, after configuring those connection settings: + +```sh +beacon-backup -config /private/config.yaml -output /private/beacon-20260913.tar.gz +``` + +Use an output directory controlled by the operator. Staging directories are +0700 and files are 0600 on Unix; Windows operators must use a directory with +appropriately restricted ACLs. Existing destinations, including symlinks, are +never replaced. A hard link publishes the finished archive atomically, so the +destination filesystem must support hard links. Unsupported filesystems fail +without publishing an output. Normal failure, timeout and handled interruption +remove temporary files; a power loss or SIGKILL can leave a private +`.beacon-backup-*` staging directory for the operator to inspect. + +## Format 1 + +The tar contains exactly three regular files with fixed names: + +- `manifest.json`: format version, creation time, exporter version, dump format, + uncompressed payload sizes/SHA-256 hashes and explicit exclusions. +- `database.sql`: one consistent `pg_dump` snapshot of schema and data, including + Beacon's migration journal. Ownership, ACLs and tablespace placement are omitted + so objects can be restored under the destination operator. +- `config.yaml`: the supplied saved file, byte-for-byte, including comments and + any keys. It is captured before the database snapshot; avoid configuration edits + during export if the files must describe the same deployment state. + +This is sensitive, unencrypted data. Store and transfer it privately. The bundle +does not include deployment environment variables, `.env`, external files such as +`borderFile` inputs or TLS keys, runtime-only changes, PostgreSQL roles/cluster +settings, Redis or service/deployment files. Retain those separately. A config +export is not the sanitized admin-config response and is not a complete server +recovery package by itself. + +Defaults are ten minutes and a 1 GiB uncompressed SQL limit. Saved YAML is capped +at 1 MiB. `-timeout` and `-max-bytes` set finite positive limits (SQL maximum 1 TiB). +Allow disk space for both the uncompressed SQL and compressed bundle, roughly +twice the chosen SQL limit plus overhead. A five-second lock-wait limit prevents +waiting indefinitely behind schema changes. Export failure publishes no backup; +check the client version, privileges, connection settings and available capacity +privately. The command deliberately does not expose raw client diagnostics. + +## Restore verification + +Use trusted bundles only: PostgreSQL dumps can contain executable SQL. Inspect +the fixed members, verify the gzip stream and the manifest's payload sizes and +hashes, and extract into private staging. For a **new, empty disposable database**, +with its own explicit `PGDATABASE` and target-role connection settings: + +```sh +psql -X --set ON_ERROR_STOP=on --single-transaction --file database.sql +``` + +Restore with a compatible PostgreSQL version and the required extensions already +available. Reconcile the migration journal and representative records/relationships +before relying on the bundle. Review saved configuration and restore external +secrets/files separately before starting a new Beacon server. Never restore over +live data simply to test an export; overwrite/import requires a separate workflow. + +PostgreSQL references: [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.html), +[connection environment](https://www.postgresql.org/docs/16/libpq-envars.html), +[password files](https://www.postgresql.org/docs/16/libpq-pgpass.html). + +CI runs the compiled command and its PostgreSQL round-trip test with a dedicated +PostgreSQL 16 service. To repeat it privately, build the command, set the `PG*` +connection variables for an isolated test server and set +`BEACON_BACKUP_TEST_POSTGRES=1` plus `BEACON_BACKUP_TEST_BINARY` to the command's +absolute path. Run `go test ./internal/backup -run '^TestExportPostgres$' -v`. +The test role needs permission to create/drop its two randomly named databases; +the test migrates and restores only those databases and removes them afterward. diff --git a/internal/backup/export.go b/internal/backup/export.go new file mode 100644 index 00000000..63abf68c --- /dev/null +++ b/internal/backup/export.go @@ -0,0 +1,242 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package backup exports a database and saved configuration into a private bundle. +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "time" +) + +const ( + DefaultMaxBytes = int64(1 << 30) + DefaultTimeout = 10 * time.Minute + maxConfigBytes = 1 << 20 +) + +var ErrTooLarge = errors.New("backup size limit exceeded") + +// Options selects local files and finite resource limits. Connection settings +// come from libpq's PG* environment; PGDATABASE must explicitly name the database. +type Options struct { + ConfigPath string + OutputPath string + MaxBytes int64 // Maximum uncompressed database dump size. + Timeout time.Duration + Version string +} + +// Manifest describes format 1; hashes cover the uncompressed archive members. +type Manifest struct { + FormatVersion int `json:"format_version"` + CreatedAt time.Time `json:"created_at"` + ToolVersion string `json:"tool_version"` + DatabaseFormat string `json:"database_format"` + Files []File `json:"files"` + Excluded []string `json:"excluded"` +} + +// File identifies one payload by its fixed archive name, size and SHA-256. +type File struct { + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` +} + +// Export creates a complete .tar.gz without replacing an existing destination. +// pg_dump must be installed on PATH. The destination filesystem must support +// hard links; a link publishes the finished private file without an overwrite race. +// The bundle contains secrets if they are present in the database or saved YAML. +func Export(ctx context.Context, opts Options) error { + return export(ctx, opts, dumpCommand) +} + +func dumpCommand(ctx context.Context) *exec.Cmd { + return exec.CommandContext(ctx, "pg_dump", "--format=plain", "--no-owner", + "--no-acl", "--no-tablespaces", "--no-password", "--lock-wait-timeout=5000") +} + +func export(ctx context.Context, opts Options, command func(context.Context) *exec.Cmd) (exportErr error) { + if opts.ConfigPath == "" || opts.OutputPath == "" || opts.MaxBytes <= 0 || opts.MaxBytes > 1<<40 || opts.Timeout <= 0 { + return errors.New("config, output, positive timeout and max-bytes (at most 1 TiB) are required") + } + if os.Getenv("PGDATABASE") == "" { + return errors.New("PGDATABASE must explicitly name the database; POSTGRES_DSN is not used") + } + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) + defer cancel() + if err := ctx.Err(); err != nil { + return err + } + if _, err := os.Lstat(opts.OutputPath); !errors.Is(err, os.ErrNotExist) { + return errors.New("output already exists or cannot be inspected") + } + config, err := readConfig(opts.ConfigPath) + if err != nil { + return err + } + parent, err := filepath.Abs(filepath.Dir(opts.OutputPath)) + if err != nil { + return errors.New("invalid output directory") + } + temp, err := os.MkdirTemp(parent, ".beacon-backup-") + if err != nil { + return errors.New("cannot create private backup staging directory") + } + defer os.RemoveAll(temp) + dump, err := os.OpenFile(filepath.Join(temp, "database.sql"), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return errors.New("cannot create private database dump") + } + defer closeBackupFile(dump, &exportErr) + hash := sha256.New() + output := &limitedWriter{ctx: ctx, cancel: cancel, dst: io.MultiWriter(dump, hash), remaining: opts.MaxBytes} + cmd := command(ctx) + cmd.Stdout, cmd.Stderr = output, io.Discard + cmd.WaitDelay = time.Second + err = cmd.Run() + if output.err != nil { + return output.err + } + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil { + // Client diagnostics may contain credentials, hostnames or database contents. + return errors.New("pg_dump failed; check the installed client and private connection settings") + } + size := opts.MaxBytes - output.remaining + if size == 0 { + return errors.New("pg_dump produced an empty dump") + } + if _, err = dump.Seek(0, io.SeekStart); err != nil { + return errors.New("cannot read completed database dump") + } + configHash := sha256.Sum256(config) + manifest := Manifest{ + FormatVersion: 1, CreatedAt: time.Now().UTC(), ToolVersion: opts.Version, + DatabaseFormat: "postgresql-plain-sql", + Files: []File{ + {Name: "database.sql", Size: size, SHA256: hex.EncodeToString(hash.Sum(nil))}, + {Name: "config.yaml", Size: int64(len(config)), SHA256: hex.EncodeToString(configHash[:])}, + }, + Excluded: []string{"deployment_environment", "external_config_files", "runtime_only_changes", "roles_ownership_acl_tablespaces", "redis_cache"}, + } + metadata, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return errors.New("cannot encode backup manifest") + } + archivePath := filepath.Join(temp, "bundle.tar.gz") + archive, err := os.OpenFile(archivePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return errors.New("cannot create private backup archive") + } + defer closeBackupFile(archive, &exportErr) + // Input is bounded; reserve room for YAML, tar headers and compression overhead. + archiveOutput := &limitedWriter{ctx: ctx, cancel: cancel, dst: archive, remaining: opts.MaxBytes + opts.MaxBytes/100 + 2*maxConfigBytes} + gz := gzip.NewWriter(archiveOutput) + tw := tar.NewWriter(gz) + for _, member := range []struct { + name string + size int64 + data io.Reader + }{ + {"manifest.json", int64(len(metadata)), bytes.NewReader(metadata)}, + {"database.sql", size, dump}, + {"config.yaml", int64(len(config)), bytes.NewReader(config)}, + } { + if err = tw.WriteHeader(&tar.Header{Name: member.name, Size: member.size, Mode: 0600, ModTime: manifest.CreatedAt}); err == nil { + _, err = io.Copy(tw, member.data) + } + if err != nil { + return errors.New("cannot write complete backup archive") + } + } + if err = dump.Close(); err != nil { + return errors.New("cannot close completed database dump") + } + if err = tw.Close(); err != nil { + return errors.New("cannot complete backup tar") + } + if err = gz.Close(); err != nil { + return errors.New("cannot complete backup compression") + } + if err = archive.Sync(); err != nil { + return errors.New("cannot sync completed backup") + } + if err = archive.Close(); err != nil { + return errors.New("cannot close completed backup") + } + if err = ctx.Err(); err != nil { + return err + } + if err = os.Link(archivePath, opts.OutputPath); err != nil { + return errors.New("cannot publish backup without overwriting; check destination and hard-link support") + } + return nil +} + +// Cleanup also checks close errors. Both writable files are explicitly closed +// before publication; a deferred second close may therefore report ErrClosed. +func closeBackupFile(file *os.File, exportErr *error) { + if err := file.Close(); err != nil && !errors.Is(err, os.ErrClosed) && *exportErr == nil { + *exportErr = errors.New("cannot close private backup file") + } +} + +func readConfig(path string) ([]byte, error) { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Size() > maxConfigBytes { + return nil, errors.New("config must be a readable regular file of at most 1 MiB") + } + f, err := os.Open(path) + if err != nil { + return nil, errors.New("cannot read saved config") + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, maxConfigBytes+1)) + if err != nil || len(data) > maxConfigBytes { + return nil, errors.New("cannot read saved config within the 1 MiB limit") + } + return data, nil +} + +type limitedWriter struct { + ctx context.Context + cancel context.CancelFunc + dst io.Writer + remaining int64 + err error +} + +func (w *limitedWriter) Write(p []byte) (n int, err error) { + if w.err == nil { + w.err = w.ctx.Err() + } + if w.err == nil && int64(len(p)) > w.remaining { + w.err = ErrTooLarge + } + if w.err == nil { + n, w.err = w.dst.Write(p) + w.remaining -= int64(n) + if w.err == nil && n != len(p) { + w.err = io.ErrShortWrite + } + } + if w.err != nil { + w.cancel() + } + return n, w.err +} diff --git a/internal/backup/export_integration_test.go b/internal/backup/export_integration_test.go new file mode 100644 index 00000000..886a16cb --- /dev/null +++ b/internal/backup/export_integration_test.go @@ -0,0 +1,216 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// TestExportPostgres runs only on an explicitly selected private test server with +// pg_dump/psql installed. It creates and drops two randomly named test databases. +// BEACON_BACKUP_TEST_BINARY points to the exact compiled command being verified. +func TestExportPostgres(t *testing.T) { + if os.Getenv("BEACON_BACKUP_TEST_POSTGRES") != "1" { + t.Skip("set BEACON_BACKUP_TEST_POSTGRES=1 and BEACON_BACKUP_TEST_BINARY on a private PostgreSQL test server") + } + binary := os.Getenv("BEACON_BACKUP_TEST_BINARY") + if !filepath.IsAbs(binary) { + t.Fatal("the exact backup command must be an absolute path") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, "") + if err != nil { + t.Fatal("cannot connect to private test PostgreSQL") + } + defer admin.Close(context.Background()) + names := []string{"beacon_backup_test_" + strings.ToLower(rand.Text()), "beacon_backup_test_" + strings.ToLower(rand.Text())} + for _, name := range names { + ident := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil { + t.Fatal(err) + } + defer func() { + cleanup, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + if _, err := admin.Exec(cleanup, "DROP DATABASE "+ident); err != nil { + t.Errorf("cannot remove own test database: %v", err) + } + }() + } + poolFor := func(name string) *pgxpool.Pool { + t.Helper() + cfg, err := pgxpool.ParseConfig("") + if err != nil { + t.Fatal("invalid private test connection settings") + } + cfg.ConnConfig.Database = name + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal("cannot open own test database") + } + return pool + } + source, target := poolFor(names[0]), poolFor(names[1]) + defer source.Close() + defer target.Close() + if err := db.RunMigrations(ctx, source); err != nil { + t.Fatal(err) + } + _, err = source.Exec(ctx, ` +CREATE TABLE backup_fixture (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, message text, raw bytea, detail jsonb); +INSERT INTO backup_fixture (message, raw, detail) VALUES ('café 雪', decode('00ff10', 'hex'), '{"nested":[1,null,true]}'), ('line one +line two', NULL, '{}'); +CREATE TABLE backup_child (id bigint PRIMARY KEY REFERENCES backup_fixture(id)); +INSERT INTO backup_child VALUES (1); +CREATE VIEW backup_view AS SELECT id, message FROM backup_fixture; +CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM backup_child; +`) + if err != nil { + t.Fatal(err) + } + const check = `SELECT json_build_object( + 'rows', (SELECT json_agg(x ORDER BY id) FROM backup_fixture x), + 'children', (SELECT json_agg(x ORDER BY id) FROM backup_child x), + 'view', (SELECT json_agg(x ORDER BY id) FROM backup_view x), + 'materialized', (SELECT json_agg(x) FROM backup_materialized x), + 'migrations', (SELECT json_agg(x ORDER BY filename) FROM schema_migrations x), + 'columns', (SELECT md5(string_agg(table_name||column_name||data_type, ',' ORDER BY table_name,ordinal_position)) FROM information_schema.columns WHERE table_schema='public') +)::text` + var before, after string + if err := source.QueryRow(ctx, check).Scan(&before); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + config := []byte("# saved configuration, including a synthetic key\nchannel_keys:\n keys:\n '00': {key: '00000000000000000000000000000000', name: fixture}\n") + configPath, output := filepath.Join(dir, "config.yaml"), filepath.Join(dir, "backup.tar.gz") + if err := os.WriteFile(configPath, config, 0600); err != nil { + t.Fatal(err) + } + command := exec.CommandContext(ctx, binary, "-config", configPath, "-output", output, "-max-bytes", "16777216", "-timeout", "1m") + command.Env = append(os.Environ(), "PGDATABASE="+names[0]) + if err := command.Run(); err != nil { + t.Fatal("exact backup command failed") + } + original, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + for _, failure := range []string{"existing", "size", "connection", "timeout"} { + t.Run(failure, func(t *testing.T) { + failedOutput := filepath.Join(dir, failure+".tar.gz") + limit, timeout, database := "16777216", "30s", names[0] + if failure == "existing" { + failedOutput = output + } else if failure == "size" { + limit = "64" + } else if failure == "connection" { + database = "beacon_backup_test_missing_" + strings.ToLower(rand.Text()) + } else { + lock, err := source.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer lock.Rollback(context.Background()) + if _, err := lock.Exec(ctx, "LOCK TABLE backup_fixture IN ACCESS EXCLUSIVE MODE"); err != nil { + t.Fatal(err) + } + timeout = "1s" + } + failed := exec.CommandContext(ctx, binary, "-config", configPath, "-output", failedOutput, "-max-bytes", limit, "-timeout", timeout) + failed.Env = append(os.Environ(), "PGDATABASE="+database) + message, err := failed.CombinedOutput() + if err == nil || bytes.Contains(message, []byte(database)) || bytes.Contains(message, []byte("café")) { + t.Fatal("expected a failure without database identity or payload diagnostics") + } + if failure == "existing" { + retained, err := os.ReadFile(output) + if err != nil || !bytes.Equal(retained, original) { + t.Fatal("existing output changed") + } + } else if _, err := os.Lstat(failedOutput); !os.IsNotExist(err) { + t.Fatal("failed export published an output") + } + staging, _ := filepath.Glob(filepath.Join(dir, ".beacon-backup-*")) + if len(staging) != 0 { + t.Fatal("failed export retained private staging") + } + }) + } + f, err := os.Open(output) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tr, members := tar.NewReader(gz), map[string][]byte{} + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + members[h.Name], err = io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + t.Fatal(err) + } + var manifest Manifest + if err := json.Unmarshal(members["manifest.json"], &manifest); err != nil { + t.Fatal(err) + } + if len(members) != 3 || manifest.FormatVersion != 1 || !bytes.Equal(members["config.yaml"], config) { + t.Fatal("unexpected bundle members or saved config") + } + for _, member := range manifest.Files { + hash := sha256.Sum256(members[member.Name]) + if member.SHA256 != hex.EncodeToString(hash[:]) || member.Size != int64(len(members[member.Name])) { + t.Fatal("bundle checksum mismatch") + } + } + restore := exec.CommandContext(ctx, "psql", "-X", "--set=ON_ERROR_STOP=on", "--single-transaction") + restore.Env = append(os.Environ(), "PGDATABASE="+names[1]) + restore.Stdin = bytes.NewReader(members["database.sql"]) + if err := restore.Run(); err != nil { + t.Fatal("restore into own empty database failed") + } + if err := target.QueryRow(ctx, check).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatal("schema, journal, data, relationships or materialized view changed after restore") + } + var nextID int + if err := target.QueryRow(ctx, "INSERT INTO backup_fixture (message) VALUES ('next') RETURNING id").Scan(&nextID); err != nil || nextID != 3 { + t.Fatal("identity sequence did not survive restore") + } + t.Log("exact command restored all migrations, columns, Unicode/bytea/JSON data, relationships, views and identity sequence") +} diff --git a/internal/backup/export_test.go b/internal/backup/export_test.go new file mode 100644 index 00000000..a3481af7 --- /dev/null +++ b/internal/backup/export_test.go @@ -0,0 +1,232 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +const testSQL = "CREATE TABLE fixture (id bigint PRIMARY KEY, name text);\nINSERT INTO fixture VALUES (1, 'café');\n" + +// TestDumpProcess supplies an actual subprocess, including failure and cancellation. +func TestDumpProcess(t *testing.T) { + mode := os.Getenv("BEACON_BACKUP_TEST_PROCESS") + if mode == "" { + return + } + switch mode { + case "ok": + _, _ = io.WriteString(os.Stdout, testSQL) + case "fail": + _, _ = io.WriteString(os.Stdout, "incomplete dump") + _, _ = io.WriteString(os.Stderr, "PRIVATE_PASSWORD_CANARY") + os.Exit(17) + case "large": + _, _ = os.Stdout.Write(bytes.Repeat([]byte("x"), 65536)) + case "hang": + _, _ = io.WriteString(os.Stdout, "incomplete dump") + time.Sleep(time.Minute) + case "empty": + default: + os.Exit(2) + } + os.Exit(0) +} + +func helper(t *testing.T, mode string) func(context.Context) *exec.Cmd { + t.Helper() + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + return func(ctx context.Context) *exec.Cmd { + cmd := exec.CommandContext(ctx, exe, "-test.run=^TestDumpProcess$") + cmd.Env = append(os.Environ(), "BEACON_BACKUP_TEST_PROCESS="+mode) + return cmd + } +} + +func setup(t *testing.T) Options { + t.Helper() + t.Setenv("PGDATABASE", "fixture") + dir := t.TempDir() + opts := Options{ConfigPath: filepath.Join(dir, "saved.yaml"), OutputPath: filepath.Join(dir, "backup.tar.gz"), + MaxBytes: int64(len(testSQL)), Timeout: 10 * time.Second, Version: "test-revision"} + if err := os.WriteFile(opts.ConfigPath, []byte("# preserve comments and this synthetic key\nchannel_keys:\n keys:\n '00': {key: '00000000000000000000000000000000', name: fixture}\n"), 0600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + paths, err := filepath.Glob(filepath.Join(dir, ".beacon-backup-*")) + if err != nil || len(paths) != 0 { + t.Errorf("private staging was not cleaned: %v %v", paths, err) + } + }) + return opts +} + +func TestExportBundle(t *testing.T) { + opts := setup(t) + if err := export(context.Background(), opts, helper(t, "ok")); err != nil { + t.Fatal(err) + } + f, err := os.Open(opts.OutputPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + info, err := f.Stat() + if err != nil || (runtime.GOOS != "windows" && info.Mode().Perm() != 0600) { + t.Fatalf("backup must be private: %v %v", info, err) + } + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tr := tar.NewReader(gz) + files := map[string][]byte{} + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil || header.Mode != 0600 || header.Typeflag != tar.TypeReg { + t.Fatalf("invalid archive member: %v %v", header, err) + } + if _, duplicate := files[header.Name]; duplicate { + t.Fatal("duplicate archive name") + } + files[header.Name], err = io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + t.Fatal("invalid gzip footer:", err) + } + config, _ := os.ReadFile(opts.ConfigPath) + if len(files) != 3 || string(files["database.sql"]) != testSQL || !bytes.Equal(files["config.yaml"], config) { + t.Fatal("payloads differ from the supplied SQL and saved config") + } + var manifest Manifest + if err := json.Unmarshal(files["manifest.json"], &manifest); err != nil { + t.Fatal(err) + } + if manifest.FormatVersion != 1 || manifest.ToolVersion != opts.Version || manifest.CreatedAt.IsZero() || len(manifest.Files) != 2 || len(manifest.Excluded) != 5 { + t.Fatalf("incomplete manifest: %+v", manifest) + } + for _, member := range manifest.Files { + digest := sha256.Sum256(files[member.Name]) + if member.Size != int64(len(files[member.Name])) || member.SHA256 != hex.EncodeToString(digest[:]) { + t.Errorf("manifest mismatch for %s", member.Name) + } + } +} + +func TestExportFailureCleanup(t *testing.T) { + for _, mode := range []string{"fail", "large", "hang", "empty"} { + t.Run(mode, func(t *testing.T) { + opts := setup(t) + if mode == "hang" { + opts.Timeout = time.Second + } + err := export(context.Background(), opts, helper(t, mode)) + if err == nil || strings.Contains(err.Error(), "PRIVATE_PASSWORD_CANARY") { + t.Fatalf("expected sanitized failure, got %v", err) + } + if mode == "large" && !errors.Is(err, ErrTooLarge) { + t.Fatalf("wrong limit error: %v", err) + } + if mode == "hang" && !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("wrong timeout error: %v", err) + } + if _, err := os.Lstat(opts.OutputPath); !errors.Is(err, os.ErrNotExist) { + t.Fatal("failed export published an output") + } + }) + } +} + +func TestExportNeverReplacesDestination(t *testing.T) { + for _, race := range []bool{false, true} { + t.Run(map[bool]string{false: "existing", true: "created-during-export"}[race], func(t *testing.T) { + opts := setup(t) + putExisting := func() { + if err := os.WriteFile(opts.OutputPath, []byte("retain this backup"), 0600); err != nil { + t.Fatal(err) + } + } + if !race { + putExisting() + } + cmd := helper(t, "ok") + err := export(context.Background(), opts, func(ctx context.Context) *exec.Cmd { + if !race { + t.Fatal("existing output must be rejected before starting pg_dump") + } + putExisting() + return cmd(ctx) + }) + data, readErr := os.ReadFile(opts.OutputPath) + if err == nil || readErr != nil || string(data) != "retain this backup" { + t.Fatalf("existing backup changed: %v %v", err, readErr) + } + }) + } +} + +func TestExportValidation(t *testing.T) { + for _, mode := range []string{"database", "limit", "config", "canceled"} { + t.Run(mode, func(t *testing.T) { + opts := setup(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + switch mode { + case "database": + t.Setenv("PGDATABASE", "") + case "limit": + opts.MaxBytes = 0 + case "config": + if err := os.WriteFile(opts.ConfigPath, make([]byte, maxConfigBytes+1), 0600); err != nil { + t.Fatal(err) + } + case "canceled": + cancel() + } + if err := export(ctx, opts, func(context.Context) *exec.Cmd { t.Fatal("dump ran on invalid input"); return nil }); err == nil { + t.Fatal("invalid export succeeded") + } + }) + } +} + +func TestDumpArgumentsExcludeConnectionSecrets(t *testing.T) { + t.Setenv("PGPASSWORD", "PRIVATE_PASSWORD_CANARY") + t.Setenv("POSTGRES_DSN", "postgres://user:PRIVATE_PASSWORD_CANARY@localhost/db") + cmd := dumpCommand(context.Background()) + if strings.Contains(strings.Join(cmd.Args, " "), "PRIVATE_PASSWORD_CANARY") { + t.Fatal("credentials reached command arguments") + } + args := strings.Join(cmd.Args, " ") + for _, required := range []string{"--no-password", "--lock-wait-timeout=5000", "--no-owner", "--no-acl", "--no-tablespaces", "--format=plain"} { + if !strings.Contains(args, required) { + t.Errorf("missing dump boundary: %s", required) + } + } +} From b21279c747537f408f634de89dc7ab6de36d4a56 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 18 Sep 2026 06:17:59 -0400 Subject: [PATCH 032/141] feat(db): add operator account lifecycle storage (#148) --- db/accounts.go | 78 ++++++++++++++++ db/accounts_integration_test.go | 155 ++++++++++++++++++++++++++++++++ db/accounts_test.go | 27 ++++++ db/migrations/034_accounts.sql | 12 +++ db/queries/queries.sql | 25 ++++++ db/sqlc/mock/querier.go | 60 +++++++++++++ db/sqlc/models.go | 7 ++ db/sqlc/querier.go | 6 ++ db/sqlc/queries.sql.go | 90 +++++++++++++++++++ internal/api/accounts.go | 60 +++++++++++++ internal/api/accounts_test.go | 27 ++++++ 11 files changed, 547 insertions(+) create mode 100644 db/accounts.go create mode 100644 db/accounts_integration_test.go create mode 100644 db/accounts_test.go create mode 100644 db/migrations/034_accounts.sql create mode 100644 internal/api/accounts.go create mode 100644 internal/api/accounts_test.go diff --git a/db/accounts.go b/db/accounts.go new file mode 100644 index 00000000..336dcc61 --- /dev/null +++ b/db/accounts.go @@ -0,0 +1,78 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +var _ api.AccountStore = (*Store)(nil) + +func (s *Store) CreateAccount(ctx context.Context, name string) (api.Account, error) { + name, err := api.NormalizeAccountName(name) + if err != nil { + return api.Account{}, err + } + row, err := s.q.CreateAccount(ctx, name) + if errors.Is(err, pgx.ErrNoRows) { + return api.Account{}, api.ErrAccountNameConflict + } + if err != nil { + return api.Account{}, err + } + return accountFromRow(row), nil +} + +func (s *Store) ListAccounts(ctx context.Context) ([]api.Account, error) { + // ponytail: unpaginated operator list; add cursors if account counts grow large. + rows, err := s.q.ListAccounts(ctx) + if err != nil { + return nil, err + } + items := make([]api.Account, 0, len(rows)) + for _, row := range rows { + items = append(items, accountFromRow(row)) + } + return items, nil +} + +func (s *Store) GetAccount(ctx context.Context, id uuid.UUID) (api.Account, error) { + row, err := s.q.GetAccount(ctx, id) + if errors.Is(err, pgx.ErrNoRows) { + return api.Account{}, api.ErrAccountNotFound + } + if err != nil { + return api.Account{}, err + } + return accountFromRow(row), nil +} + +func (s *Store) DeactivateAccount(ctx context.Context, id uuid.UUID) error { + result, err := s.q.DeactivateAccount(ctx, id) + if err != nil { + return err + } + if !result.Found { + return api.ErrAccountNotFound + } + if !result.Deactivated { + return api.ErrAccountInactive + } + return nil +} + +func accountFromRow(row sqlc.Account) api.Account { + account := api.Account{ID: row.ID, Name: row.Name, CreatedAt: row.CreatedAt.Time.UTC(), Active: !row.DeactivatedAt.Valid} + if row.DeactivatedAt.Valid { + when := row.DeactivatedAt.Time.UTC() + account.DeactivatedAt = &when + } + return account +} diff --git a/db/accounts_integration_test.go b/db/accounts_integration_test.go new file mode 100644 index 00000000..02700506 --- /dev/null +++ b/db/accounts_integration_test.go @@ -0,0 +1,155 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestAccountsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for account regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + setup, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer setup.Close(context.Background()) + schema := "account_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + quoted := pgx.Identifier{schema}.Sanitize() + if _, err := setup.Exec(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatal(err) + } + defer func() { + if _, err := setup.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error(err) + } + }() + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.MaxConns = 4 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + sql, err := migrationFiles.ReadFile("migrations/034_accounts.sql") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err := applyMigration(ctx, pool, string(sql)); err != nil { + t.Fatal(err) + } + } + store := New(pool, 0, 0) + items, err := store.ListAccounts(ctx) + if err != nil || items == nil || len(items) != 0 { + t.Fatal("empty list", err) + } + if _, err := store.CreateAccount(ctx, "\t"); !errors.Is(err, api.ErrAccountNameInvalid) { + t.Fatal("invalid name accepted", err) + } + first, err := store.CreateAccount(ctx, " Montréal 🦀 ") + if err != nil || first.Name != "Montréal 🦀" || !first.Active || first.ID == uuid.Nil || first.CreatedAt.IsZero() { + t.Fatal("create", err) + } + if _, err := store.CreateAccount(ctx, first.Name); !errors.Is(err, api.ErrAccountNameConflict) { + t.Fatal("duplicate", err) + } + if err := store.DeactivateAccount(ctx, first.ID); err != nil { + t.Fatal(err) + } + inactive, err := store.GetAccount(ctx, first.ID) + if err != nil || inactive.Active || inactive.DeactivatedAt == nil { + t.Fatal("deactivate", err) + } + if err := store.DeactivateAccount(ctx, first.ID); !errors.Is(err, api.ErrAccountInactive) { + t.Fatal("repeated deactivate", err) + } + again, _ := store.GetAccount(ctx, first.ID) + if !again.DeactivatedAt.Equal(*inactive.DeactivatedAt) { + t.Fatal("deactivation timestamp changed") + } + reused, err := store.CreateAccount(ctx, first.Name) + if err != nil || reused.ID == first.ID { + t.Fatal("name reuse", err) + } + if _, err := store.CreateAccount(ctx, "montréal 🦀"); err != nil { + t.Fatal("names must be case-sensitive", err) + } + missing := uuid.New() + if _, err := store.GetAccount(ctx, missing); !errors.Is(err, api.ErrAccountNotFound) { + t.Fatal("missing get", err) + } + if err := store.DeactivateAccount(ctx, missing); !errors.Is(err, api.ErrAccountNotFound) { + t.Fatal("missing deactivate", err) + } + items, err = store.ListAccounts(ctx) + if err != nil || len(items) != 3 { + t.Fatal("list lifecycle", err) + } + for i := 1; i < len(items); i++ { + if items[i-1].CreatedAt.Before(items[i].CreatedAt) || (items[i-1].CreatedAt.Equal(items[i].CreatedAt) && items[i-1].ID.String() < items[i].ID.String()) { + t.Fatal("unstable list order") + } + } + start := make(chan struct{}) + results := make(chan error, 8) + for i := 0; i < 8; i++ { + go func() { <-start; _, err := store.CreateAccount(ctx, "concurrent"); results <- err }() + } + close(start) + created := 0 + for i := 0; i < 8; i++ { + err := <-results + if err == nil { + created++ + } else if !errors.Is(err, api.ErrAccountNameConflict) { + t.Fatal("concurrent create", err) + } + } + if created != 1 { + t.Fatalf("created %d accounts with the same name", created) + } + for round := 0; round < 8; round++ { + account, err := store.CreateAccount(ctx, "race-deactivate") + if err != nil { + t.Fatal(err) + } + gate := make(chan struct{}) + for i := 0; i < 2; i++ { + go func() { <-gate; results <- store.DeactivateAccount(ctx, account.ID) }() + } + close(gate) + changed := 0 + for i := 0; i < 2; i++ { + err := <-results + if err == nil { + changed++ + } else if !errors.Is(err, api.ErrAccountInactive) { + t.Fatal("concurrent deactivate", err) + } + } + if changed != 1 { + t.Fatalf("deactivated %d times", changed) + } + } +} diff --git a/db/accounts_test.go b/db/accounts_test.go new file mode 100644 index 00000000..e8489ee1 --- /dev/null +++ b/db/accounts_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +func TestAccountFromRow(t *testing.T) { + when := time.Date(2026, 1, 1, 12, 0, 0, 0, time.FixedZone("test", 3600)) + row := sqlc.Account{ID: uuid.New(), Name: "test", CreatedAt: pgtype.Timestamptz{Time: when, Valid: true}} + active := accountFromRow(row) + if active.ID != row.ID || active.Name != row.Name || !active.Active || active.DeactivatedAt != nil || !active.CreatedAt.Equal(when) || active.CreatedAt.Location() != time.UTC { + t.Fatal("active account mapping") + } + row.DeactivatedAt = pgtype.Timestamptz{Time: when.Add(time.Hour), Valid: true} + inactive := accountFromRow(row) + if inactive.Active || inactive.DeactivatedAt == nil || !inactive.DeactivatedAt.Equal(row.DeactivatedAt.Time) { + t.Fatal("inactive account mapping") + } +} diff --git a/db/migrations/034_accounts.sql b/db/migrations/034_accounts.sql new file mode 100644 index 00000000..16b57fc5 --- /dev/null +++ b/db/migrations/034_accounts.sql @@ -0,0 +1,12 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +CREATE TABLE IF NOT EXISTS accounts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL CHECK (char_length(name) BETWEEN 1 AND 128 AND name = btrim(name)), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deactivated_at TIMESTAMPTZ +); + +CREATE UNIQUE INDEX IF NOT EXISTS accounts_name_uidx + ON accounts (name) WHERE deactivated_at IS NULL; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 3bd18618..aceca9e7 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1454,3 +1454,28 @@ OR ( AND iata = nn.iata ) ) > 1; + +-- name: CreateAccount :one +INSERT INTO accounts (name) VALUES (sqlc.arg(name)) +ON CONFLICT (name) WHERE deactivated_at IS NULL DO NOTHING +RETURNING id, name, created_at, deactivated_at; + +-- name: ListAccounts :many +SELECT id, name, created_at, deactivated_at FROM accounts +ORDER BY created_at DESC, id DESC; + +-- name: GetAccount :one +SELECT id, name, created_at, deactivated_at FROM accounts WHERE id = $1; + +-- name: DeactivateAccount :one +-- Lock the current row before deciding the outcome, including when another +-- deactivation commits while this statement is waiting for its row lock. +WITH target AS MATERIALIZED ( + SELECT a.id, a.deactivated_at FROM accounts a WHERE a.id = $1 FOR UPDATE +), changed AS ( + UPDATE accounts a SET deactivated_at = NOW() + FROM target t WHERE a.id = t.id AND t.deactivated_at IS NULL + RETURNING a.id +) +SELECT EXISTS(SELECT 1 FROM target) AS found, + EXISTS(SELECT 1 FROM changed) AS deactivated; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index e43434b0..09077ffd 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -43,6 +43,36 @@ func (m *MockQuerier) EXPECT() *MockQuerierMockRecorder { return m.recorder } +// CreateAccount mocks base method. +func (m *MockQuerier) CreateAccount(ctx context.Context, name string) (db.Account, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateAccount", ctx, name) + ret0, _ := ret[0].(db.Account) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateAccount indicates an expected call of CreateAccount. +func (mr *MockQuerierMockRecorder) CreateAccount(ctx, name any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateAccount", reflect.TypeOf((*MockQuerier)(nil).CreateAccount), ctx, name) +} + +// DeactivateAccount mocks base method. +func (m *MockQuerier) DeactivateAccount(ctx context.Context, id uuid.UUID) (db.DeactivateAccountRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeactivateAccount", ctx, id) + ret0, _ := ret[0].(db.DeactivateAccountRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeactivateAccount indicates an expected call of DeactivateAccount. +func (mr *MockQuerierMockRecorder) DeactivateAccount(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeactivateAccount", reflect.TypeOf((*MockQuerier)(nil).DeactivateAccount), ctx, id) +} + // DeleteOldChannelIATAs mocks base method. func (m *MockQuerier) DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error { m.ctrl.T.Helper() @@ -142,6 +172,21 @@ func (mr *MockQuerierMockRecorder) DeleteOldTraceIATAs(ctx, lastHeard any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceIATAs", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceIATAs), ctx, lastHeard) } +// GetAccount mocks base method. +func (m *MockQuerier) GetAccount(ctx context.Context, id uuid.UUID) (db.Account, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAccount", ctx, id) + ret0, _ := ret[0].(db.Account) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAccount indicates an expected call of GetAccount. +func (mr *MockQuerierMockRecorder) GetAccount(ctx, id any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccount", reflect.TypeOf((*MockQuerier)(nil).GetAccount), ctx, id) +} + // GetChannelByID mocks base method. func (m *MockQuerier) GetChannelByID(ctx context.Context, id int32) (db.Channel, error) { m.ctrl.T.Helper() @@ -831,6 +876,21 @@ func (mr *MockQuerierMockRecorder) InsertObserverTelemetry(ctx, arg any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertObserverTelemetry", reflect.TypeOf((*MockQuerier)(nil).InsertObserverTelemetry), ctx, arg) } +// ListAccounts mocks base method. +func (m *MockQuerier) ListAccounts(ctx context.Context) ([]db.Account, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListAccounts", ctx) + ret0, _ := ret[0].([]db.Account) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListAccounts indicates an expected call of ListAccounts. +func (mr *MockQuerierMockRecorder) ListAccounts(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAccounts", reflect.TypeOf((*MockQuerier)(nil).ListAccounts), ctx) +} + // ListAllChannelMessages mocks base method. func (m *MockQuerier) ListAllChannelMessages(ctx context.Context, arg db.ListAllChannelMessagesParams) ([]db.ListAllChannelMessagesRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index a2a5432d..80ab12bb 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -9,6 +9,13 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +type Account struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + DeactivatedAt pgtype.Timestamptz `json:"deactivated_at"` +} + type Channel struct { ID int32 `json:"id"` ChannelHash []byte `json:"channel_hash"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index ec502af7..2c022aa1 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -12,6 +12,10 @@ import ( ) type Querier interface { + CreateAccount(ctx context.Context, name string) (Account, error) + // Lock the current row before deciding the outcome, including when another + // deactivation commits while this statement is waiting for its row lock. + DeactivateAccount(ctx context.Context, id uuid.UUID) (DeactivateAccountRow, error) // Keeps the channel IATA filter in step with packet retention. DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error // Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- @@ -35,6 +39,7 @@ type Querier interface { DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error // Keeps the trace IATA filter in step with packet retention. DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error + GetAccount(ctx context.Context, id uuid.UUID) (Account, error) GetChannelByID(ctx context.Context, id int32) (Channel, error) // Returns neighbors of a node that are in a different IATA. GetCrossIATANeighbors(ctx context.Context, arg GetCrossIATANeighborsParams) ([]GetCrossIATANeighborsRow, error) @@ -116,6 +121,7 @@ type Querier interface { // Inserts a telemetry snapshot for an observer. The reported_at timestamp should // be truncated to the configured resolution before calling to ensure deduplication. InsertObserverTelemetry(ctx context.Context, arg InsertObserverTelemetryParams) error + ListAccounts(ctx context.Context) ([]Account, error) // Returns all messages across all channels with optional time, IATA, scope and cursor filters. // Pass empty string for iata or scope to skip those filters. // Pass cursor=0 to start from the beginning. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index fec385d6..6dc837d2 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -12,6 +12,50 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const createAccount = `-- name: CreateAccount :one +INSERT INTO accounts (name) VALUES ($1) +ON CONFLICT (name) WHERE deactivated_at IS NULL DO NOTHING +RETURNING id, name, created_at, deactivated_at +` + +func (q *Queries) CreateAccount(ctx context.Context, name string) (Account, error) { + row := q.db.QueryRow(ctx, createAccount, name) + var i Account + err := row.Scan( + &i.ID, + &i.Name, + &i.CreatedAt, + &i.DeactivatedAt, + ) + return i, err +} + +const deactivateAccount = `-- name: DeactivateAccount :one +WITH target AS MATERIALIZED ( + SELECT a.id, a.deactivated_at FROM accounts a WHERE a.id = $1 FOR UPDATE +), changed AS ( + UPDATE accounts a SET deactivated_at = NOW() + FROM target t WHERE a.id = t.id AND t.deactivated_at IS NULL + RETURNING a.id +) +SELECT EXISTS(SELECT 1 FROM target) AS found, + EXISTS(SELECT 1 FROM changed) AS deactivated +` + +type DeactivateAccountRow struct { + Found bool `json:"found"` + Deactivated bool `json:"deactivated"` +} + +// Lock the current row before deciding the outcome, including when another +// deactivation commits while this statement is waiting for its row lock. +func (q *Queries) DeactivateAccount(ctx context.Context, id uuid.UUID) (DeactivateAccountRow, error) { + row := q.db.QueryRow(ctx, deactivateAccount, id) + var i DeactivateAccountRow + err := row.Scan(&i.Found, &i.Deactivated) + return i, err +} + const deleteOldChannelIATAs = `-- name: DeleteOldChannelIATAs :exec DELETE FROM channel_iatas WHERE last_heard < $1 ` @@ -130,6 +174,22 @@ func (q *Queries) DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Time return err } +const getAccount = `-- name: GetAccount :one +SELECT id, name, created_at, deactivated_at FROM accounts WHERE id = $1 +` + +func (q *Queries) GetAccount(ctx context.Context, id uuid.UUID) (Account, error) { + row := q.db.QueryRow(ctx, getAccount, id) + var i Account + err := row.Scan( + &i.ID, + &i.Name, + &i.CreatedAt, + &i.DeactivatedAt, + ) + return i, err +} + const getChannelByID = `-- name: GetChannelByID :one SELECT id, channel_hash, key_fingerprint, name, hashtag, is_hashtag, is_public, key_known, first_seen, last_seen, message_count FROM channels WHERE id = $1 ` @@ -2054,6 +2114,36 @@ func (q *Queries) InsertObserverTelemetry(ctx context.Context, arg InsertObserve return err } +const listAccounts = `-- name: ListAccounts :many +SELECT id, name, created_at, deactivated_at FROM accounts +ORDER BY created_at DESC, id DESC +` + +func (q *Queries) ListAccounts(ctx context.Context) ([]Account, error) { + rows, err := q.db.Query(ctx, listAccounts) + if err != nil { + return nil, err + } + defer rows.Close() + items := []Account{} + for rows.Next() { + var i Account + if err := rows.Scan( + &i.ID, + &i.Name, + &i.CreatedAt, + &i.DeactivatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listAllChannelMessages = `-- name: ListAllChannelMessages :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count diff --git a/internal/api/accounts.go b/internal/api/accounts.go new file mode 100644 index 00000000..df64fbf3 --- /dev/null +++ b/internal/api/accounts.go @@ -0,0 +1,60 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "context" + "errors" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/google/uuid" +) + +var ( + ErrAccountNotFound = errors.New("account not found") + ErrAccountInactive = errors.New("account is already inactive") + ErrAccountNameConflict = errors.New("an active account already uses this name") + ErrAccountNameInvalid = errors.New("account name must be 1-128 characters without control characters") +) + +// Account is an operator-defined record, not a login or authentication principal. +type Account struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + DeactivatedAt *time.Time `json:"deactivated_at,omitempty"` + Active bool `json:"active"` +} + +type AccountList struct { + Items []Account `json:"items"` +} + +// AccountStore is separate from public Reader/cache and ingest write interfaces. +type AccountStore interface { + CreateAccount(context.Context, string) (Account, error) + ListAccounts(context.Context) ([]Account, error) + GetAccount(context.Context, uuid.UUID) (Account, error) + DeactivateAccount(context.Context, uuid.UUID) error +} + +// NormalizeAccountName applies the same validation to every account writer. +func NormalizeAccountName(name string) (string, error) { + if !utf8.ValidString(name) { + return "", ErrAccountNameInvalid + } + name = strings.TrimSpace(name) + if length := utf8.RuneCountInString(name); length < 1 || length > 128 { + return "", ErrAccountNameInvalid + } + for _, r := range name { + if unicode.IsControl(r) { + return "", ErrAccountNameInvalid + } + } + return name, nil +} diff --git a/internal/api/accounts_test.go b/internal/api/accounts_test.go new file mode 100644 index 00000000..be4e247c --- /dev/null +++ b/internal/api/accounts_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "errors" + "strings" + "testing" +) + +func TestNormalizeAccountName(t *testing.T) { + for _, tc := range []struct { + input, want string + invalid bool + }{ + {" Montréal 🦀 \t", "Montréal 🦀", false}, {"A", "A", false}, + {strings.Repeat("é", 128), strings.Repeat("é", 128), false}, + {"", "", true}, {"\u00a0\t", "", true}, {strings.Repeat("é", 129), "", true}, + {"x\x00y", "", true}, {"x\ny", "", true}, {"\xff", "", true}, + } { + got, err := NormalizeAccountName(tc.input) + if got != tc.want || errors.Is(err, ErrAccountNameInvalid) != tc.invalid { + t.Fatalf("name validation: got %q, error=%v", got, err) + } + } +} From bcdea76c26d42e3f8df950a890fb226f637d8425 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 18 Sep 2026 06:18:06 -0400 Subject: [PATCH 033/141] feat(stats): filter scope counts by IATA and region (#142) * feat(stats): filter scope counts by IATA and region * perf(stats): aggregate region observations before counting scopes --- db/queries/queries.sql | 34 ++++++- db/scope_stats_integration_test.go | 101 +++++++++++++++++++ db/sqlc/mock/querier.go | 8 +- db/sqlc/querier.go | 7 +- db/sqlc/queries.sql.go | 38 ++++++-- db/stats.go | 4 +- db/stats_test.go | 4 +- docs/docs.go | 33 +++++++ docs/swagger.json | 33 +++++++ docs/swagger.yaml | 26 +++++ internal/api/handlers/scope_stats_test.go | 112 ++++++++++++++++++++++ internal/api/handlers/stats.go | 21 +++- internal/api/handlers/stats_test.go | 2 +- internal/api/handlers/stub_reader_test.go | 6 +- internal/api/reader.go | 3 +- internal/cache/cache_test.go | 6 +- internal/cache/reader.go | 13 ++- internal/cache/scope_stats_test.go | 59 ++++++++++++ 18 files changed, 476 insertions(+), 34 deletions(-) create mode 100644 db/scope_stats_integration_test.go create mode 100644 internal/api/handlers/scope_stats_test.go create mode 100644 internal/cache/scope_stats_test.go diff --git a/db/queries/queries.sql b/db/queries/queries.sql index aceca9e7..bdc9c916 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1102,14 +1102,38 @@ WHERE ($1::text = '' OR preset = $1::text) ORDER BY preset, iata, source_type; -- name: GetScopeStats :many --- Count each table on its own; the old cross-join blew up to millions of rows --- before COUNT(DISTINCT) (~10s). +-- Aggregate matching observations once, separately from node memberships to avoid +-- a cross-join. Empty IATAs keep the original global counts, including associations +-- whose observations have expired; the filtered aggregates are empty in that case. +WITH observation_counts AS ( + SELECT p.scope_id, + COUNT(DISTINCT p.packet_hash) AS packet_count, + COUNT(DISTINCT os.observer_id) AS observer_count + FROM packet_observations po + JOIN packets p ON p.packet_hash = po.packet_hash + LEFT JOIN observer_scopes os ON os.scope_id = p.scope_id AND os.observer_id = po.observer_id + WHERE po.iata = ANY(sqlc.arg(iatas)::bpchar[]) AND p.scope_id IS NOT NULL + GROUP BY p.scope_id +), node_counts AS ( + SELECT n.default_scope_id AS scope_id, COUNT(*) AS node_count + FROM nodes n + WHERE n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY(sqlc.arg(iatas)::bpchar[])) + GROUP BY n.default_scope_id +) SELECT ts.name, - (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count, - (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count, - (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count + CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0 + THEN (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) + ELSE COALESCE(oc.packet_count, 0) END::bigint AS packet_count, + CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0 + THEN (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) + ELSE COALESCE(oc.observer_count, 0) END::bigint AS observer_count, + CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0 + THEN (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) + ELSE COALESCE(nc.node_count, 0) END::bigint AS node_count FROM transport_scopes ts +LEFT JOIN observation_counts oc ON oc.scope_id = ts.id +LEFT JOIN node_counts nc ON nc.scope_id = ts.id ORDER BY ts.name; -- ============================================================ diff --git a/db/scope_stats_integration_test.go b/db/scope_stats_integration_test.go new file mode 100644 index 00000000..94e1845e --- /dev/null +++ b/db/scope_stats_integration_test.go @@ -0,0 +1,101 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "reflect" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +// Run on a migrated private database. All fixture tables and writes are rolled back. +func TestScopeStatsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"transport_scopes", "packets", "packet_observations", "observer_scopes", "nodes", "node_iatas"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) +SELECT i,name,decode(repeat('00',16),'hex'),decode(lpad(to_hex(i),16,'0'),'hex') +FROM (VALUES (1,'#a'),(2,'#b'),(3,'#unused')) v(i,name); +INSERT INTO packets (packet_hash,scope_id,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),scope_id,4,0,1,'\x00','\x00',NOW(),NOW() +FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,2),(8,NULL)) v(i,scope_id); +INSERT INTO observer_scopes (observer_id,scope_id) +SELECT md5(i::text)::uuid,scope_id FROM (VALUES (1,1),(2,1),(4,1),(5,1),(1,2),(2,2),(3,2)) v(i,scope_id); +INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT id,decode(lpad(to_hex(packet),2,'0'),'hex'),md5(observer::text)::uuid,iata, + '2026-01-01'::timestamptz+id*interval '1 second',0,1,0 +FROM (VALUES (1,1,1,'YVR'),(2,1,4,'YVR'),(3,1,2,'YYJ'),(4,2,2,'YVR'), + (5,3,1,'YYZ'),(6,5,3,'YVR'),(7,6,1,'YYJ'),(8,7,2,'YYZ'),(9,8,1,'YVR')) v(id,packet,observer,iata); +INSERT INTO nodes (id,public_key,node_type,default_scope_id) +SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),2,scope_id +FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,NULL)) v(i,scope_id); +INSERT INTO node_iatas (node_id,iata) +SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YYZ'),(5,'YYJ'),(6,'YVR'),(6,'YYJ'),(7,'YVR')) v(i,iata); +`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + for _, tc := range []struct { + name string + iatas []string + a, b [3]int64 // packets, observers, nodes + }{ + {"global", nil, [3]int64{4, 4, 4}, [3]int64{3, 3, 2}}, + {"empty filter", []string{}, [3]int64{4, 4, 4}, [3]int64{3, 3, 2}}, + {"YVR", []string{"YVR"}, [3]int64{2, 3, 2}, [3]int64{1, 1, 1}}, + {"YYJ", []string{"YYJ"}, [3]int64{1, 1, 1}, [3]int64{1, 1, 2}}, + {"YYZ", []string{"YYZ"}, [3]int64{1, 1, 1}, [3]int64{1, 1, 0}}, + {"overlapping IATAs", []string{"YVR", "YYJ"}, [3]int64{2, 3, 2}, [3]int64{2, 2, 2}}, + {"duplicate IATAs", []string{"YYJ", "YVR", "YYJ"}, [3]int64{2, 3, 2}, [3]int64{2, 2, 2}}, + {"unknown IATA", []string{"ZZZ"}, [3]int64{}, [3]int64{}}, + } { + t.Run(tc.name, func(t *testing.T) { + rows, err := store.GetScopeStats(ctx, tc.iatas) + if err != nil { + t.Fatal(err) + } + want := []api.ScopeStats{ + {Name: "#a", PacketCount: tc.a[0], ObserverCount: tc.a[1], NodeCount: tc.a[2]}, + {Name: "#b", PacketCount: tc.b[0], ObserverCount: tc.b[1], NodeCount: tc.b[2]}, + {Name: "#unused"}, + } + if !reflect.DeepEqual(rows, want) { + t.Fatalf("counts = %+v, want %+v", rows, want) + } + }) + } + if _, err := tx.Exec(ctx, "DELETE FROM transport_scopes"); err != nil { + t.Fatal(err) + } + rows, err := store.GetScopeStats(ctx, nil) + if err != nil || rows == nil || len(rows) != 0 { + t.Fatalf("empty roster = %+v, %v", rows, err) + } +} diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 09077ffd..d11eafc5 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -653,18 +653,18 @@ func (mr *MockQuerierMockRecorder) GetScopeNames(ctx any) *gomock.Call { } // GetScopeStats mocks base method. -func (m *MockQuerier) GetScopeStats(ctx context.Context) ([]db.GetScopeStatsRow, error) { +func (m *MockQuerier) GetScopeStats(ctx context.Context, iatas []string) ([]db.GetScopeStatsRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetScopeStats", ctx) + ret := m.ctrl.Call(m, "GetScopeStats", ctx, iatas) ret0, _ := ret[0].([]db.GetScopeStatsRow) ret1, _ := ret[1].(error) return ret0, ret1 } // GetScopeStats indicates an expected call of GetScopeStats. -func (mr *MockQuerierMockRecorder) GetScopeStats(ctx any) *gomock.Call { +func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, iatas any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, iatas) } // GetScopesByIATAs mocks base method. diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 2c022aa1..a6e119cc 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -82,9 +82,10 @@ type Querier interface { GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) GetScopeNames(ctx context.Context) ([]string, error) - // Count each table on its own; the old cross-join blew up to millions of rows - // before COUNT(DISTINCT) (~10s). - GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) + // Aggregate matching observations once, separately from node memberships to avoid + // a cross-join. Empty IATAs keep the original global counts, including associations + // whose observations have expired; the filtered aggregates are empty in that case. + GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds // the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 6dc837d2..ae0b9d6b 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1418,12 +1418,35 @@ func (q *Queries) GetScopeNames(ctx context.Context) ([]string, error) { } const getScopeStats = `-- name: GetScopeStats :many +WITH observation_counts AS ( + SELECT p.scope_id, + COUNT(DISTINCT p.packet_hash) AS packet_count, + COUNT(DISTINCT os.observer_id) AS observer_count + FROM packet_observations po + JOIN packets p ON p.packet_hash = po.packet_hash + LEFT JOIN observer_scopes os ON os.scope_id = p.scope_id AND os.observer_id = po.observer_id + WHERE po.iata = ANY($1::bpchar[]) AND p.scope_id IS NOT NULL + GROUP BY p.scope_id +), node_counts AS ( + SELECT n.default_scope_id AS scope_id, COUNT(*) AS node_count + FROM nodes n + WHERE n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($1::bpchar[])) + GROUP BY n.default_scope_id +) SELECT ts.name, - (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) AS packet_count, - (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) AS observer_count, - (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) AS node_count + CASE WHEN COALESCE(cardinality($1::bpchar[]), 0) = 0 + THEN (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) + ELSE COALESCE(oc.packet_count, 0) END::bigint AS packet_count, + CASE WHEN COALESCE(cardinality($1::bpchar[]), 0) = 0 + THEN (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) + ELSE COALESCE(oc.observer_count, 0) END::bigint AS observer_count, + CASE WHEN COALESCE(cardinality($1::bpchar[]), 0) = 0 + THEN (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) + ELSE COALESCE(nc.node_count, 0) END::bigint AS node_count FROM transport_scopes ts +LEFT JOIN observation_counts oc ON oc.scope_id = ts.id +LEFT JOIN node_counts nc ON nc.scope_id = ts.id ORDER BY ts.name ` @@ -1434,10 +1457,11 @@ type GetScopeStatsRow struct { NodeCount int64 `json:"node_count"` } -// Count each table on its own; the old cross-join blew up to millions of rows -// before COUNT(DISTINCT) (~10s). -func (q *Queries) GetScopeStats(ctx context.Context) ([]GetScopeStatsRow, error) { - rows, err := q.db.Query(ctx, getScopeStats) +// Aggregate matching observations once, separately from node memberships to avoid +// a cross-join. Empty IATAs keep the original global counts, including associations +// whose observations have expired; the filtered aggregates are empty in that case. +func (q *Queries) GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) { + rows, err := q.db.Query(ctx, getScopeStats, iatas) if err != nil { return nil, err } diff --git a/db/stats.go b/db/stats.go index b8dc915c..abceda1d 100644 --- a/db/stats.go +++ b/db/stats.go @@ -243,8 +243,8 @@ func (s *Store) GetRadioPresets(ctx context.Context, preset string, iatas []stri return items, nil } -func (s *Store) GetScopeStats(ctx context.Context) ([]api.ScopeStats, error) { - rows, err := s.q.GetScopeStats(ctx) +func (s *Store) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { + rows, err := s.q.GetScopeStats(ctx, iatas) if err != nil { return nil, err } diff --git a/db/stats_test.go b/db/stats_test.go index 70518705..e5843113 100644 --- a/db/stats_test.go +++ b/db/stats_test.go @@ -227,13 +227,13 @@ func TestGetScopeStats(t *testing.T) { mock := mockdb.NewMockQuerier(ctrl) mock.EXPECT(). - GetScopeStats(gomock.Any()). + GetScopeStats(gomock.Any(), []string{"YVR"}). Return([]sqlc.GetScopeStatsRow{ {Name: "default", PacketCount: 100, ObserverCount: 5, NodeCount: 20}, }, nil) store := &Store{q: mock} - items, err := store.GetScopeStats(context.Background()) + items, err := store.GetScopeStats(context.Background(), []string{"YVR"}) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/docs/docs.go b/docs/docs.go index b7cf2acd..6505b41e 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1988,6 +1988,7 @@ const docTemplate = `{ }, "/stats/scopes": { "get": { + "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.", "produces": [ "application/json" ], @@ -1995,6 +1996,32 @@ const docTemplate = `{ "Stats" ], "summary": "Scope statistics", + "parameters": [ + { + "type": "string", + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "string", + "description": "Single IATA code; used when iatas is absent", + "name": "iata", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs", + "name": "region", + "in": "query" + } + ], "responses": { "200": { "description": "OK", @@ -2005,6 +2032,12 @@ const docTemplate = `{ } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { diff --git a/docs/swagger.json b/docs/swagger.json index 5182f729..4d470303 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1986,6 +1986,7 @@ }, "/stats/scopes": { "get": { + "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.", "produces": [ "application/json" ], @@ -1993,6 +1994,32 @@ "Stats" ], "summary": "Scope statistics", + "parameters": [ + { + "type": "string", + "description": "Comma-separated IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "string", + "description": "Single IATA code; used when iatas is absent", + "name": "iata", + "in": "query" + }, + { + "type": "integer", + "description": "Filter by region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs", + "name": "region", + "in": "query" + } + ], "responses": { "200": { "description": "OK", @@ -2003,6 +2030,12 @@ } } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, "500": { "description": "Internal Server Error", "schema": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 64962a77..c3fb6072 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2509,6 +2509,28 @@ paths: - Stats /stats/scopes: get: + description: Counts each packet, observer and node once per scope. IATA filters + use retained observations for packets/observers and node IATA memberships + for nodes. Without filters, returns global totals. Scopes with zero matching + counts remain listed; an empty region returns an empty array. + parameters: + - description: Comma-separated IATA codes + in: query + name: iatas + type: string + - description: Single IATA code; used when iatas is absent + in: query + name: iata + type: string + - description: Filter by region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Filter by region slug, expands to member IATAs; combined with + explicit IATAs + in: query + name: region + type: string produces: - application/json responses: @@ -2518,6 +2540,10 @@ paths: items: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats' type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' "500": description: Internal Server Error schema: diff --git a/internal/api/handlers/scope_stats_test.go b/internal/api/handlers/scope_stats_test.go new file mode 100644 index 00000000..d63ba21b --- /dev/null +++ b/internal/api/handlers/scope_stats_test.go @@ -0,0 +1,112 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "slices" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func TestGetStatsScopes_RegionErrors(t *testing.T) { + for _, query := range []string{"regionId=invalid", "regionId=2147483648", "regionId=999", "region=missing"} { + t.Run(query, func(t *testing.T) { + reader := stubReader{ + getRegion: func(context.Context, int32) (*api.Region, error) { + return nil, errors.New("not found") + }, + getRegionBySlug: func(context.Context, string) (*api.Region, error) { + return nil, errors.New("not found") + }, + } + w := httptest.NewRecorder() + getStatsScopes(reader)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+query, nil)) + if w.Code != http.StatusBadRequest { + t.Fatalf("got status %d, want 400", w.Code) + } + }) + } +} + +func TestGetStatsScopes_Filters(t *testing.T) { + for _, tc := range []struct { + query string + want []string + empty bool + }{ + {"", nil, false}, + {"iata=yvr", []string{"YVR"}, false}, + {"iatas=yyj,%20yvr,yyj", []string{"YYJ", "YVR", "YYJ"}, false}, + {"iatas=YVR&iata=YYZ", []string{"YVR"}, false}, + {"regionId=7", []string{"YVR", "YYJ"}, false}, + {"region=west", []string{"YVR", "YYJ"}, false}, + {"regionId=7®ion=missing", []string{"YVR", "YYJ"}, false}, + {"iatas=YYZ,YVR®ion=west", []string{"YYZ", "YVR", "YVR", "YYJ"}, false}, + {"region=empty", nil, true}, + {"region=empty&iata=YVR", []string{"YVR"}, false}, + } { + t.Run(tc.query, func(t *testing.T) { + called := false + reader := stubReader{ + getRegion: func(_ context.Context, id int32) (*api.Region, error) { + if id != 7 { + t.Fatalf("unexpected region ID %d", id) + } + return &api.Region{IATAs: []string{"YVR", "YYJ"}}, nil + }, + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + if slug != "west" { + t.Fatalf("unexpected region slug %s", slug) + } + return &api.Region{IATAs: []string{"YVR", "YYJ"}}, nil + }, + getScopeStats: func(_ context.Context, iatas []string) ([]api.ScopeStats, error) { + called = true + if !slices.Equal(iatas, tc.want) { + t.Fatalf("IATAs = %v, want %v", iatas, tc.want) + } + return []api.ScopeStats{{Name: "#test", PacketCount: 2, ObserverCount: 1, NodeCount: 3}}, nil + }, + } + w := httptest.NewRecorder() + getStatsScopes(reader)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + if called == tc.empty { + t.Fatalf("reader called = %v, empty region = %v", called, tc.empty) + } + var rows []api.ScopeStats + if err := json.Unmarshal(w.Body.Bytes(), &rows); err != nil { + t.Fatal(err) + } + if tc.empty { + if rows == nil || len(rows) != 0 { + t.Fatalf("empty region returned %s", w.Body.String()) + } + } else if len(rows) != 1 || rows[0].PacketCount != 2 || rows[0].ObserverCount != 1 || rows[0].NodeCount != 3 { + t.Fatalf("response changed: %+v", rows) + } + }) + } +} + +func TestGetStatsScopes_ReaderError(t *testing.T) { + w := httptest.NewRecorder() + getStatsScopes(stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) { + return nil, errors.New("database unavailable") + }})(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil)) + if w.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", w.Code) + } +} diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index b59a3031..abd6c4df 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -414,14 +414,33 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // getStatsScopes godoc // // @Summary Scope statistics +// @Description Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array. // @Tags Stats // @Produce json +// @Param iatas query string false "Comma-separated IATA codes" +// @Param iata query string false "Single IATA code; used when iatas is absent" +// @Param regionId query int false "Filter by region ID, expands to member IATAs" +// @Param region query string false "Filter by region slug, expands to member IATAs; combined with explicit IATAs" // @Success 200 {object} []api.ScopeStats +// @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /stats/scopes [get] func getStatsScopes(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - stats, err := reader.GetScopeStats(r.Context()) + iatas := parseIATAs(r) + if regionID := r.URL.Query().Get("regionId"); regionID != "" || r.URL.Query().Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(r.Context(), regionID, r.URL.Query().Get("region"), reader) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + respond(w, http.StatusOK, []api.ScopeStats{}) + return + } + } + stats, err := reader.GetScopeStats(r.Context(), iatas) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/stats_test.go b/internal/api/handlers/stats_test.go index fe08c8bf..8225fc8c 100644 --- a/internal/api/handlers/stats_test.go +++ b/internal/api/handlers/stats_test.go @@ -282,7 +282,7 @@ func TestGetStatsRadioPresets_OK(t *testing.T) { func TestGetStatsScopes_OK(t *testing.T) { r := chi.NewRouter() r.Get("/stats/scopes", getStatsScopes(stubReader{ - getScopeStats: func(_ context.Context) ([]api.ScopeStats, error) { + getScopeStats: func(_ context.Context, _ []string) ([]api.ScopeStats, error) { return []api.ScopeStats{{Name: "#bc", PacketCount: 100}}, nil }, })) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 4cd3d87a..d87076e6 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -50,7 +50,7 @@ type stubReader struct { getStatsTopAdvertisers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) getStatsClockDrift func(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) getStatsTopTalkers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) - getScopeStats func(ctx context.Context) ([]api.ScopeStats, error) + getScopeStats func(ctx context.Context, iatas []string) ([]api.ScopeStats, error) getStatsNodeTypes func(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) getScopeNames func(ctx context.Context) ([]string, error) getScopesByIATAs func(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) @@ -303,9 +303,9 @@ func (s stubReader) GetStatsTopTalkers(ctx context.Context, iatas []string, sinc return nil, nil } -func (s stubReader) GetScopeStats(ctx context.Context) ([]api.ScopeStats, error) { +func (s stubReader) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { if s.getScopeStats != nil { - return s.getScopeStats(ctx) + return s.getScopeStats(ctx, iatas) } return nil, nil } diff --git a/internal/api/reader.go b/internal/api/reader.go index dc3e1dc5..75febdc7 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -188,7 +188,8 @@ type Reader interface { GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]TopTalker, error) // GetScopeStats returns aggregate packet, observer and node counts per transport scope. - GetScopeStats(ctx context.Context) ([]ScopeStats, error) + // Pass empty iatas for global totals. IATAs must be uppercase. + GetScopeStats(ctx context.Context, iatas []string) ([]ScopeStats, error) // GetStatsNodeTypes returns node counts grouped by type, optionally filtered by IATA. GetStatsNodeTypes(ctx context.Context, iatas []string) ([]NodeTypeCount, error) diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 68c968b7..cada469b 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -47,8 +47,10 @@ func (s *stubReader) GetRegion(_ context.Context, _ int32) (*api.Region, error) func (s *stubReader) GetRegionBySlug(_ context.Context, _ string) (*api.Region, error) { return nil, nil } -func (s *stubReader) GetScopeNames(_ context.Context) ([]string, error) { return nil, nil } -func (s *stubReader) GetScopeStats(_ context.Context) ([]api.ScopeStats, error) { return nil, nil } +func (s *stubReader) GetScopeNames(_ context.Context) ([]string, error) { return nil, nil } +func (s *stubReader) GetScopeStats(_ context.Context, _ []string) ([]api.ScopeStats, error) { + return nil, nil +} func (s *stubReader) GetScopesByIATAs(_ context.Context, _ []string) ([]api.ScopeSummary, error) { return nil, nil } diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 4c69bd27..ba5ce8e2 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "slices" "sort" "strings" "time" @@ -141,9 +142,15 @@ func (cr *CachedReader) GetScopeNames(ctx context.Context) ([]string, error) { } // GetScopeStats implements [api.Reader]. -func (cr *CachedReader) GetScopeStats(ctx context.Context) ([]api.ScopeStats, error) { - return getOrSet(ctx, cr.c, keyScopeStats, cr.ttl.Reference, func() ([]api.ScopeStats, error) { - return cr.inner.GetScopeStats(ctx) +func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { + segment := "all" + if len(iatas) > 0 { + sorted := append([]string(nil), iatas...) + sort.Strings(sorted) + segment = strings.Join(slices.Compact(sorted), ",") + } + return getOrSet(ctx, cr.c, keyScopeStats+":"+segment, cr.ttl.Reference, func() ([]api.ScopeStats, error) { + return cr.inner.GetScopeStats(ctx, iatas) }) } diff --git a/internal/cache/scope_stats_test.go b/internal/cache/scope_stats_test.go new file mode 100644 index 00000000..188a5b19 --- /dev/null +++ b/internal/cache/scope_stats_test.go @@ -0,0 +1,59 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type scopeStatsReader struct { + api.Reader + calls int64 +} + +func (r *scopeStatsReader) GetScopeStats(context.Context, []string) ([]api.ScopeStats, error) { + r.calls++ + return []api.ScopeStats{{Name: "#test", PacketCount: r.calls}}, nil +} + +func TestScopeStatsCacheSeparatesIATAs(t *testing.T) { + c, _ := newTestClient(t) + inner := &scopeStatsReader{} + reader := NewCachedReader(inner, c, CacheTTLs{Reference: time.Minute}) + for _, tc := range []struct { + iatas []string + want int64 + }{ + {nil, 1}, + {[]string{"YVR"}, 2}, + {[]string{"YYJ"}, 3}, + {[]string{"YYJ", "YVR"}, 4}, + {[]string{"YVR", "YYJ", "YVR"}, 4}, + {[]string{"YVR"}, 2}, + {[]string{}, 1}, + {[]string{"ZZZ"}, 5}, + {[]string{""}, 6}, + {nil, 1}, + } { + before := slices.Clone(tc.iatas) + rows, err := reader.GetScopeStats(context.Background(), tc.iatas) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].PacketCount != tc.want { + t.Fatalf("IATAs %v: got %+v, want cached count %d", tc.iatas, rows, tc.want) + } + if !slices.Equal(before, tc.iatas) { + t.Fatalf("caller IATAs mutated: %v -> %v", before, tc.iatas) + } + } + if inner.calls != 6 { + t.Fatalf("underlying calls = %d, want 6", inner.calls) + } +} From 85b1c3961d7f4f92c19af5ac04482bd104b9d434 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 18 Sep 2026 06:18:19 -0400 Subject: [PATCH 034/141] feat(packets): populate advert summaries in history and live events (#143) --- db/packet_summary_integration_test.go | 116 +++++++++++++++++++++++++ db/packets.go | 9 ++ db/queries/queries.sql | 6 ++ db/sqlc/queries.sql.go | 12 +++ docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 2 +- internal/api/packets.go | 2 +- internal/ingest/packet.go | 6 ++ internal/ingest/packet_summary_test.go | 116 +++++++++++++++++++++++++ 10 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 db/packet_summary_integration_test.go create mode 100644 internal/ingest/packet_summary_test.go diff --git a/db/packet_summary_integration_test.go b/db/packet_summary_integration_test.go new file mode 100644 index 00000000..7f3003d0 --- /dev/null +++ b/db/packet_summary_integration_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +type summaryQueryCounter struct { + pgx.Tx + calls int +} + +func (c *summaryQueryCounter) Query(ctx context.Context, query string, args ...any) (pgx.Rows, error) { + c.calls++ + return c.Tx.Query(ctx, query, args...) +} + +func TestPacketSummariesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"packets", "packet_observations", "observers", "transport_scopes"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,parsed_payload,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(id),2,'0'),'hex'),kind,0,1,'\x00','\x00',parsed::jsonb, + '2026-01-01'::timestamptz+id*interval '1 second','2026-01-01'::timestamptz+id*interval '1 second' +FROM (VALUES + (1,4,'{"type":"ADVERT","appData":{"name":"MD00-Repeater"}}'), + (2,4,'{"type":"ADVERT","appData":{"name":"Relay 📡"}}'), + (3,4,'{"type":"ADVERT","appData":{"name":""}}'), + (4,4,'{"type":"ADVERT","appData":{}}'), + (5,4,NULL), + (6,4,'{"type":"ADVERT","appData":{"name":null}}'), + (7,4,'{"type":"ADVERT","appData":{"name":42}}'), + (8,2,'{"appData":{"name":"not an advert"}}'), + (9,4,'{"type":"ADVERT","appData":[]}') +) v(id,kind,parsed); +`) + if err != nil { + t.Fatal(err) + } + counter := &summaryQueryCounter{Tx: tx} + store := &Store{q: sqlc.New(counter)} + check := func(items []api.PacketSummary) { + t.Helper() + if len(items) != 9 { + t.Fatalf("got %d packets, want 9", len(items)) + } + for _, item := range items { + want := map[string]string{"01": "MD00-Repeater", "02": "Relay 📡"}[item.PacketHash] + if want == "" { + if item.Summary != nil { + t.Fatalf("unexpected summary for %s: %q", item.PacketHash, *item.Summary) + } + } else if item.Summary == nil || *item.Summary != want { + t.Fatalf("summary for %s = %v, want %q", item.PacketHash, item.Summary, want) + } + } + if counter.calls != 1 { + t.Fatalf("list made %d queries, want 1", counter.calls) + } + counter.calls = 0 + } + page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 20) + if err != nil { + t.Fatal(err) + } + check(page.Items) // names remain available before any observation arrives + _, err = tx.Exec(ctx, ` +INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT id,decode(lpad(to_hex(id),2,'0'),'hex'),'00000000-0000-0000-0000-000000000001','YVR', + '2026-01-01'::timestamptz+id*interval '1 second',0,1,0 FROM generate_series(1,9) id; +`) + if err != nil { + t.Fatal(err) + } + for _, iatas := range [][]string{nil, {"YVR"}} { + page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 20) + if err != nil { + t.Fatal(err) + } + check(page.Items) + rows, err := store.ListPacketsAfterID(ctx, 0, -1, -1, iatas, "", 20) + if err != nil { + t.Fatal(err) + } + check(rows) + } +} diff --git a/db/packets.go b/db/packets.go index ddf0ad22..2ea2c716 100644 --- a/db/packets.go +++ b/db/packets.go @@ -137,6 +137,9 @@ func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int1 LastHeardAt: v.LastHeardAt.Time.UnixMilli(), ObservationCount: int32(v.ObservationCount), } + if v.Summary != "" { + item.Summary = &v.Summary + } if v.LatestObserverID != (uuid.UUID{}) { endpoints, _ := decodePacketEndpointSnapshot(v.LatestObserverResolvedEndpoints) item.LatestObserver = &api.PacketLatestObserver{ @@ -219,6 +222,9 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes LastHeardAt: v.LastHeardAt.Time.UnixMilli(), ObservationCount: int32(v.ObservationCount), } + if v.Summary != "" { + item.Summary = &v.Summary + } if v.LatestObserverID != (uuid.UUID{}) { endpoints, _ := decodePacketEndpointSnapshot(v.LatestObserverResolvedEndpoints) item.LatestObserver = &api.PacketLatestObserver{ @@ -278,6 +284,9 @@ func (s *Store) ListPacketsAfterID(ctx context.Context, afterObservationID int64 LastHeardAt: v.LastHeardAt.Time.UnixMilli(), ObservationCount: int32(v.ObservationCount), } + if v.Summary != "" { + item.Summary = &v.Summary + } if v.LatestObserverID != (uuid.UUID{}) { endpoints, _ := decodePacketEndpointSnapshot(v.LatestObserverResolvedEndpoints) item.LatestObserver = &api.PacketLatestObserver{ diff --git a/db/queries/queries.sql b/db/queries/queries.sql index bdc9c916..7ca704d2 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -463,6 +463,8 @@ SELECT COUNT(*) FROM packet_observations WHERE packet_hash = $1; SELECT p.packet_hash, p.payload_type, + COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, @@ -558,6 +560,8 @@ page AS ( SELECT p.packet_hash, p.payload_type, + COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, @@ -596,6 +600,8 @@ ORDER BY sh.site_heard_at DESC; SELECT p.packet_hash, p.payload_type, + COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index ae0b9d6b..e912b9f1 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -3157,6 +3157,8 @@ const listPackets = `-- name: ListPackets :many SELECT p.packet_hash, p.payload_type, + COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, @@ -3206,6 +3208,7 @@ type ListPacketsParams struct { type ListPacketsRow struct { PacketHash []byte `json:"packet_hash"` PayloadType int16 `json:"payload_type"` + Summary string `json:"summary"` RouteType int16 `json:"route_type"` FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` @@ -3245,6 +3248,7 @@ func (q *Queries) ListPackets(ctx context.Context, arg ListPacketsParams) ([]Lis if err := rows.Scan( &i.PacketHash, &i.PayloadType, + &i.Summary, &i.RouteType, &i.FirstHeardAt, &i.LastHeardAt, @@ -3274,6 +3278,8 @@ const listPacketsAfterID = `-- name: ListPacketsAfterID :many SELECT p.packet_hash, p.payload_type, + COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, @@ -3312,6 +3318,7 @@ type ListPacketsAfterIDParams struct { type ListPacketsAfterIDRow struct { PacketHash []byte `json:"packet_hash"` PayloadType int16 `json:"payload_type"` + Summary string `json:"summary"` RouteType int16 `json:"route_type"` FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` @@ -3348,6 +3355,7 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID if err := rows.Scan( &i.PacketHash, &i.PayloadType, + &i.Summary, &i.RouteType, &i.FirstHeardAt, &i.LastHeardAt, @@ -3419,6 +3427,8 @@ page AS ( SELECT p.packet_hash, p.payload_type, + COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, @@ -3467,6 +3477,7 @@ type ListPacketsByIATAsParams struct { type ListPacketsByIATAsRow struct { PacketHash []byte `json:"packet_hash"` PayloadType int16 `json:"payload_type"` + Summary string `json:"summary"` RouteType int16 `json:"route_type"` FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` @@ -3521,6 +3532,7 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs if err := rows.Scan( &i.PacketHash, &i.PayloadType, + &i.Summary, &i.RouteType, &i.FirstHeardAt, &i.LastHeardAt, diff --git a/docs/docs.go b/docs/docs.go index 6505b41e..d65c73e3 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -3568,7 +3568,7 @@ const docTemplate = `{ "type": "string" }, "summary": { - "description": "human-readable payload summary", + "description": "advert name from this packet; omitted when unavailable or unsupported", "type": "string" } } diff --git a/docs/swagger.json b/docs/swagger.json index 4d470303..f1bc16ff 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -3566,7 +3566,7 @@ "type": "string" }, "summary": { - "description": "human-readable payload summary", + "description": "advert name from this packet; omitted when unavailable or unsupported", "type": "string" } } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index c3fb6072..e1c5dfe3 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -813,7 +813,7 @@ definitions: description: matched transport scope name e.g. "#bc" type: string summary: - description: human-readable payload summary + description: advert name from this packet; omitted when unavailable or unsupported type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.PacketTransportCodes: diff --git a/internal/api/packets.go b/internal/api/packets.go index c75aa6f6..bb4ac37e 100644 --- a/internal/api/packets.go +++ b/internal/api/packets.go @@ -40,7 +40,7 @@ type PacketSummary struct { LastHeardAt int64 `json:"lastHeardAt"` // epoch ms ObservationCount int32 `json:"observationCount"` LatestObserver *PacketLatestObserver `json:"latestObserver,omitempty"` - Summary *string `json:"summary,omitempty"` // human-readable payload summary + Summary *string `json:"summary,omitempty"` // advert name from this packet; omitted when unavailable or unsupported } // PacketPathLength is the decoded path_length byte from a packet observation. diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index be8d007c..59408def 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -81,6 +81,7 @@ type packetObservationEvent struct { IsFirstObservation bool `json:"isFirstObservation"` ObservationCount int64 `json:"observationCount"` Scope *string `json:"scope,omitempty"` + Summary *string `json:"summary,omitempty"` // same advert name as REST list/backfill rows } `json:"packet"` Observation struct { ObserverID string `json:"observerId"` @@ -343,6 +344,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ var channelHash []byte originPubkey := []byte(nil) var parsedPayload json.RawMessage + var summary *string var traceTag []byte // For PayloadTypeTrace, packet.Path holds one SNR byte per hop (not hashes -- see // below), so the "physical route" hashes for resolvedPath/known-route purposes come @@ -415,6 +417,9 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if hasName { n := strings.ToValidUTF8(appData.Name, "\uFFFD") name = &n + if n != "" { + summary = name + } } deviceRole := int(flags & 0x0F) @@ -892,6 +897,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ evt.Packet.RouteType = packet.RouteType() evt.Packet.RouteTypeName = api.RouteTypeName(int16(packet.RouteType())) evt.Packet.IsFirstObservation = isNew + evt.Packet.Summary = summary evt.Observation.ObserverID = id.String() evt.Observation.ObserverName = observerName evt.Observation.IATA = iata diff --git a/internal/ingest/packet_summary_test.go b/internal/ingest/packet_summary_test.go new file mode 100644 index 00000000..07003355 --- /dev/null +++ b/internal/ingest/packet_summary_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/meshcore-go/meshcore-go" +) + +// The hub has no subscription acknowledgement. Wait for a probe to make it +// through before ingesting the test packet, rather than assuming a short sleep. +func waitForSummarySubscriber(t *testing.T, ctx context.Context, h *hub.Hub, client *hub.Client) { + t.Helper() + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + for { + h.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + select { + case <-client.Send: + return + case <-ticker.C: + case <-ctx.Done(): + t.Fatal("hub subscription did not become ready") + } + } +} + +func TestAdvertSummaryLive(t *testing.T) { + for _, tc := range []struct { + name string + hasName, tamper bool + want string + }{ + {"MD00-Repeater", true, false, "MD00-Repeater"}, + {"Relay 📡", true, false, "Relay 📡"}, + {"R\xffP", true, false, "R\uFFFDP"}, + {"", true, false, ""}, + {"", false, false, ""}, + {"unverified", true, true, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Run("wire", func(t *testing.T) { + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} + w.db = db + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + w.hub.AddScope(client, "summary", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + + waitForSummarySubscriber(t, ctx, w.hub, client) + defer w.hub.Remove(client) + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + identity, err := meshcore.NewIdentityFromBytes(pub) + if err != nil { + t.Fatal(err) + } + flags := byte(meshcore.AdvertTypeRepeater) + if tc.hasName { + flags |= 0x80 + } + advert := &meshcore.Advert{PublicKey: identity, Timestamp: 12345, RawAppData: append([]byte{flags}, []byte(tc.name)...)} + advert.Sign(priv) + if tc.tamper { + advert.RawAppData[0] ^= 1 + } + payload, err := advert.ToBytes() + if err != nil { + t.Fatal(err) + } + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeAdvert, 0), Payload: payload} + w.handlePacket(ctx, "YVR", "0102", packetEnvelope(t, packet)) + + for { + select { + case event := <-client.Send: + if event.Type != hub.EventPacketObservation { + continue + } + for _, raw := range []json.RawMessage{event.Payload, event.PayloadResolved} { + var got struct { + Packet struct { + Summary *string `json:"summary"` + } `json:"packet"` + } + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + if tc.want == "" { + if got.Packet.Summary != nil { + t.Fatalf("unexpected summary %q", *got.Packet.Summary) + } + } else if got.Packet.Summary == nil || *got.Packet.Summary != tc.want { + t.Fatalf("live summary = %v, want %q", got.Packet.Summary, tc.want) + } + } + return + case <-ctx.Done(): + t.Fatal("packet observation event missing") + } + } + }) + }) + } +} From 93f85cab510a1d83a6f34eba14096457fc1a3981 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 18 Sep 2026 06:18:27 -0400 Subject: [PATCH 035/141] feat(auth): protect the admin subtree with a bearer key (#146) --- README.md | 16 ++++++ cmd/beacon/main.go | 2 +- config.yaml.example | 5 ++ env.example | 3 ++ internal/api/middleware/auth.go | 39 +++++++++++---- internal/api/middleware/auth_test.go | 75 ++++++++++++++++++++++++++++ internal/api/router/auth_test.go | 56 +++++++++++++++++++++ internal/api/router/router.go | 20 +++----- internal/api/router/router_test.go | 2 +- internal/config/auth_test.go | 54 ++++++++++++++++++++ internal/config/config.go | 11 ++++ 11 files changed, 258 insertions(+), 25 deletions(-) create mode 100644 internal/api/middleware/auth_test.go create mode 100644 internal/api/router/auth_test.go create mode 100644 internal/config/auth_test.go diff --git a/README.md b/README.md index 7fd773f8..0d3d7c7d 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,22 @@ matching a now-known channel and decrypts them. Watch the startup log for ## Configuration +### Admin authentication + +The `/api/v1/admin` subtree requires `Authorization: Bearer `. Set the +operator key with `BEACON_API_KEY` or `auth.api_key` in YAML. A set environment +variable overrides YAML; an explicitly empty value disables admin access. +With no key, admin requests return JSON 503 while public reads and WebSockets +continue normally. With a key, missing, incorrect or duplicate Authorization +headers return JSON 401 with `WWW-Authenticate: Bearer`. + +Admin operations are not implemented yet: a valid key currently reaches a 404. +Global CORS preflights remain public. Use a long, randomly generated key, keep +it out of source control and logs, and send it only in the Authorization header, +never the URL or request body. Require HTTPS at the reverse proxy and restrict +direct access to Beacon's HTTP listener to that proxy or a private connection. +Changing the key requires a restart. No API key is issued automatically. + ### Environment variables (`.env`) | Variable | Default | Description | diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index bb31b2ab..43e6168c 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -275,7 +275,7 @@ func main() { go scheduler.Start(ctx) // ── HTTP server ────────────────────────────────────────────────────────── - r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, cfg.CORS, cfg.Server) + r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, cfg.CORS, cfg.Server, cfg.Auth) srv := &http.Server{ Addr: addr, diff --git a/config.yaml.example b/config.yaml.example index 439c51ac..c07852f3 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,6 +1,11 @@ # Beacon configuration file # Copy to config.yaml and adjust as needed. +# Public reads stay available. No key means admin routes return 503. +# Prefer BEACON_API_KEY in the service environment; never commit a real key. +auth: + api_key: "" + server: # Only these direct proxy peers may set the client IP through X-Real-IP. # The proxy must overwrite that header, not pass through client input. diff --git a/env.example b/env.example index 96bf825d..3594554b 100644 --- a/env.example +++ b/env.example @@ -1,5 +1,8 @@ LISTEN_ADDR=:8080 +# Optional admin bearer key. If set, overrides auth.api_key (even when empty). +# BEACON_API_KEY= + POSTGRES_DSN=postgres://beacon:beacon@localhost:5432/beacon?sslmode=disable # Redis (optional — leave REDIS_ADDR unset to disable caching) diff --git a/internal/api/middleware/auth.go b/internal/api/middleware/auth.go index efa49c69..3e775f3d 100644 --- a/internal/api/middleware/auth.go +++ b/internal/api/middleware/auth.go @@ -3,17 +3,36 @@ package middleware -import "net/http" +import ( + "crypto/subtle" + "io" + "net/http" + "strings" +) -// NoopAuth is a placeholder for the authentication middleware that will be -// wired onto the private route group when auth is implemented (see Future -// Features → Admin authentication in the design doc). -// -// Replace this with a real JWT/session validation middleware before shipping -// any write endpoints or admin functionality. -func NoopAuth(next http.Handler) http.Handler { +// BearerAuth protects admin routes with one operator key. An empty key disables +// access; public routes must be mounted outside this middleware. +func BearerAuth(apiKey string, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // TODO: validate bearer token, set user in context, return 401 on failure. - next.ServeHTTP(w, r) + w.Header().Set("Cache-Control", "no-store") + status := http.StatusServiceUnavailable + body := `{"error":{"code":"service_unavailable","message":"admin authentication is not configured"}}` + if apiKey != "" { + if values := r.Header.Values("Authorization"); len(values) == 1 { + scheme, token, found := strings.Cut(values[0], " ") + token = strings.TrimLeft(token, " ") // Bearer permits one or more spaces. + if found && strings.EqualFold(scheme, "Bearer") && token != "" && !strings.ContainsAny(token, " \t\r\n,") && + subtle.ConstantTimeCompare([]byte(token), []byte(apiKey)) == 1 { + next.ServeHTTP(w, r) + return + } + } + status = http.StatusUnauthorized + body = `{"error":{"code":"unauthorized","message":"valid bearer token required"}}` + w.Header().Set("WWW-Authenticate", "Bearer") + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, body+"\n") }) } diff --git a/internal/api/middleware/auth_test.go b/internal/api/middleware/auth_test.go new file mode 100644 index 00000000..e394d923 --- /dev/null +++ b/internal/api/middleware/auth_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestBearerAuth(t *testing.T) { + const key = "synthetic-test-key" + for _, tc := range []struct { + name, key string + headers []string + status int + }{ + {"disabled", "", nil, 503}, {"disabled with token", "", []string{"Bearer " + key}, 503}, + {"missing", key, nil, 401}, {"empty", key, []string{""}, 401}, + {"wrong scheme", key, []string{"Basic " + key}, 401}, {"wrong token", key, []string{"Bearer wrong"}, 401}, + {"empty token", key, []string{"Bearer "}, 401}, {"no separator", key, []string{"Bearer" + key}, 401}, + {"tab separator", key, []string{"Bearer\t" + key}, 401}, {"extra token", key, []string{"Bearer " + key + " extra"}, 401}, + {"line break", key, []string{"Bearer " + key + "\r\n"}, 401}, + {"duplicate", key, []string{"Bearer " + key, "Bearer " + key}, 401}, + {"combined", key, []string{"Bearer " + key + ", Bearer " + key}, 401}, + {"case sensitive token", key, []string{"Bearer SYNTHETIC-TEST-KEY"}, 401}, + {"valid", key, []string{"Bearer " + key}, 200}, + {"scheme case", key, []string{"bEaReR " + key}, 200}, + {"multiple spaces", key, []string{"Bearer " + key}, 200}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + handler := BearerAuth(tc.key, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.WriteHeader(200) + })) + // Query and form tokens alone must never grant access. + req := httptest.NewRequest("POST", "/admin?access_token="+key, strings.NewReader("access_token="+key)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + for _, value := range tc.headers { + req.Header.Add("Authorization", value) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != tc.status || (calls == 1) != (tc.status == 200) { + t.Fatalf("status=%d downstream calls=%d", w.Code, calls) + } + if w.Header().Get("Cache-Control") != "no-store" || strings.Contains(w.Body.String(), key) { + t.Fatal("cache policy missing or credential reflected") + } + if tc.status == 200 { + return + } + var body struct { + Error struct{ Code, Message string } + } + if w.Header().Get("Content-Type") != "application/json" || json.Unmarshal(w.Body.Bytes(), &body) != nil || body.Error.Message == "" { + t.Fatal("invalid JSON error contract") + } + wantCode := "service_unavailable" + if tc.status == 401 { + wantCode = "unauthorized" + if w.Header().Get("WWW-Authenticate") != "Bearer" { + t.Fatal("missing bearer challenge") + } + } + if body.Error.Code != wantCode { + t.Fatalf("error code=%s", body.Error.Code) + } + }) + } +} diff --git a/internal/api/router/auth_test.go b/internal/api/router/auth_test.go new file mode 100644 index 00000000..9c66f6d9 --- /dev/null +++ b/internal/api/router/auth_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestAdminAuthBoundary(t *testing.T) { + for _, key := range []string{"", "synthetic-test-key"} { + handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{APIKey: key}) + for _, path := range []string{"/api/v1/admin", "/api/v1/admin/", "/api/v1/admin/config", "/api/v1/admin//config"} { + for _, method := range []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"} { + for _, token := range []string{"", "wrong", "synthetic-test-key"} { + req := httptest.NewRequest(method, path, nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + want := http.StatusServiceUnavailable + if key != "" { + want = 401 + if token == key { + want = 404 + } + } + if w.Code != want { + t.Fatalf("%s %s status=%d want=%d", method, path, w.Code, want) + } + } + } + } + for path, want := range map[string]int{"/api/v1/brokers": 200, "/api/v1/administrator": 404, "/swagger": 301} { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest("GET", path, nil)) + if w.Code != want { + t.Fatalf("public %s status=%d want=%d", path, w.Code, want) + } + } + req := httptest.NewRequest("OPTIONS", "/api/v1/admin/config", nil) + req.Header.Set("Origin", "https://example.test") + req.Header.Set("Access-Control-Request-Method", "GET") + req.Header.Set("Access-Control-Request-Headers", "Authorization") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + if w.Code != 200 || w.Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("CORS preflight blocked") + } + } +} diff --git a/internal/api/router/router.go b/internal/api/router/router.go index 5f4852a7..a2f5d777 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -3,8 +3,7 @@ // Package router wires all HTTP routes onto the Chi router and injects // dependencies (hub, reader, ingest workers) into the handler closures. -// All routes are mounted under /api/v1 with public and private groups -// stubbed for future auth middleware. +// REST routes are mounted under /api/v1; its admin subtree requires a bearer key. package router import ( @@ -39,9 +38,8 @@ import ( // /regions → regions subrouter // /stats → stats subrouter // -// The private group is stubbed and ready for the auth middleware drop-in -// described in Future Features → Admin authentication. -func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP int, corsCfg config.CORSConfig, serverCfg config.ServerConfig) http.Handler { +// Admin handlers are added separately; the reserved subtree is protected now. +func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP int, corsCfg config.CORSConfig, serverCfg config.ServerConfig, authCfg config.AuthConfig) http.Handler { r := chi.NewRouter() // ── CORS ───────────────────────────────────────────────────────────────── @@ -91,7 +89,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP // ── Public REST API (v1) ───────────────────────────────────────────────── r.Route("/api/v1", func(r chi.Router) { - // Public group — no authentication required (all of v1 is public). + // Public group — no authentication required. r.Group(func(r chi.Router) { r.Mount("/packets", handlers.PacketsRouter(reader)) r.Mount("/nodes", handlers.NodesRouter(reader)) @@ -107,13 +105,9 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP r.Mount("/traces", handlers.TracesRouter(reader)) }) - // Private group — auth middleware applied. - // Stubbed for the admin endpoints described in Future Features. - // Swap mw.NoopAuth for a real JWT/session middleware when ready. - r.Group(func(r chi.Router) { - r.Use(mw.NoopAuth) - // r.Mount("/admin", handlers.AdminRouter()) - }) + // Protect the entire subtree, including its root and unknown paths. + // Replace the empty router with the admin handlers when they are added. + r.Mount("/admin", mw.BearerAuth(authCfg.APIKey, chi.NewRouter())) }) return r diff --git a/internal/api/router/router_test.go b/internal/api/router/router_test.go index 9a42eb3e..6b8c5a22 100644 --- a/internal/api/router/router_test.go +++ b/internal/api/router/router_test.go @@ -36,7 +36,7 @@ func TestWebSocketLimitUsesTrustedClientIP(t *testing.T) { func checkWebSocketLimit(t *testing.T, cfg config.ServerConfig, secondIP string, wantStatus int) { t.Helper() - server := httptest.NewServer(New(hub.New(), nil, nil, 1, config.CORSConfig{}, cfg)) + server := httptest.NewServer(New(hub.New(), nil, nil, 1, config.CORSConfig{}, cfg, config.AuthConfig{})) defer server.Close() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() diff --git a/internal/config/auth_test.go b/internal/config/auth_test.go new file mode 100644 index 00000000..e3fccf58 --- /dev/null +++ b/internal/config/auth_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadAuth(t *testing.T) { + for _, tc := range []struct { + name, env, want string + file, set bool + }{ + {"no config", "", "", false, false}, + {"YAML", "", "yaml-test-key", true, false}, + {"environment overrides", "env-test-key", "env-test-key", true, true}, + {"empty environment disables", "", "", true, true}, + {"environment without file", "env-test-key", "env-test-key", false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("BEACON_API_KEY", tc.env) + if !tc.set { + if err := os.Unsetenv("BEACON_API_KEY"); err != nil { + t.Fatal(err) + } + } + path := filepath.Join(t.TempDir(), "config.yaml") + if tc.file { + if err := os.WriteFile(path, []byte("auth:\n api_key: yaml-test-key\n"), 0600); err != nil { + t.Fatal(err) + } + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if cfg.Auth.APIKey != tc.want { + t.Fatal("wrong auth configuration precedence") + } + encoded, err := json.Marshal(cfg) + if err != nil || strings.Contains(string(encoded), "test-key") { + t.Fatal("key exposed in JSON") + } + if strings.Contains(Resolve(cfg).String(), "test-key") { + t.Fatal("key exposed in startup summary") + } + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 4a9f4173..d4ada2a8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,6 +17,7 @@ import ( // Config is the top-level structure of the Beacon config file. type Config struct { + Auth AuthConfig `yaml:"auth"` Server ServerConfig `yaml:"server"` IATAs map[string]IATAConfig `yaml:"iatas"` Regions []RegionConfig `yaml:"regions"` @@ -35,6 +36,12 @@ type Config struct { Observers ObserversConfig `yaml:"observers"` } +// AuthConfig holds the operator key for the protected admin subtree. +// The key is excluded from JSON; it must not be exposed by configuration APIs. +type AuthConfig struct { + APIKey string `yaml:"api_key" json:"-"` +} + // ServerConfig controls which direct peers may supply the client address. type ServerConfig struct { // TrustedProxies accepts IPv4/IPv6 CIDRs; an empty list trusts no proxy. @@ -333,6 +340,7 @@ func Load(path string) (*Config, error) { data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { + cfg.Auth.APIKey = os.Getenv("BEACON_API_KEY") return cfg, nil } return nil, err @@ -340,6 +348,9 @@ func Load(path string) (*Config, error) { if err := yaml.Unmarshal(data, cfg); err != nil { return nil, err } + if value, set := os.LookupEnv("BEACON_API_KEY"); set { + cfg.Auth.APIKey = value + } for i, prefix := range cfg.Server.TrustedProxies { if !prefix.IsValid() { return nil, fmt.Errorf("server.trusted_proxies[%d] must be a valid CIDR", i) From 0067e513570d0c8019e8bdc70545383cba440295 Mon Sep 17 00:00:00 2001 From: n30nex Date: Fri, 18 Sep 2026 06:18:42 -0400 Subject: [PATCH 036/141] feat(cache): add payload-free debug cache diagnostics (#145) --- internal/cache/cache.go | 5 +++ internal/cache/logging_test.go | 64 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 internal/cache/logging_test.go diff --git a/internal/cache/cache.go b/internal/cache/cache.go index abee2760..92377544 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -11,6 +11,7 @@ import ( "context" "encoding/json" "errors" + "log/slog" "time" "github.com/MeshCore-Beacon/beacon-server/internal/config" @@ -86,11 +87,13 @@ func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Durati raw, err := c.rdb.Get(ctx, key).Bytes() if err != nil && !errors.Is(err, redis.Nil) { // real Redis error, degrade gracefully + slog.DebugContext(ctx, "cache bypass", "component", "cache", "reason", "read_error") return fetch() } var zero, out T if errors.Is(err, redis.Nil) { // cache miss — fetch, store, return + slog.DebugContext(ctx, "cache miss", "component", "cache") val, err := fetch() if err != nil { return zero, err @@ -104,6 +107,7 @@ func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Durati } if err = json.Unmarshal(raw, &out); err != nil { // corrupt cache entry — overwrite it + slog.DebugContext(ctx, "cache invalid entry", "component", "cache") val, err := fetch() if err != nil { return zero, err @@ -115,5 +119,6 @@ func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Durati _ = c.rdb.Set(ctx, key, data, ttl) return val, nil } + slog.DebugContext(ctx, "cache hit", "component", "cache") return out, nil } diff --git a/internal/cache/logging_test.go b/internal/cache/logging_test.go new file mode 100644 index 00000000..d16eed5e --- /dev/null +++ b/internal/cache/logging_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "bytes" + "context" + "encoding/json" + "log" + "log/slog" + "strings" + "testing" + "time" +) + +func TestCacheDiagnostics(t *testing.T) { + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) + var out bytes.Buffer + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, &slog.HandlerOptions{Level: slog.LevelDebug}))) + c, mr := newTestClient(t) + key, value := "private-cache-key", "private-cache-value" + calls := 0 + fetch := func() (string, error) { calls++; return value, nil } + for i := 0; i < 2; i++ { + got, err := getOrSet(context.Background(), c, key, time.Minute, fetch) + if err != nil || got != value { + t.Fatalf("cache result changed: %v", err) + } + } + if calls != 1 { + t.Fatalf("fetches=%d, want 1", calls) + } + mr.Set(key, "invalid JSON") + if _, err := getOrSet(context.Background(), c, key, time.Minute, fetch); err != nil { + t.Fatal(err) + } + lines := bytes.Split(bytes.TrimSpace(out.Bytes()), []byte("\n")) + want := []string{"cache miss", "cache hit", "cache invalid entry"} + if len(lines) != len(want) { + t.Fatalf("got %d records, want %d", len(lines), len(want)) + } + for i, line := range lines { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal(err) + } + if record["component"] != "cache" || record["level"] != "DEBUG" || record["msg"] != want[i] { + t.Fatalf("unexpected record: %v", record) + } + } + if strings.Contains(out.String(), key) || strings.Contains(out.String(), value) { + t.Fatal("cache key or payload leaked") + } + out.Reset() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, nil))) + if _, err := getOrSet(context.Background(), c, key, time.Minute, fetch); err != nil { + t.Fatal(err) + } + if out.Len() != 0 { + t.Fatal("debug diagnostics appeared at info level") + } +} From 595ef4442b5e535fdae9dedaf52480129bf61195 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 08:43:24 -0400 Subject: [PATCH 037/141] feat(api): rate limit REST requests by trusted client IP (#140) --- SHOULDERS.md | 2 + cmd/beacon/main.go | 3 +- config.yaml.example | 11 ++ docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 5 +- go.mod | 4 + go.sum | 4 + internal/api/middleware/rate_limit.go | 36 +++++ internal/api/router/auth_test.go | 2 +- internal/api/router/rate_limit_test.go | 181 +++++++++++++++++++++++++ internal/api/router/router.go | 4 +- internal/api/router/router_test.go | 18 ++- internal/config/config.go | 33 ++++- internal/config/rate_limit_test.go | 45 ++++++ 15 files changed, 343 insertions(+), 9 deletions(-) create mode 100644 internal/api/middleware/rate_limit.go create mode 100644 internal/api/router/rate_limit_test.go create mode 100644 internal/config/rate_limit_test.go diff --git a/SHOULDERS.md b/SHOULDERS.md index 5948cf42..05076de6 100644 --- a/SHOULDERS.md +++ b/SHOULDERS.md @@ -12,6 +12,8 @@ projects that make this possible: - [sqlc](https://sqlc.dev) — Type-safe SQL code generation - [chi](https://github.com/go-chi/chi) — HTTP router - [go-chi/cors](https://github.com/go-chi/cors) — CORS middleware for chi +- [go-chi/httprate](https://github.com/go-chi/httprate) — Per-client HTTP + sliding-window rate limits and retry headers - [go-redis](https://github.com/redis/go-redis) — Redis client - [coder/websocket](https://github.com/coder/websocket) — WebSocket implementation diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 43e6168c..6f3f4606 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -41,6 +41,7 @@ var version = "dev" // @title MeshCore Beacon API // @version 1.6.0 // @description MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket. +// @description REST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget. // @termsOfService https://github.com/MeshCore-Beacon/beacon-server // @contact.name MeshCore Beacon @@ -275,7 +276,7 @@ func main() { go scheduler.Start(ctx) // ── HTTP server ────────────────────────────────────────────────────────── - r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, cfg.CORS, cfg.Server, cfg.Auth) + r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, cfg.CORS, cfg.Server, cfg.Auth, resolved.RateLimit) srv := &http.Server{ Addr: addr, diff --git a/config.yaml.example b/config.yaml.example index c07852f3..a65669df 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -25,6 +25,17 @@ server: # and pass its validated client address rather than a client-supplied header. trusted_proxies: [] +# Applies to /api/v1/* only. Both counters are local to this Beacon instance. +# Set trusted_proxies correctly before using a reverse proxy, or clients share +# the proxy's budget. IPv6 clients share a /64 budget to prevent address rotation. +ratelimit: + enabled: true + requests_per_minute: 300 + # Maximum requests in a one-second sliding window, not token-bucket capacity. + # Omitted/zero defaults to requests_per_minute. Negative limits prevent startup. + # A 429 returns error.code=rate_limited and Retry-After (1 or 60 seconds). + burst: 300 + iatas: YVR: name: Vancouver International diff --git a/docs/docs.go b/docs/docs.go index d65c73e3..451cd45e 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4149,7 +4149,7 @@ var SwaggerInfo = &swag.Spec{ BasePath: "/api/v1", Schemes: []string{"http", "https"}, Title: "MeshCore Beacon API", - Description: "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.", + Description: "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.\nREST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget.", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, LeftDelim: "{{", diff --git a/docs/swagger.json b/docs/swagger.json index f1bc16ff..90acfb29 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -5,7 +5,7 @@ ], "swagger": "2.0", "info": { - "description": "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.", + "description": "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.\nREST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget.", "title": "MeshCore Beacon API", "termsOfService": "https://github.com/MeshCore-Beacon/beacon-server", "contact": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index e1c5dfe3..cb0b716e 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1182,8 +1182,9 @@ info: contact: name: MeshCore Beacon url: https://github.com/MeshCore-Beacon/beacon-server - description: MeshCore network observation backend. Ingests LoRa packets from MQTT - brokers, stores in PostgreSQL, and streams live events via WebSocket. + description: |- + MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket. + REST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget. license: name: AGPL-3-or-later termsOfService: https://github.com/MeshCore-Beacon/beacon-server diff --git a/go.mod b/go.mod index ac5e38c3..d309cbf9 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/eclipse/paho.mqtt.golang v1.5.1 github.com/go-chi/chi/v5 v5.3.0 github.com/go-chi/cors v1.2.2 + github.com/go-chi/httprate v0.16.0 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.9.2 github.com/joho/godotenv v1.5.1 @@ -33,16 +34,19 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect github.com/yuin/gopher-lua v1.1.1 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.39.0 // indirect golang.org/x/tools v0.47.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index ef86ff73..b4d9df21 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM= github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8= +github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -88,6 +90,8 @@ github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= diff --git a/internal/api/middleware/rate_limit.go b/internal/api/middleware/rate_limit.go new file mode 100644 index 00000000..dcd3b444 --- /dev/null +++ b/internal/api/middleware/rate_limit.go @@ -0,0 +1,36 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "net/http" + "time" + + "github.com/go-chi/httprate" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +// RateLimit applies the minute budget and one-second burst cap to the API. +// TrustedProxyIP must run first; the access logger must wrap this middleware. +func RateLimit(cfg config.ResolvedRateLimitConfig) func(http.Handler) http.Handler { + if !cfg.Enabled { + return func(next http.Handler) http.Handler { return next } + } + options := []httprate.Option{ + httprate.WithLimitHandler(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"error":{"code":"rate_limited","message":"Request rate limit exceeded. Retry later."}}`)) + }), + // Only expose the backoff header, since the two windows have different budgets. + httprate.WithResponseHeaders(httprate.ResponseHeaders{RetryAfter: "Retry-After"}), + } + // RemoteAddr has already been resolved and forwarding headers removed. + minute := httprate.LimitBy(cfg.RequestsPerMinute, time.Minute, httprate.KeyByIP, options...) + burst := httprate.LimitBy(cfg.Burst, time.Second, httprate.KeyByIP, options...) + return func(next http.Handler) http.Handler { + return minute(burst(next)) + } +} diff --git a/internal/api/router/auth_test.go b/internal/api/router/auth_test.go index 9c66f6d9..25cef5ed 100644 --- a/internal/api/router/auth_test.go +++ b/internal/api/router/auth_test.go @@ -13,7 +13,7 @@ import ( func TestAdminAuthBoundary(t *testing.T) { for _, key := range []string{"", "synthetic-test-key"} { - handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{APIKey: key}) + handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{APIKey: key}, config.ResolvedRateLimitConfig{}) for _, path := range []string{"/api/v1/admin", "/api/v1/admin/", "/api/v1/admin/config", "/api/v1/admin//config"} { for _, method := range []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"} { for _, token := range []string{"", "wrong", "synthetic-test-key"} { diff --git a/internal/api/router/rate_limit_test.go b/internal/api/router/rate_limit_test.go new file mode 100644 index 00000000..676419f2 --- /dev/null +++ b/internal/api/router/rate_limit_test.go @@ -0,0 +1,181 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "bytes" + "encoding/json" + "log" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/go-chi/chi/v5/middleware" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestDefaultAPIRateLimit(t *testing.T) { + handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.Resolve(&config.Config{}).RateLimit) + for i := 0; i <= 300; i++ { + request := httptest.NewRequest(http.MethodGet, "/api/v1/brokers", nil) + request.RemoteAddr = "198.51.100.1:1234" + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + want := http.StatusOK + if i == 300 { + want = http.StatusTooManyRequests + } + if response.Code != want { + t.Fatalf("request %d: got %d, want %d", i+1, response.Code, want) + } + } +} + +func rateRequest(handler http.Handler, path, peer, headerIP string) *httptest.ResponseRecorder { + request := httptest.NewRequest(http.MethodGet, path, nil) + request.RemoteAddr = peer + request.Header.Set("Origin", "https://example.test") + request.Header.Set("X-Real-IP", headerIP) + request.Header.Set("X-Forwarded-For", headerIP) + request.Header.Set("True-Client-IP", headerIP) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response +} + +func TestAPIRateLimitContractAndLogging(t *testing.T) { + var logs bytes.Buffer + previous := middleware.DefaultLogger + middleware.DefaultLogger = middleware.RequestLogger(&middleware.DefaultLogFormatter{Logger: log.New(&logs, "", 0), NoColor: true}) + t.Cleanup(func() { middleware.DefaultLogger = previous }) + proxy := config.ServerConfig{TrustedProxies: []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}} + handler := New(nil, nil, nil, 5, config.CORSConfig{}, proxy, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 2, Burst: 10}) + for _, path := range []string{"/api/v1/brokers", "/api/v1/packets?limit=0"} { + response := rateRequest(handler, path, "127.0.0.1:1234", "198.51.100.25") + if response.Code != http.StatusOK && response.Code != http.StatusBadRequest { + t.Fatalf("initial request: %d", response.Code) + } + } + response := rateRequest(handler, "/api/v1/brokers", "127.0.0.1:1234", "198.51.100.25") + var body struct { + Error struct{ Code, Message string } + } + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if response.Code != http.StatusTooManyRequests || body.Error.Code != "rate_limited" || body.Error.Message == "" { + t.Fatalf("wrong rate-limit response: %d %s", response.Code, response.Body.String()) + } + if response.Header().Get("Content-Type") != "application/json" || response.Header().Get("Retry-After") != "60" || !strings.EqualFold(response.Header().Get("Access-Control-Expose-Headers"), "Retry-After") || response.Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatalf("missing JSON/backoff/CORS headers: %v", response.Header()) + } + if strings.Count(logs.String(), " - 429 ") != 1 || !strings.Contains(logs.String(), "from 198.51.100.25 - 429 ") || !strings.Contains(logs.String(), "/api/v1/brokers") { + t.Fatalf("expected one rejection log with resolved client and path: %s", logs.String()) + } +} + +func TestAPIRateLimitClientIdentity(t *testing.T) { + for _, tc := range []struct { + name, firstPeer, secondPeer, firstHeader, secondHeader string + trusted, shared bool + }{ + {"untrusted header rotation", "198.51.100.1:1", "198.51.100.1:2", "192.0.2.1", "192.0.2.2", false, true}, + {"independent peers", "198.51.100.1:1", "198.51.100.2:1", "192.0.2.1", "192.0.2.1", false, false}, + {"trusted independent clients", "127.0.0.1:1", "127.0.0.1:2", "198.51.100.1", "198.51.100.2", true, false}, + {"trusted shared client", "127.0.0.1:1", "127.0.0.1:2", "198.51.100.1", "198.51.100.1", true, true}, + {"IPv6 shared prefix", "[2001:db8:1::1]:1", "[2001:db8:1::2]:2", "", "", false, true}, + {"IPv6 independent prefixes", "[2001:db8:1::1]:1", "[2001:db8:2::1]:2", "", "", false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + proxy := config.ServerConfig{} + if tc.trusted { + proxy.TrustedProxies = []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")} + } + handler := New(nil, nil, nil, 5, config.CORSConfig{}, proxy, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1}) + if response := rateRequest(handler, "/api/v1/brokers", tc.firstPeer, tc.firstHeader); response.Code != http.StatusOK { + t.Fatalf("first client: %d", response.Code) + } + want := http.StatusOK + if tc.shared { + want = http.StatusTooManyRequests + } + if response := rateRequest(handler, "/api/v1/brokers", tc.secondPeer, tc.secondHeader); response.Code != want { + t.Fatalf("second client: %d, want %d", response.Code, want) + } + }) + } +} + +func TestAPIRateLimitWindowsAndExclusions(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 4, Burst: 2}) + request := func() *httptest.ResponseRecorder { + return rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", "") + } + preflight := httptest.NewRequest(http.MethodOptions, "/api/v1/brokers", nil) + preflight.RemoteAddr = "198.51.100.1:1" + preflight.Header.Set("Origin", "https://example.test") + preflight.Header.Set("Access-Control-Request-Method", "GET") + response := httptest.NewRecorder() + handler.ServeHTTP(response, preflight) + if response.Code != http.StatusOK || request().Code != http.StatusOK || request().Code != http.StatusOK { + t.Fatal("preflight consumed the API budget") + } + if response := request(); response.Code != http.StatusTooManyRequests || response.Header().Get("Retry-After") != "1" { + t.Fatalf("burst limit: %d %v", response.Code, response.Header()) + } + time.Sleep(2 * time.Second) + if request().Code != http.StatusOK { + t.Fatal("burst did not recover") + } + if response := request(); response.Code != http.StatusTooManyRequests || response.Header().Get("Retry-After") != "60" { + t.Fatalf("minute limit: %d %v", response.Code, response.Header()) + } + response = httptest.NewRecorder() + handler.ServeHTTP(response, preflight) + if response.Code != http.StatusOK { + t.Fatal("exhausted API budget blocked preflight") + } + if response := rateRequest(handler, "/swagger", "198.51.100.1:1", ""); response.Code != http.StatusMovedPermanently { + t.Fatalf("Swagger shares the API budget: %d", response.Code) + } + time.Sleep(2 * time.Minute) + if request().Code != http.StatusOK { + t.Fatal("minute budget did not recover") + } + }) + handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: false, RequestsPerMinute: 1, Burst: 1}) + for range 5 { + if response := rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", ""); response.Code != http.StatusOK || response.Header().Get("Retry-After") != "" { + t.Fatal("disabled limiter still applied") + } + } +} + +func TestAPIRateLimitConcurrentRequests(t *testing.T) { + handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 5, Burst: 5}) + var accepted, rejected atomic.Int32 + var group sync.WaitGroup + for range 20 { + group.Go(func() { + switch rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", "").Code { + case http.StatusOK: + accepted.Add(1) + case http.StatusTooManyRequests: + rejected.Add(1) + } + }) + } + group.Wait() + if accepted.Load() != 5 || rejected.Load() != 15 { + t.Fatalf("concurrent budget: accepted=%d rejected=%d", accepted.Load(), rejected.Load()) + } +} diff --git a/internal/api/router/router.go b/internal/api/router/router.go index a2f5d777..84584133 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -39,7 +39,7 @@ import ( // /stats → stats subrouter // // Admin handlers are added separately; the reserved subtree is protected now. -func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP int, corsCfg config.CORSConfig, serverCfg config.ServerConfig, authCfg config.AuthConfig) http.Handler { +func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP int, corsCfg config.CORSConfig, serverCfg config.ServerConfig, authCfg config.AuthConfig, rateLimitCfg config.ResolvedRateLimitConfig) http.Handler { r := chi.NewRouter() // ── CORS ───────────────────────────────────────────────────────────────── @@ -63,6 +63,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP AllowedOrigins: allowedOrigins, AllowedMethods: allowedMethods, AllowedHeaders: allowedHeaders, + ExposedHeaders: []string{"Retry-After"}, AllowCredentials: corsCfg.AllowCredentials, MaxAge: maxAge, })) @@ -89,6 +90,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP // ── Public REST API (v1) ───────────────────────────────────────────────── r.Route("/api/v1", func(r chi.Router) { + r.Use(mw.RateLimit(rateLimitCfg)) // Public group — no authentication required. r.Group(func(r chi.Router) { r.Mount("/packets", handlers.PacketsRouter(reader)) diff --git a/internal/api/router/router_test.go b/internal/api/router/router_test.go index 6b8c5a22..d3565bca 100644 --- a/internal/api/router/router_test.go +++ b/internal/api/router/router_test.go @@ -36,8 +36,24 @@ func TestWebSocketLimitUsesTrustedClientIP(t *testing.T) { func checkWebSocketLimit(t *testing.T, cfg config.ServerConfig, secondIP string, wantStatus int) { t.Helper() - server := httptest.NewServer(New(hub.New(), nil, nil, 1, config.CORSConfig{}, cfg, config.AuthConfig{})) + server := httptest.NewServer(New(hub.New(), nil, nil, 1, config.CORSConfig{}, cfg, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1})) defer server.Close() + // Exhaust this client's REST budget before checking its independent WS cap. + for _, want := range []int{http.StatusOK, http.StatusTooManyRequests} { + request, err := http.NewRequest(http.MethodGet, server.URL+"/api/v1/brokers", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("X-Real-IP", "198.51.100.1") + response, err := server.Client().Do(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != want { + t.Fatalf("REST budget setup: %d, want %d", response.StatusCode, want) + } + } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() url := "ws" + strings.TrimPrefix(server.URL, "http") + "/ws" diff --git a/internal/config/config.go b/internal/config/config.go index d4ada2a8..140b102b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,6 +30,7 @@ type Config struct { Scopes []ScopeConfig `yaml:"scopes"` Cache CacheConfig `yaml:"cache"` CORS CORSConfig `yaml:"cors"` + RateLimit RateLimitConfig `yaml:"ratelimit"` Background BackgroundConfig `yaml:"background"` Presence PresenceConfig `yaml:"presence"` Nodes NodesConfig `yaml:"nodes"` @@ -69,6 +70,7 @@ func (c *ServerConfig) UnmarshalYAML(node *yaml.Node) error { // ResolvedConfig holds all runtime configuration with defaults applied. type ResolvedConfig struct { + RateLimit ResolvedRateLimitConfig TelemetryResolution time.Duration TelemetryRetention time.Duration PacketRetention time.Duration @@ -95,6 +97,20 @@ type ResolvedConfig struct { ObserverDeleteAfter time.Duration } +// RateLimitConfig controls the per-client REST API request budget. +type RateLimitConfig struct { + Enabled *bool `yaml:"enabled"` // Defaults to true; false disables both windows. + RequestsPerMinute int `yaml:"requests_per_minute"` + Burst int `yaml:"burst"` // One-second window cap, not token-bucket capacity. +} + +// ResolvedRateLimitConfig has defaults applied and no optional values. +type ResolvedRateLimitConfig struct { + Enabled bool + RequestsPerMinute int + Burst int +} + // PresenceConfig controls coalescing of presence bookkeeping writes // (observer last_seen, observer_brokers, packet last_heard_at bumps). type PresenceConfig struct { @@ -356,6 +372,9 @@ func Load(path string) (*Config, error) { return nil, fmt.Errorf("server.trusted_proxies[%d] must be a valid CIDR", i) } } + if cfg.RateLimit.RequestsPerMinute < 0 || cfg.RateLimit.Burst < 0 { + return nil, fmt.Errorf("ratelimit.requests_per_minute and ratelimit.burst must be positive or zero for defaults") + } configDir := filepath.Dir(path) for iata, details := range cfg.IATAs { if details.BorderFile != "" && !filepath.IsAbs(details.BorderFile) { @@ -369,6 +388,11 @@ func Load(path string) (*Config, error) { // Resolve returns a ResolvedConfig with defaults applied for any zero values. func Resolve(cfg *Config) ResolvedConfig { r := ResolvedConfig{ + RateLimit: ResolvedRateLimitConfig{ + Enabled: cfg.RateLimit.Enabled == nil || *cfg.RateLimit.Enabled, + RequestsPerMinute: cfg.RateLimit.RequestsPerMinute, + Burst: cfg.RateLimit.Burst, + }, TelemetryResolution: cfg.Telemetry.Resolution.Duration, TelemetryRetention: cfg.Telemetry.Retention.Duration, PacketRetention: cfg.Packets.Retention.Duration, @@ -388,6 +412,12 @@ func Resolve(cfg *Config) ResolvedConfig { NodeDeleteAfter: cfg.Nodes.DeleteAfter.Duration, ObserverDeleteAfter: cfg.Observers.DeleteAfter.Duration, } + if r.RateLimit.RequestsPerMinute == 0 { + r.RateLimit.RequestsPerMinute = 300 + } + if r.RateLimit.Burst == 0 { + r.RateLimit.Burst = r.RateLimit.RequestsPerMinute + } if r.TelemetryResolution == 0 { r.TelemetryResolution = time.Hour } @@ -440,10 +470,11 @@ func Resolve(cfg *Config) ResolvedConfig { func (r ResolvedConfig) String() string { return fmt.Sprintf( - "telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s", + "telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d", r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, r.MaxConnsPerIP, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold, r.NodeStaleThreshold, r.NodeDeleteAfter, r.ObserverDeleteAfter, + r.RateLimit.Enabled, r.RateLimit.RequestsPerMinute, r.RateLimit.Burst, ) } diff --git a/internal/config/rate_limit_test.go b/internal/config/rate_limit_test.go new file mode 100644 index 00000000..9e4e135b --- /dev/null +++ b/internal/config/rate_limit_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRateLimitConfig(t *testing.T) { + for _, tc := range []struct { + name, yaml string + want ResolvedRateLimitConfig + wantError bool + }{ + {"omitted", "{}", ResolvedRateLimitConfig{true, 300, 300}, false}, + {"disabled", "ratelimit: {enabled: false}", ResolvedRateLimitConfig{false, 300, 300}, false}, + {"custom", "ratelimit: {enabled: true, requests_per_minute: 900, burst: 100}", ResolvedRateLimitConfig{true, 900, 100}, false}, + {"burst default follows budget", "ratelimit: {requests_per_minute: 600}", ResolvedRateLimitConfig{true, 600, 600}, false}, + {"zero defaults", "ratelimit: {requests_per_minute: 0, burst: 0}", ResolvedRateLimitConfig{true, 300, 300}, false}, + {"negative minute", "ratelimit: {requests_per_minute: -1}", ResolvedRateLimitConfig{}, true}, + {"negative burst", "ratelimit: {burst: -1}", ResolvedRateLimitConfig{}, true}, + {"invalid disabled config", "ratelimit: {enabled: false, burst: -1}", ResolvedRateLimitConfig{}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if (err != nil) != tc.wantError { + t.Fatalf("Load error = %v, want error = %v", err, tc.wantError) + } + if err == nil && Resolve(cfg).RateLimit != tc.want { + t.Fatalf("resolved limits = %+v, want %+v", Resolve(cfg).RateLimit, tc.want) + } + }) + } + cfg, err := Load(filepath.Join(t.TempDir(), "missing.yaml")) + if err != nil || Resolve(cfg).RateLimit != (ResolvedRateLimitConfig{true, 300, 300}) { + t.Fatalf("missing config did not enable defaults: %v", err) + } +} From dab228e9bbe29c19cc7125d9831d12b91ed6d21f Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 10:36:26 -0400 Subject: [PATCH 038/141] feat(logging): configure levels and structured runtime output (#144) * feat(logging): configure levels and structured runtime output * fix(logging): bound request-derived log fields * fix(logging): retain malformed IATA source attributes safely * test(logging): capture structured REST rate-limit records --- README.md | 14 ++ cmd/beacon/main.go | 69 +++++---- cmd/beacon/main_test.go | 60 ++++++++ config.yaml.example | 5 + db/migrate.go | 9 +- db/nodes.go | 8 +- db/observers.go | 6 +- db/packets.go | 10 +- db/stats.go | 4 +- env.example | 4 + internal/api/handlers/limits.go | 4 +- internal/api/handlers/limits_logging_test.go | 37 +++++ internal/api/handlers/responses.go | 7 +- internal/api/handlers/stats.go | 18 +-- internal/api/middleware/logging.go | 63 ++++++++ internal/api/middleware/logging_test.go | 143 +++++++++++++++++++ internal/api/router/rate_limit_test.go | 26 +++- internal/api/router/router.go | 2 +- internal/background/background.go | 8 +- internal/background/tasks_test.go | 60 ++++---- internal/config/config.go | 7 + internal/config/config_test.go | 14 ++ internal/config/seed.go | 4 +- internal/hub/hub.go | 7 +- internal/ingest/backfill.go | 6 +- internal/ingest/ingest.go | 29 ++-- internal/ingest/ingest_test.go | 2 + internal/ingest/logging_test.go | 63 ++++++++ internal/ingest/neighbors.go | 16 +-- internal/ingest/packet.go | 47 +++--- internal/ingest/side_effects.go | 21 ++- internal/ingest/status.go | 29 ++-- internal/logging/logging.go | 47 ++++++ internal/logging/logging_test.go | 97 +++++++++++++ internal/presence/coalescer.go | 9 +- internal/ws/handler.go | 47 +++--- 36 files changed, 801 insertions(+), 201 deletions(-) create mode 100644 cmd/beacon/main_test.go create mode 100644 internal/api/handlers/limits_logging_test.go create mode 100644 internal/api/middleware/logging.go create mode 100644 internal/api/middleware/logging_test.go create mode 100644 internal/ingest/logging_test.go create mode 100644 internal/logging/logging.go create mode 100644 internal/logging/logging_test.go diff --git a/README.md b/README.md index 0d3d7c7d..0c6e26b1 100644 --- a/README.md +++ b/README.md @@ -453,3 +453,17 @@ Beacon. Beacon stands on the shoulders of giants. See [SHOULDERS.md](SHOULDERS.md) for the full list of open source projects that make this possible. + +## Application logging + +Beacon writes application logs to stderr. Configure the minimum level and output format in `config.yaml`: + +```yaml +log: + level: info # debug, info, warn, error + format: text # text or json +``` + +`LOG_LEVEL` and `LOG_FORMAT` override file settings; empty settings use `info` and `text`. Invalid values prevent startup. Configuration-loading failures can use the bootstrap text logger before file settings are available; failures after initialization retain error severity at every supported level. + +Records include a component field. Ingest workers also include their broker name, and HTTP completion records include the validated client address, route, status and duration. Query strings and protocol hello payloads are excluded. Expected ingest skips and routine WebSocket lifecycle details are debug-level. Changing the application's format does not change Caddy/Apache access logs or their fail2ban configuration. Collect/rotate stderr through Docker or systemd. diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 6f3f4606..da445eb0 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -7,7 +7,7 @@ import ( "context" "encoding/hex" "fmt" - "log" + "log/slog" "net/http" "os" "os/signal" @@ -26,6 +26,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "github.com/MeshCore-Beacon/beacon-server/internal/logging" "github.com/MeshCore-Beacon/beacon-server/internal/presence" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" @@ -73,7 +74,6 @@ var version = "dev" // @tag.name Stats // @tag.description Network statistics and time series func main() { - log.Printf("beacon version %s", version) _ = godotenv.Load() addr := os.Getenv("LISTEN_ADDR") if addr == "" { @@ -87,15 +87,23 @@ func main() { cfg, err := config.Load(configPath) if err != nil { - log.Fatalf("failed to load config: %v", err) + slog.Error("failed to load config", "component", "startup", "error", err) + os.Exit(1) } + logger, err := logging.New(os.Stderr, cfg.Log) + if err != nil { + slog.Error("invalid logging configuration", "component", "startup", "error", err) + os.Exit(1) + } + slog.SetDefault(logger) + slog.Info("beacon starting", "component", "startup", "version", version) if len(cfg.Server.TrustedProxies) == 0 { - log.Print("warning: server.trusted_proxies is empty; client IP headers are ignored and proxied clients share a WebSocket connection limit") + slog.Warn("warning: server.trusted_proxies is empty; client IP headers are ignored and proxied clients share a WebSocket connection limit", "component", "startup") } resolved := config.Resolve(cfg) - log.Printf("config: loaded — %s", resolved) + slog.Info(fmt.Sprintf("config: loaded — %s", resolved), "component", "startup") // ── Hub ────────────────────────────────────────────────────────────────── h := hub.New() @@ -107,12 +115,15 @@ func main() { pool, err := pgxpool.New(ctx, getEnv("POSTGRES_DSN")) if err != nil { - log.Fatalf("failed to connect to postgres at %s: %v", os.Getenv("POSTGRES_DSN_HOST"), err) + // Parse errors can embed the complete DSN, including its password. + slog.Error("invalid PostgreSQL connection configuration; check POSTGRES_DSN", "component", "startup") + os.Exit(1) } defer pool.Close() if err := db.RunMigrations(ctx, pool); err != nil { - log.Fatalf("migrations failed: %v", err) + slog.Error("migrations failed", "component", "startup", "error", err) + os.Exit(1) } store := db.New(pool, resolved.ClockDriftThreshold, resolved.NodeStaleThreshold) @@ -139,29 +150,30 @@ func main() { }(), ) if err := redisClient.Ping(ctx); err != nil { - log.Printf("warning: redis unavailable at %s, caching disabled: %v", redisAddr, err) + slog.Warn(fmt.Sprintf("warning: redis unavailable at %s, caching disabled", redisAddr), "component", "startup", "error", err) } else { ttls := cache.ResolveTTLs(cfg.Cache) reader = cache.NewCachedReader(store, redisClient, ttls) defer redisClient.Close() - log.Printf("cache: Redis connected at %s (stats=%s reference=%s nodes=%s observers=%s)", - redisAddr, ttls.Stats, ttls.Reference, ttls.Nodes, ttls.Observers) + slog.Info(fmt.Sprintf("cache: Redis connected at %s (stats=%s reference=%s nodes=%s observers=%s)", redisAddr, ttls.Stats, ttls.Reference, ttls.Nodes, ttls.Observers), "component", "startup") } } // ── Seed config data ───────────────────────────────────────────────────── if err := config.Seed(ctx, cfg, store); err != nil { - log.Fatalf("failed to seed config: %v", err) + slog.Error("failed to seed config", "component", "startup", "error", err) + os.Exit(1) } // ── Build transport scope keystore ─────────────────────────────────────── scopes := scopestore.New() scopeEntries, err := store.GetTransportScopes(ctx) if err != nil { - log.Fatalf("failed to load transport scopes: %v", err) + slog.Error("failed to load transport scopes", "component", "startup", "error", err) + os.Exit(1) } scopes.Load(scopeEntries) - log.Printf("loaded %d transport scopes", len(scopeEntries)) + slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") // ── Build channel keystore ────────────────────────────────────────────── entries := make(map[string][]keystore.Entry) @@ -178,7 +190,7 @@ func main() { } if !keystore.EntryExists(entries[hashHex], entry) { entries[hashHex] = append(entries[hashHex], entry) - log.Printf("config: loaded hashtag channel #%s (hash=%s)", tag, hashHex) + slog.Info(fmt.Sprintf("config: loaded hashtag channel #%s (hash=%s)", tag, hashHex), "component", "startup") } } @@ -186,7 +198,7 @@ func main() { for hashHex, keyCfg := range cfg.ChannelKeys.Keys { key, err := hex.DecodeString(keyCfg.Key) if err != nil { - log.Printf("warning: invalid channel key for hash %s, skipping: %v", hashHex, err) + slog.Warn(fmt.Sprintf("warning: invalid channel key for hash %s, skipping", hashHex), "component", "startup", "error", err) continue } entry := keystore.Entry{ @@ -196,7 +208,7 @@ func main() { } if !keystore.EntryExists(entries[hashHex], entry) { entries[hashHex] = append(entries[hashHex], entry) - log.Printf("config: loaded explicit channel key for hash %s name=%q", hashHex, keyCfg.Name) + slog.Info(fmt.Sprintf("config: loaded explicit channel key for hash %s name=%q", hashHex, keyCfg.Name), "component", "startup") } } @@ -208,18 +220,17 @@ func main() { // built, so adding a channel key to the config surfaces its history on the next boot // instead of leaving it stranded in the DB indefinitely. if n, err := ingest.BackfillChannelMessages(ctx, store, keys); err != nil { - log.Printf("config: channel message backfill failed: %v", err) + slog.Error("config: channel message backfill failed", "component", "startup", "error", err) } else if n > 0 { - log.Printf("config: backfilled %d previously-undecrypted channel message(s)", n) + slog.Info(fmt.Sprintf("config: backfilled %d previously-undecrypted channel message(s)", n), "component", "startup") } // ── Build geographic ingest filter ─────────────────────────────────────────────────────────── allowedIATAs := iatadb.BuildAllowedSet(cfg.Ingest.AllowCountries, cfg.Ingest.AllowContinents) if allowedIATAs != nil { - log.Printf("config: ingest filter active — %d allowed IATAs (countries=%v continents=%v)", - len(allowedIATAs), cfg.Ingest.AllowCountries, cfg.Ingest.AllowContinents) + slog.Info(fmt.Sprintf("config: ingest filter active — %d allowed IATAs (countries=%v continents=%v)", len(allowedIATAs), cfg.Ingest.AllowCountries, cfg.Ingest.AllowContinents), "component", "startup") } else { - log.Printf("config: ingest filter inactive — accepting all IATAs") + slog.Info("config: ingest filter inactive — accepting all IATAs", "component", "startup") } broker1 := ingest.New( @@ -279,14 +290,16 @@ func main() { r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, cfg.CORS, cfg.Server, cfg.Auth, resolved.RateLimit) srv := &http.Server{ - Addr: addr, - Handler: r, + Addr: addr, + Handler: r, + ErrorLog: slog.NewLogLogger(slog.Default().With("component", "http").Handler(), slog.LevelError), } go func() { - fmt.Printf("Beacon listening on %s\n", addr) + slog.Info(fmt.Sprintf("Beacon listening on %s", addr), "component", "startup") if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Fatalf("server error: %v", err) + slog.Error("server error", "component", "startup", "error", err) + os.Exit(1) } }() @@ -295,12 +308,12 @@ func main() { signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) <-quit - log.Println("shutting down...") + slog.Info("shutting down...", "component", "startup") cancel() // stops ingest workers shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) defer shutdownCancel() if err := srv.Shutdown(shutdownCtx); err != nil { - log.Printf("server shutdown error: %v", err) + slog.Error("server shutdown error", "component", "startup", "error", err) } coalescer.Flush(shutdownCtx) } @@ -311,7 +324,7 @@ func main() { func getEnv(key string) string { v := os.Getenv(key) if v == "" { - log.Printf("warning: %s is not set", key) + slog.Warn(fmt.Sprintf("warning: %s is not set", key), "component", "startup") } return v } diff --git a/cmd/beacon/main_test.go b/cmd/beacon/main_test.go new file mode 100644 index 00000000..e62082c5 --- /dev/null +++ b/cmd/beacon/main_test.go @@ -0,0 +1,60 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "bytes" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestStartupLogging(t *testing.T) { + if os.Getenv("BEACON_TEST_LOG_STARTUP") == "1" { + main() + return + } + for _, level := range []string{"debug", "info", "warn", "error"} { + t.Run(level, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte("log: {level: debug, format: text}\n"), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(os.Args[0], "-test.run=^TestStartupLogging$") + cmd.Dir = dir + // Invalid port fails during parsing, before any database/network connection. + cmd.Env = append(os.Environ(), "BEACON_TEST_LOG_STARTUP=1", "CONFIG_PATH="+path, "LOG_LEVEL="+level, "LOG_FORMAT=json", "POSTGRES_DSN=postgres://test:do-not-log-this@localhost:invalid/beacon") + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatal("startup unexpectedly succeeded") + } + if strings.Contains(string(out), "do-not-log-this") { + t.Fatal("DSN credential leaked") + } + foundError := false + for _, line := range bytes.Split(bytes.TrimSpace(out), []byte("\n")) { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatalf("non-JSON startup output: %s", line) + } + if record["level"] == "ERROR" { + foundError = true + } + if level == "error" && record["level"] != "ERROR" { + t.Fatal("error threshold ignored") + } + if level == "warn" && record["level"] != "WARN" && record["level"] != "ERROR" { + t.Fatal("warn threshold ignored") + } + } + if !foundError { + t.Fatal("startup failure was silent") + } + }) + } +} diff --git a/config.yaml.example b/config.yaml.example index a65669df..a805495e 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -6,6 +6,11 @@ auth: api_key: "" +log: + # Environment LOG_LEVEL / LOG_FORMAT override these values. + level: info # debug, info, warn, error + format: text # text or json + server: # Only these direct proxy peers may set the client IP through X-Real-IP. # The proxy must overwrite that header, not pass through client input. diff --git a/db/migrate.go b/db/migrate.go index eebad5f2..7aa67255 100644 --- a/db/migrate.go +++ b/db/migrate.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io/fs" + "log/slog" "regexp" "sort" "strings" @@ -70,13 +71,13 @@ func applyMigration(ctx context.Context, db execQuerier, sql string) error { return fmt.Errorf("%w (checking index %s: %v)", err, name, scanErr) } if valid { - fmt.Printf("index %s already built, recording migration\n", name) + slog.Info(fmt.Sprintf("index %s already built, recording migration", name), "component", "db") return nil } if _, dropErr := db.Exec(ctx, "DROP INDEX CONCURRENTLY IF EXISTS "+ident); dropErr != nil { return fmt.Errorf("dropping invalid index %s: %w", name, dropErr) } - fmt.Printf("dropped invalid index %s, rebuilding\n", name) + slog.Warn(fmt.Sprintf("dropped invalid index %s, rebuilding", name), "component", "db") _, err = db.Exec(ctx, sql) return err } @@ -119,7 +120,7 @@ func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { ); err != nil { return fmt.Errorf("failed to bootstrap migrations: %w", err) } - fmt.Println("bootstrapped existing schema as 001_initial_schema.sql") + slog.Info("bootstrapped existing schema as 001_initial_schema.sql", "component", "db") } } @@ -166,7 +167,7 @@ func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { return fmt.Errorf("failed to record migration %s: %w", entry.Name(), err) } - fmt.Printf("applied migration: %s\n", entry.Name()) + slog.Info(fmt.Sprintf("applied migration: %s", entry.Name()), "component", "db") } return nil diff --git a/db/nodes.go b/db/nodes.go index a3a10744..9cc53e33 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -9,7 +9,7 @@ import ( "encoding/json" "errors" "fmt" - "log" + "log/slog" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -132,7 +132,7 @@ func (s *Store) ListNodes(ctx context.Context, nodeType int16, iatas []string, s } if len(v.Iatas) > 0 { if err := json.Unmarshal(v.Iatas, &node.IATAs); err != nil { - log.Printf("store: failed to unmarshal node iatas: %v", err) + slog.Error("store: failed to unmarshal node iatas", "component", "db", "error", err) node.IATAs = []api.NodeIATA{} } } @@ -184,13 +184,13 @@ func (s *Store) GetNode(ctx context.Context, nodeID uuid.UUID) (*api.Node, error } neighbors, err := s.GetNodeNeighbors(ctx, nodeID) if err != nil { - log.Printf("store: GetNodeNeighbors failed for %s: %v", nodeID, err) + slog.Error(fmt.Sprintf("store: GetNodeNeighbors failed for %s", nodeID), "component", "db", "error", err) neighbors = []api.NodeNeighbor{} } node.Neighbors = neighbors if len(row.Iatas) > 0 { if err := json.Unmarshal(row.Iatas, &node.IATAs); err != nil { - log.Printf("store: failed to unmarshal node iatas: %v", err) + slog.Error("store: failed to unmarshal node iatas", "component", "db", "error", err) node.IATAs = []api.NodeIATA{} } } diff --git a/db/observers.go b/db/observers.go index 43948c1d..a3448cf1 100644 --- a/db/observers.go +++ b/db/observers.go @@ -7,7 +7,7 @@ import ( "context" "encoding/hex" "fmt" - "log" + "log/slog" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -122,7 +122,7 @@ func (s *Store) GetObserver(ctx context.Context, observerID uuid.UUID) (*api.Obs } scopes, err := s.GetObserverScopes(ctx, observerID) if err != nil { - log.Printf("store: GetObserverScopes failed for %s: %v", observerID, err) + slog.Error(fmt.Sprintf("store: GetObserverScopes failed for %s", observerID), "component", "db", "error", err) scopes = []string{} } observer.Scopes = scopes @@ -357,7 +357,7 @@ func (s *Store) ListObserverAdverts(ctx context.Context, observerID uuid.UUID, c Limit: limit + 1, // fetch one extra to detect hasMore }) if err != nil { - log.Printf("api: ListObserverAdverts failed: %v", err) + slog.Error("api: ListObserverAdverts failed", "component", "db", "error", err) return api.Page[api.AdvertObservation]{}, err } hasMore := len(rows) > int(limit) diff --git a/db/packets.go b/db/packets.go index 2ea2c716..e6512694 100644 --- a/db/packets.go +++ b/db/packets.go @@ -10,7 +10,7 @@ import ( "encoding/json" "errors" "fmt" - "log" + "log/slog" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -477,7 +477,7 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, if len(traceRawHashes) > 0 { resolved, err := s.ResolvePathHashes(ctx, v.Iata, traceRawHashes) if err != nil { - log.Printf("store: path resolution failed for observation %d: %v", v.ID, err) + slog.Error(fmt.Sprintf("store: path resolution failed for observation %d", v.ID), "component", "db", "error", err) } else { resolvedPath = api.BuildResolvedPath(traceRawHashes, resolved) } @@ -490,7 +490,7 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, } resolved, err := s.ResolvePathHashes(ctx, v.Iata, hashes) if err != nil { - log.Printf("store: path resolution failed for observation %d: %v", v.ID, err) + slog.Error(fmt.Sprintf("store: path resolution failed for observation %d", v.ID), "component", "db", "error", err) } else { resolvedPath = api.BuildResolvedPath(hashes, resolved) } @@ -513,7 +513,7 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, obs.ResolvedSource = &hop } else if len(sourceHashByte) == 1 { if r, err := s.ResolveEndpointHashes(ctx, v.Iata, [][]byte{sourceHashByte}); err != nil { - log.Printf("store: source resolution failed for observation %d: %v", v.ID, err) + slog.Error(fmt.Sprintf("store: source resolution failed for observation %d", v.ID), "component", "db", "error", err) } else { hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] obs.ResolvedSource = &hop @@ -521,7 +521,7 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, } if len(destHashByte) == 1 { if r, err := s.ResolveEndpointHashes(ctx, v.Iata, [][]byte{destHashByte}); err != nil { - log.Printf("store: destination resolution failed for observation %d: %v", v.ID, err) + slog.Error(fmt.Sprintf("store: destination resolution failed for observation %d", v.ID), "component", "db", "error", err) } else { hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] obs.ResolvedDestination = &hop diff --git a/db/stats.go b/db/stats.go index abceda1d..0d91df70 100644 --- a/db/stats.go +++ b/db/stats.go @@ -6,7 +6,7 @@ package db import ( "context" "encoding/json" - "log" + "log/slog" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -186,7 +186,7 @@ func (s *Store) GetStatsClockDrift(ctx context.Context, iatas []string, limit in } if len(v.Iatas) > 0 { if err := json.Unmarshal(v.Iatas, &entry.IATAs); err != nil { - log.Printf("store: failed to unmarshal clock drift node iatas: %v", err) + slog.Error("store: failed to unmarshal clock drift node iatas", "component", "db", "error", err) entry.IATAs = []api.NodeIATA{} } } diff --git a/env.example b/env.example index 3594554b..97d7cbc4 100644 --- a/env.example +++ b/env.example @@ -3,6 +3,10 @@ LISTEN_ADDR=:8080 # Optional admin bearer key. If set, overrides auth.api_key (even when empty). # BEACON_API_KEY= +# Optional application logging overrides (defaults: info / text). +# LOG_LEVEL=info +# LOG_FORMAT=json + POSTGRES_DSN=postgres://beacon:beacon@localhost:5432/beacon?sslmode=disable # Redis (optional — leave REDIS_ADDR unset to disable caching) diff --git a/internal/api/handlers/limits.go b/internal/api/handlers/limits.go index 8ee6fb40..dfc48233 100644 --- a/internal/api/handlers/limits.go +++ b/internal/api/handlers/limits.go @@ -5,7 +5,7 @@ package handlers import ( "errors" - "log" + "log/slog" "net/http" "strconv" ) @@ -26,7 +26,7 @@ func parseLimit(r *http.Request, defaultLimit int32) (int32, error) { return 0, errors.New("limit must be positive") } if limit > int64(maxListLimit) { - log.Printf("api: clamped list limit from %d to %d for %s on %s", limit, maxListLimit, r.RemoteAddr, r.URL.Path) + slog.Info("api: clamped list limit", "component", "api", "requested_limit", limit, "limit", maxListLimit) return maxListLimit, nil } return int32(limit), nil diff --git a/internal/api/handlers/limits_logging_test.go b/internal/api/handlers/limits_logging_test.go new file mode 100644 index 00000000..54bfa147 --- /dev/null +++ b/internal/api/handlers/limits_logging_test.go @@ -0,0 +1,37 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "bytes" + "encoding/json" + "log" + "log/slog" + "net/http/httptest" + "strings" + "testing" +) + +func TestLimitLogOmitsRequestValues(t *testing.T) { + var out bytes.Buffer + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, nil))) + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) + req := httptest.NewRequest("GET", "/private-path%0Aforged?limit=1000", nil) + req.RemoteAddr = "private-peer" + limit, err := parseLimit(req, 50) + if err != nil || limit != 200 { + t.Fatalf("limit=%d error=%v", limit, err) + } + if strings.Contains(out.String(), "private-") || strings.Contains(out.String(), "forged") { + t.Fatal("limit log retained request identity") + } + var record map[string]any + if err := json.Unmarshal(out.Bytes(), &record); err != nil { + t.Fatal(err) + } + if record["requested_limit"] != float64(1000) || record["limit"] != float64(200) || record["component"] != "api" { + t.Fatalf("clamp diagnostics missing: %v", record) + } +} diff --git a/internal/api/handlers/responses.go b/internal/api/handlers/responses.go index 3a12ec12..e1ef3b28 100644 --- a/internal/api/handlers/responses.go +++ b/internal/api/handlers/responses.go @@ -6,7 +6,8 @@ package handlers import ( "encoding/json" - "log" + "fmt" + "log/slog" "net/http" "strings" ) @@ -25,7 +26,7 @@ func respond(w http.ResponseWriter, status int, data any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) if err := json.NewEncoder(w).Encode(data); err != nil { - log.Printf("api: failed to encode response: %v", err) + slog.Error("api: failed to encode response", "component", "api", "error", err) } } @@ -33,7 +34,7 @@ func respond(w http.ResponseWriter, status int, data any) { // The error code is derived automatically from the HTTP status text. func respondError(w http.ResponseWriter, status int, message string) { if status >= 500 { - log.Printf("api: error %d: %s", status, message) + slog.Error(fmt.Sprintf("api: error %d: %s", status, message), "component", "api") } code := strings.ToLower(strings.ReplaceAll(http.StatusText(status), " ", "_")) respond(w, status, map[string]APIError{"error": {Code: code, Message: message}}) diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index abd6c4df..71828d82 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -4,7 +4,7 @@ package handlers import ( - "log" + "log/slog" "net/http" "strconv" "time" @@ -66,7 +66,7 @@ func getStatsOverview(reader api.Reader) http.HandlerFunc { } overview, err := reader.GetStatsOverview(r.Context(), iatas) if err != nil { - log.Printf("api: GetStatsOverview failed: %v", err) + slog.Error("api: GetStatsOverview failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -108,7 +108,7 @@ func getStatsObservations(reader api.Reader) http.HandlerFunc { } points, err := reader.GetStatsObservations(r.Context(), iatas, since) if err != nil { - log.Printf("api: GetStatsObservations failed: %v", err) + slog.Error("api: GetStatsObservations failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -150,7 +150,7 @@ func getStatsPayloadBreakdown(reader api.Reader) http.HandlerFunc { } breakdown, err := reader.GetStatsPayloadBreakdown(r.Context(), iatas, since) if err != nil { - log.Printf("api: GetStatsPayloadBreakdown failed: %v", err) + slog.Error("api: GetStatsPayloadBreakdown failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -188,7 +188,7 @@ func getStatsTopNodes(reader api.Reader) http.HandlerFunc { } nodes, err := reader.GetStatsTopNodes(r.Context(), iatas, limit) if err != nil { - log.Printf("api: GetStatsTopNodes failed: %v", err) + slog.Error("api: GetStatsTopNodes failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -236,7 +236,7 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { } observers, err := reader.GetStatsTopObservers(r.Context(), iatas, since, limit) if err != nil { - log.Printf("api: GetStatsTopObservers failed: %v", err) + slog.Error("api: GetStatsTopObservers failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -284,7 +284,7 @@ func getStatsTopAdvertisers(reader api.Reader) http.HandlerFunc { } advertisers, err := reader.GetStatsTopAdvertisers(r.Context(), iatas, since, limit) if err != nil { - log.Printf("api: GetStatsTopAdvertisers failed: %v", err) + slog.Error("api: GetStatsTopAdvertisers failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -322,7 +322,7 @@ func getStatsClockDrift(reader api.Reader) http.HandlerFunc { } entries, err := reader.GetStatsClockDrift(r.Context(), iatas, limit) if err != nil { - log.Printf("api: GetStatsClockDrift failed: %v", err) + slog.Error("api: GetStatsClockDrift failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } @@ -370,7 +370,7 @@ func getStatsTopTalkers(reader api.Reader) http.HandlerFunc { } talkers, err := reader.GetStatsTopTalkers(r.Context(), iatas, since, limit) if err != nil { - log.Printf("api: GetStatsTopTalkers failed: %v", err) + slog.Error("api: GetStatsTopTalkers failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") return } diff --git a/internal/api/middleware/logging.go b/internal/api/middleware/logging.go new file mode 100644 index 00000000..cfff07df --- /dev/null +++ b/internal/api/middleware/logging.go @@ -0,0 +1,63 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "fmt" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/go-chi/chi/v5" + chimw "github.com/go-chi/chi/v5/middleware" +) + +// RequestLogger uses chi's response wrapper to preserve streaming/hijacking support. +// Place it after TrustedProxyIP and RequestID, and before Recoverer. +func RequestLogger(next http.Handler) http.Handler { + return chimw.RequestLogger(requestLogFormatter{})(next) +} + +type requestLogFormatter struct{} +type requestLogEntry struct{ request *http.Request } + +func (requestLogFormatter) NewLogEntry(r *http.Request) chimw.LogEntry { + return &requestLogEntry{request: r} +} + +func (e *requestLogEntry) logger() *slog.Logger { + path := "unmatched" + if ctx := chi.RouteContext(e.request.Context()); ctx != nil && ctx.RoutePattern() != "" { + path = ctx.RoutePattern() + } + return slog.Default().With("component", "http", "method", singleLineLogValue(e.request.Method), "path", path, + "client_ip", singleLineLogValue(e.request.RemoteAddr), "request_id", singleLineLogValue(chimw.GetReqID(e.request.Context()))) +} + +// Strip line breaks before request fields reach a handler or downstream log consumer. +func singleLineLogValue(value string) string { + value = strings.ReplaceAll(value, "\r", "") + return strings.ReplaceAll(value, "\n", "") +} + +func (e *requestLogEntry) Write(status, bytes int, _ http.Header, elapsed time.Duration, _ any) { + if status == 0 { + status = http.StatusOK + } + level := slog.LevelInfo + if status >= 500 { + level = slog.LevelError + } else if status >= 400 { + level = slog.LevelWarn + } + if !slog.Default().Enabled(e.request.Context(), level) { + return + } + e.logger().Log(e.request.Context(), level, "request complete", "status", status, "bytes", bytes, "duration_ms", float64(elapsed)/float64(time.Millisecond)) +} + +func (e *requestLogEntry) Panic(value any, stack []byte) { + e.logger().ErrorContext(e.request.Context(), "request panic", "panic", fmt.Sprint(value), "stack", string(stack)) +} diff --git a/internal/api/middleware/logging_test.go b/internal/api/middleware/logging_test.go new file mode 100644 index 00000000..f949389a --- /dev/null +++ b/internal/api/middleware/logging_test.go @@ -0,0 +1,143 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "bytes" + "encoding/json" + "log" + "log/slog" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "testing" + + "github.com/go-chi/chi/v5" + chimw "github.com/go-chi/chi/v5/middleware" +) + +func TestRequestLogging(t *testing.T) { + for _, tc := range []struct { + status int + panic bool + level slog.Level + wantLevel string + }{ + {200, false, slog.LevelInfo, "INFO"}, {429, false, slog.LevelWarn, "WARN"}, + {500, false, slog.LevelError, "ERROR"}, {200, false, slog.LevelWarn, ""}, {500, true, slog.LevelError, "ERROR"}, + } { + t.Run(tc.level.String()+"/"+http.StatusText(tc.status), func(t *testing.T) { + var out bytes.Buffer + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, &slog.HandlerOptions{Level: tc.level}))) + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) + r := chi.NewRouter() + r.Use(chimw.RequestID) + r.Use(TrustedProxyIP([]netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")})) + r.Use(RequestLogger) + r.Use(chimw.Recoverer) + r.Get("/items/{id}", func(w http.ResponseWriter, _ *http.Request) { + if tc.panic { + panic("test panic") + } + w.WriteHeader(tc.status) + }) + req := httptest.NewRequest(http.MethodGet, "/items/private-id?token=private-token", nil) + req.RemoteAddr = "192.0.2.1:4444" + req.Header.Set("X-Real-IP", "203.0.113.7") + req.Header.Set("True-Client-IP", "198.51.100.8") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != tc.status { + t.Fatalf("status=%d", w.Code) + } + if tc.wantLevel == "" { + if out.Len() != 0 { + t.Fatal("info request was not filtered") + } + return + } + if strings.Contains(out.String(), "private-id") || strings.Contains(out.String(), "private-token") || strings.Contains(out.String(), "198.51.100.8") { + t.Fatal("request values leaked or spoofed identity used") + } + var completed bool + for _, line := range bytes.Split(bytes.TrimSpace(out.Bytes()), []byte("\n")) { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal(err) + } + if record["level"] != tc.wantLevel || record["component"] != "http" || record["client_ip"] != "203.0.113.7" || record["path"] != "/items/{id}" { + t.Fatalf("bad record: %v", record) + } + if record["msg"] == "request complete" { + completed = true + if record["status"] != float64(tc.status) { + t.Fatal("wrong logged status") + } + } + } + if !completed { + t.Fatal("request completion missing") + } + }) + } +} + +func TestRequestLogInputBoundary(t *testing.T) { + for _, format := range []string{"text", "json"} { + for _, path := range []string{"/items/private-path", "/unmatched-private-path%0D%0Aforged"} { + t.Run(format+path, func(t *testing.T) { + var out bytes.Buffer + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + var handler slog.Handler = slog.NewTextHandler(&out, nil) + if format == "json" { + handler = slog.NewJSONHandler(&out, nil) + } + slog.SetDefault(slog.New(handler)) + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) + r := chi.NewRouter() + r.Use(chimw.RequestID, RequestLogger) + r.Get("/items/{id}", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(200) }) + req := httptest.NewRequest(http.MethodGet, path+"?token=private-token", nil) + req.RemoteAddr = "192.0.2.1:1234" + req.Header.Set("X-Request-ID", "client\r\nforged") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + wantPath, wantStatus := "/items/{id}", 200 + if strings.HasPrefix(path, "/unmatched") { + wantPath, wantStatus = "unmatched", 404 + } + if w.Code != wantStatus || strings.Count(out.String(), "\n") != 1 || strings.ContainsAny(strings.TrimSuffix(out.String(), "\n"), "\r\n") { + t.Fatalf("status or log boundary changed: %d %q", w.Code, out.String()) + } + if strings.Contains(out.String(), "private-path") || strings.Contains(out.String(), "private-token") { + t.Fatal("raw path or query was retained") + } + if format == "json" { + var record map[string]any + if err := json.Unmarshal(out.Bytes(), &record); err != nil { + t.Fatal(err) + } + if record["path"] != wantPath || record["request_id"] != "clientforged" || record["client_ip"] != "192.0.2.1:1234" { + t.Fatalf("unexpected request fields: %v", record) + } + } else if !strings.Contains(out.String(), "path="+wantPath) || !strings.Contains(out.String(), "request_id=clientforged") { + t.Fatalf("unexpected text fields: %s", out.String()) + } + }) + } + } +} + +func TestSingleLineLogValue(t *testing.T) { + for _, tc := range []struct{ input, want string }{ + {"", ""}, {"GET", "GET"}, {"[2001:db8::1]:80", "[2001:db8::1]:80"}, + {"client\r\nentry\n", "cliententry"}, {"café", "café"}, + } { + if got := singleLineLogValue(tc.input); got != tc.want { + t.Errorf("got %q, want %q", got, tc.want) + } + } +} diff --git a/internal/api/router/rate_limit_test.go b/internal/api/router/rate_limit_test.go index 676419f2..ac7f7aec 100644 --- a/internal/api/router/rate_limit_test.go +++ b/internal/api/router/rate_limit_test.go @@ -7,6 +7,7 @@ import ( "bytes" "encoding/json" "log" + "log/slog" "net/http" "net/http/httptest" "net/netip" @@ -17,8 +18,6 @@ import ( "testing/synctest" "time" - "github.com/go-chi/chi/v5/middleware" - "github.com/MeshCore-Beacon/beacon-server/internal/config" ) @@ -53,9 +52,9 @@ func rateRequest(handler http.Handler, path, peer, headerIP string) *httptest.Re func TestAPIRateLimitContractAndLogging(t *testing.T) { var logs bytes.Buffer - previous := middleware.DefaultLogger - middleware.DefaultLogger = middleware.RequestLogger(&middleware.DefaultLogFormatter{Logger: log.New(&logs, "", 0), NoColor: true}) - t.Cleanup(func() { middleware.DefaultLogger = previous }) + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + slog.SetDefault(slog.New(slog.NewJSONHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) proxy := config.ServerConfig{TrustedProxies: []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}} handler := New(nil, nil, nil, 5, config.CORSConfig{}, proxy, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 2, Burst: 10}) for _, path := range []string{"/api/v1/brokers", "/api/v1/packets?limit=0"} { @@ -77,8 +76,21 @@ func TestAPIRateLimitContractAndLogging(t *testing.T) { if response.Header().Get("Content-Type") != "application/json" || response.Header().Get("Retry-After") != "60" || !strings.EqualFold(response.Header().Get("Access-Control-Expose-Headers"), "Retry-After") || response.Header().Get("Access-Control-Allow-Origin") == "" { t.Fatalf("missing JSON/backoff/CORS headers: %v", response.Header()) } - if strings.Count(logs.String(), " - 429 ") != 1 || !strings.Contains(logs.String(), "from 198.51.100.25 - 429 ") || !strings.Contains(logs.String(), "/api/v1/brokers") { - t.Fatalf("expected one rejection log with resolved client and path: %s", logs.String()) + rejections := 0 + for _, line := range bytes.Split(bytes.TrimSpace(logs.Bytes()), []byte("\n")) { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal(err) + } + if record["status"] == float64(http.StatusTooManyRequests) { + rejections++ + if record["level"] != "WARN" || record["client_ip"] != "198.51.100.25" || record["path"] != "/api/v1/*" { + t.Fatalf("wrong rejection log: %v", record) + } + } + } + if rejections != 1 { + t.Fatalf("got %d rejection logs, want 1", rejections) } } diff --git a/internal/api/router/router.go b/internal/api/router/router.go index 84584133..7fe35ba4 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -71,7 +71,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP // ── Global middleware ──────────────────────────────────────────────────── r.Use(middleware.RequestID) r.Use(mw.TrustedProxyIP(serverCfg.TrustedProxies)) - r.Use(middleware.Logger) + r.Use(mw.RequestLogger) r.Use(middleware.Recoverer) r.Use(middleware.CleanPath) r.Use(middleware.StripSlashes) diff --git a/internal/background/background.go b/internal/background/background.go index 1ddf322f..e537328b 100644 --- a/internal/background/background.go +++ b/internal/background/background.go @@ -6,7 +6,7 @@ package background import ( "context" - "log" + "log/slog" "time" ) @@ -36,11 +36,11 @@ func (s *Scheduler) Start(ctx context.Context) { for { select { case <-ticker.C: - log.Printf("background[%s]: running", t.Name) + slog.Debug("task running", "component", "background", "task", t.Name) if err := t.Run(ctx); err != nil { - log.Printf("background[%s]: failed: %v", t.Name, err) + slog.Error("task failed", "component", "background", "task", t.Name, "error", err) } else { - log.Printf("background[%s]: complete", t.Name) + slog.Debug("task complete", "component", "background", "task", t.Name) } case <-ctx.Done(): return diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go index ca9fb999..af1dae22 100644 --- a/internal/background/tasks_test.go +++ b/internal/background/tasks_test.go @@ -4,11 +4,15 @@ package background import ( + "bytes" "context" + "encoding/json" "errors" "log" + "log/slog" "strings" "testing" + "testing/synctest" "time" "github.com/google/uuid" @@ -155,33 +159,35 @@ func TestViewRefreshTaskCancelled(t *testing.T) { } } -type taskLog chan string - -func (w taskLog) Write(p []byte) (int, error) { - line := string(p) - if strings.Contains(line, "complete") || strings.Contains(line, "failed:") { - w <- line - } - return len(p), nil -} - func TestSchedulerFailureIsNotComplete(t *testing.T) { - lines := make(taskLog, 1) - previous := log.Writer() - log.SetOutput(lines) - defer log.SetOutput(previous) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - New([]Task{{Name: "fails", Interval: time.Millisecond, Run: func(context.Context) error { - cancel() - return errors.New("refresh unavailable") - }}}).Start(ctx) - select { - case line := <-lines: - if strings.Contains(line, "complete") || !strings.Contains(line, "refresh unavailable") { - t.Fatalf("incorrect failure status: %s", line) + synctest.Test(t, func(t *testing.T) { + var out bytes.Buffer + previous, writer, flags := slog.Default(), log.Writer(), log.Flags() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, &slog.HandlerOptions{Level: slog.LevelDebug}))) + defer func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + New([]Task{{Name: "fails", Interval: time.Millisecond, Run: func(context.Context) error { cancel(); return errors.New("refresh unavailable") }}}).Start(ctx) + time.Sleep(2 * time.Millisecond) // advance the virtual clock to the first task tick + synctest.Wait() + failed := false + for _, line := range bytes.Split(bytes.TrimSpace(out.Bytes()), []byte("\n")) { + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal(err) + } + if record["msg"] == "task complete" { + t.Fatal("failed task reported complete") + } + if record["msg"] == "task failed" { + failed = true + if record["level"] != "ERROR" || record["error"] != "refresh unavailable" || record["task"] != "fails" { + t.Fatalf("incorrect failure: %v", record) + } + } } - case <-time.After(time.Second): - t.Fatal("scheduler did not report task outcome") - } + if !failed { + t.Fatal("scheduler did not report task failure") + } + }) } diff --git a/internal/config/config.go b/internal/config/config.go index 140b102b..9455a2e5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -18,6 +18,7 @@ import ( // Config is the top-level structure of the Beacon config file. type Config struct { Auth AuthConfig `yaml:"auth"` + Log LogConfig `yaml:"log"` Server ServerConfig `yaml:"server"` IATAs map[string]IATAConfig `yaml:"iatas"` Regions []RegionConfig `yaml:"regions"` @@ -43,6 +44,12 @@ type AuthConfig struct { APIKey string `yaml:"api_key" json:"-"` } +// LogConfig controls application verbosity and stderr output format. +type LogConfig struct { + Level string `yaml:"level"` + Format string `yaml:"format"` +} + // ServerConfig controls which direct peers may supply the client address. type ServerConfig struct { // TrustedProxies accepts IPv4/IPv6 CIDRs; an empty list trusts no proxy. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e73e581a..8ce2828c 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -11,6 +11,20 @@ import ( "time" ) +func TestLoadLogConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte("log:\n level: warn\n format: json\n"), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + if cfg.Log.Level != "warn" || cfg.Log.Format != "json" { + t.Fatalf("log config ignored: %+v", cfg.Log) + } +} + func TestLoadTrustedProxies(t *testing.T) { for _, tc := range []struct { name, yaml string diff --git a/internal/config/seed.go b/internal/config/seed.go index 2fbbe558..53ea3716 100644 --- a/internal/config/seed.go +++ b/internal/config/seed.go @@ -8,7 +8,7 @@ import ( "crypto/sha256" "encoding/json" "fmt" - "log" + "log/slog" "os" "strings" ) @@ -26,7 +26,7 @@ type Seeder interface { // Seed applies config-defined regions, IATA overrides to the database. // It is safe to call on every startup — all operations are upserts. func Seed(ctx context.Context, cfg *Config, db Seeder) error { - log.Printf("config: seeding %d IATAs, %d regions, %d scopes", len(cfg.IATAs), len(cfg.Regions), len(cfg.Scopes)) + slog.Info(fmt.Sprintf("config: seeding %d IATAs, %d regions, %d scopes", len(cfg.IATAs), len(cfg.Regions), len(cfg.Scopes)), "component", "config") // IATA overrides for iata, details := range cfg.IATAs { if err := db.UpsertIATADetails(ctx, iata, details.Name, details.Lat, details.Lng); err != nil { diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 1f8866df..3671afeb 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -17,7 +17,8 @@ package hub import ( "encoding/json" - "log" + "fmt" + "log/slog" "slices" ) @@ -211,7 +212,7 @@ func (h *Hub) Broadcast(e Event) { select { case h.broadcast <- e: default: - log.Println("hub: broadcast channel full, dropping event") + slog.Warn("hub: broadcast channel full, dropping event", "component", "hub") } } @@ -280,7 +281,7 @@ func (h *Hub) Run() { default: // laggedCh itself full; write pump will catch up on next drain } - log.Printf("hub: client send buffer full, dropped event type=%s", evt.Type) + slog.Warn(fmt.Sprintf("hub: client send buffer full, dropped event type=%s", evt.Type), "component", "hub") } } } diff --git a/internal/ingest/backfill.go b/internal/ingest/backfill.go index c0469129..f402a4de 100644 --- a/internal/ingest/backfill.go +++ b/internal/ingest/backfill.go @@ -7,7 +7,7 @@ import ( "context" "encoding/hex" "fmt" - "log" + "log/slog" "strings" "time" @@ -81,7 +81,7 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa // Non-fatal: the message is stored either way, so just log and continue -- matches // the live ingest path's existing behavior of not treating this as a hard failure. if err := db.SetPacketDecrypted(ctx, packetHash); err != nil { - log.Printf("ingest: failed to set packet decrypted for %s: %v", hex.EncodeToString(packetHash), err) + slog.Error(fmt.Sprintf("ingest: failed to set packet decrypted for %s", hex.EncodeToString(packetHash)), "component", "ingest", "error", err) } } return &DecryptGroupTextResult{ @@ -112,7 +112,7 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) ( for _, p := range packets { result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload) if err != nil { - log.Printf("ingest: backfill: decrypt failed for packet %s: %v", hex.EncodeToString(p.PacketHash), err) + slog.Error(fmt.Sprintf("ingest: backfill: decrypt failed for packet %s", hex.EncodeToString(p.PacketHash)), "component", "ingest", "error", err) continue } if result != nil && result.NewMessage { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index c2e485fc..e4edf863 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -40,7 +40,8 @@ import ( "crypto/rand" "encoding/binary" "encoding/json" - "log" + "fmt" + "log/slog" "strconv" "strings" "time" @@ -216,6 +217,7 @@ type ScopeStore interface { // Worker holds the dependencies for one broker's ingest loop. type Worker struct { + log *slog.Logger cfg Config db DB hub *hub.Hub @@ -228,7 +230,8 @@ type Worker struct { // New creates an ingest Worker. Call Start() to connect and begin processing. func New(cfg Config, db DB, h *hub.Hub, keys ChannelKeyStore, scopes ScopeStore) *Worker { - return &Worker{cfg: cfg, db: db, hub: h, keys: keys, scopes: scopes} + return &Worker{cfg: cfg, db: db, hub: h, keys: keys, scopes: scopes, + log: slog.Default().With("component", "ingest", "broker", cfg.BrokerName)} } // Start connects to the broker and blocks until ctx is cancelled. It @@ -253,22 +256,22 @@ func (w *Worker) Start(ctx context.Context) { SetConnectRetry(true). SetConnectRetryInterval(5 * time.Second). SetOnConnectHandler(func(c mqtt.Client) { - log.Printf("ingest[%s]: connected to %s", w.cfg.BrokerName, w.cfg.URL) + w.log.Info("connected") w.subscribe(c) }). SetConnectionLostHandler(func(_ mqtt.Client, err error) { - log.Printf("ingest[%s]: connection lost, will reconnect: %v", w.cfg.BrokerName, err) + w.log.Warn("connection lost, will reconnect", "error", err) }) w.client = mqtt.NewClient(opts) if tok := w.client.Connect(); tok.Wait() && tok.Error() != nil { - log.Printf("ingest[%s]: initial connect failed: %v", w.cfg.BrokerName, tok.Error()) + w.log.Error("initial connect failed", "error", tok.Error()) // paho will retry; we fall through and wait for ctx } <-ctx.Done() w.client.Disconnect(500) - log.Printf("ingest[%s]: stopped", w.cfg.BrokerName) + w.log.Info("stopped") } func (w *Worker) BrokerName() string { @@ -296,7 +299,7 @@ func (w *Worker) subscribe(client mqtt.Client) { w.handleMessage(msg) }) if tok.Wait() && tok.Error() != nil { - log.Printf("ingest[%s]: subscribe error: %v", w.cfg.BrokerName, tok.Error()) + w.log.Error("subscribe error", "error", tok.Error()) } } @@ -329,14 +332,16 @@ func (w *Worker) handleMessage(msg mqtt.Message) { // iata_codes.iata is CHAR(3); anything else would fail the DB insert // downstream, so reject malformed topic segments here instead. if !isValidIATA(iata) { - log.Printf("ingest[%s]: dropped packet with malformed IATA %q on topic %s", w.cfg.BrokerName, iata, msg.Topic()) + w.log.Warn("dropped packet with malformed IATA", "iata", iata, "topic", msg.Topic()) return } // Drop packets from IATAs outside the configured geographic filter. if w.cfg.AllowedIATAs != nil { if _, ok := w.cfg.AllowedIATAs[iata]; !ok { - log.Printf("ingest[%s]: dropped packet from %s (not in allowed IATAs)", w.cfg.BrokerName, iata) + if w.log.Enabled(context.Background(), slog.LevelDebug) { + w.log.Debug(fmt.Sprintf("dropped packet from %s (not in allowed IATAs)", iata)) + } return } } @@ -358,7 +363,7 @@ func (w *Worker) handleMessage(msg mqtt.Message) { func (w *Worker) broadcast(eventType hub.EventType, iata string, payloadType uint8, channelHash string, payload any) { b, err := json.Marshal(payload) if err != nil { - log.Printf("ingest[%s]: failed to marshal %s event: %v", w.cfg.BrokerName, eventType, err) + w.log.Error(fmt.Sprintf("failed to marshal %s event", eventType), "error", err) return } w.hub.Broadcast(hub.Event{ @@ -380,13 +385,13 @@ func (w *Worker) broadcast(eventType hub.EventType, iata string, payloadType uin func (w *Worker) broadcastPacketObservation(iata string, payloadType uint8, evt packetObservationEvent, resolvedPath []api.ResolvedHop) { base, err := json.Marshal(evt) if err != nil { - log.Printf("ingest[%s]: failed to marshal packetObservation event: %v", w.cfg.BrokerName, err) + w.log.Error("failed to marshal packetObservation event", "error", err) return } evt.Observation.ResolvedPath = resolvedPath resolved, err := json.Marshal(evt) if err != nil { - log.Printf("ingest[%s]: failed to marshal packetObservation event (resolved variant): %v", w.cfg.BrokerName, err) + w.log.Error("failed to marshal packetObservation event (resolved variant)", "error", err) resolved = nil // fall back to base-only; not fatal } w.hub.Broadcast(hub.Event{ diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 47830116..ea206c5d 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "encoding/json" "errors" + "log/slog" "testing" "time" @@ -303,6 +304,7 @@ func (s *stubDB) UpdateObserverRegionScope(_ context.Context, _ uuid.UUID, _ str func newTestWorker() (*Worker, *stubDB) { db := &stubDB{} w := &Worker{ + log: slog.Default().With("component", "ingest", "broker", "test"), cfg: Config{BrokerName: "test"}, db: db, hub: hub.New(), diff --git a/internal/ingest/logging_test.go b/internal/ingest/logging_test.go new file mode 100644 index 00000000..fa8bcc49 --- /dev/null +++ b/internal/ingest/logging_test.go @@ -0,0 +1,63 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "bytes" + "encoding/json" + "log/slog" + "strconv" + "strings" + "testing" +) + +type malformedTopicMessage struct{ topic string } + +func (m malformedTopicMessage) Duplicate() bool { return false } +func (m malformedTopicMessage) Qos() byte { return 0 } +func (m malformedTopicMessage) Retained() bool { return false } +func (m malformedTopicMessage) Topic() string { return m.topic } +func (m malformedTopicMessage) MessageID() uint16 { return 0 } +func (m malformedTopicMessage) Payload() []byte { return nil } +func (m malformedTopicMessage) Ack() {} + +func TestMalformedIATAWarnNamesSource(t *testing.T) { + for _, format := range []string{"json", "text"} { + for _, iata := range []string{"XX", "XX\r\nforged=true"} { + t.Run(format+"/"+strconv.Quote(iata), func(t *testing.T) { + var output bytes.Buffer + var handler slog.Handler = slog.NewJSONHandler(&output, nil) + if format == "text" { + handler = slog.NewTextHandler(&output, nil) + } + previous := slog.Default() + slog.SetDefault(slog.New(handler)) + t.Cleanup(func() { slog.SetDefault(previous) }) + worker, _ := newTestWorker() + topic := "meshcore/" + iata + "/0011223344556677/packets" + worker.handleMessage(malformedTopicMessage{topic: topic}) + if bytes.Count(output.Bytes(), []byte("\n")) != 1 || bytes.ContainsRune(output.Bytes(), '\r') { + t.Fatalf("warning did not remain one escaped record: %q", output.String()) + } + if format == "json" { + var record map[string]any + if err := json.Unmarshal(output.Bytes(), &record); err != nil { + t.Fatal(err) + } + if record["iata"] != iata || record["topic"] != topic || record["msg"] != "dropped packet with malformed IATA" { + t.Fatalf("warning lost its source: %v", record) + } + } else { + expectedIATA, expectedTopic := iata, topic + if strings.ContainsAny(iata, "\r\n") { + expectedIATA, expectedTopic = strconv.Quote(iata), strconv.Quote(topic) + } + if !strings.Contains(output.String(), "iata="+expectedIATA) || !strings.Contains(output.String(), "topic="+expectedTopic) { + t.Fatalf("warning lost its source: %q", output.String()) + } + } + }) + } + } +} diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go index f16e08a3..f96a32a1 100644 --- a/internal/ingest/neighbors.go +++ b/internal/ingest/neighbors.go @@ -7,7 +7,7 @@ import ( "context" "encoding/hex" "encoding/json" - "log" + "fmt" ) // neighborReportEntry is one entry in the "neighbors" array of a /neighbors report. @@ -33,30 +33,30 @@ type neighborReport struct { func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, raw []byte) { var report neighborReport if err := json.Unmarshal(raw, &report); err != nil { - log.Printf("ingest[%s]: malformed neighbors envelope from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("malformed neighbors envelope from %s", pubkeyHex), "error", err) return } pubkey, err := hex.DecodeString(pubkeyHex) if err != nil { - log.Printf("ingest[%s]: invalid pubkey hex in neighbors from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("invalid pubkey hex in neighbors from %s", pubkeyHex), "error", err) return } observerID, _, err := w.db.UpsertObserver(ctx, pubkey) if err != nil { - log.Printf("ingest[%s]: db: upsert observer failed in neighbors from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert observer failed in neighbors from %s", pubkeyHex), "error", err) return } if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName); err != nil { - log.Printf("ingest[%s]: db: upsert observer broker failed in neighbors from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert observer broker failed in neighbors from %s", pubkeyHex), "error", err) } // self.scopes is always known (it's the observer's own config, not an OTA // query), so this is an unconditional write -- unlike the per-neighbor // scopes below, there's no "query failed" case to protect against here. if err := w.db.UpdateObserverRegionScope(ctx, observerID, report.Self.Scopes); err != nil { - log.Printf("ingest[%s]: db: update observer region scope failed for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: update observer region scope failed for %s", pubkeyHex), "error", err) } // The observer's own node row (as opposed to its observers row above) is @@ -72,7 +72,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra for _, n := range report.Neighbors { neighborPubkey, err := hex.DecodeString(n.PubKey) if err != nil { - log.Printf("ingest[%s]: invalid neighbor pubkey hex %q from %s: %v", w.cfg.BrokerName, n.PubKey, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("invalid neighbor pubkey hex %q from %s", n.PubKey, pubkeyHex), "error", err) continue } neighborNodeID, err := w.db.GetNodeByPubkey(ctx, neighborPubkey) @@ -98,7 +98,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra } if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, neighborNodeID, iata, &snr, regionScope); err != nil { - log.Printf("ingest[%s]: db: upsert neighbor failed for %s -> %s: %v", w.cfg.BrokerName, pubkeyHex, n.PubKey, err) + w.log.Error(fmt.Sprintf("db: upsert neighbor failed for %s -> %s", pubkeyHex, n.PubKey), "error", err) } } } diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 59408def..3242844b 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -11,7 +11,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "log" "strings" "time" @@ -290,46 +289,46 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } err := json.Unmarshal(raw, &envelope) if err != nil { - log.Printf("ingest[%s]: malformed packet envelope from %s/%s", w.cfg.BrokerName, iata, pubkeyHex) + w.log.Warn(fmt.Sprintf("malformed packet envelope from %s/%s", iata, pubkeyHex)) return } if envelope.Raw == "" { if envelope.Len == "0" || envelope.Len == "" { return // observer keepalive with no packet data } - log.Printf("ingest[%s]: malformed packet envelope from %s/%s", w.cfg.BrokerName, iata, pubkeyHex) + w.log.Warn(fmt.Sprintf("malformed packet envelope from %s/%s", iata, pubkeyHex)) return } hexBytes, err := hex.DecodeString(strings.ReplaceAll(envelope.Raw, " ", "")) if err != nil { - log.Printf("ingest[%s]: invalid hex from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("invalid hex from %s/%s", iata, pubkeyHex), "error", err) return } packet, err := meshcore.PacketFromBytes(hexBytes) if err != nil { - log.Printf("ingest[%s]: error decoding packet from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error decoding packet from %s/%s", iata, pubkeyHex), "error", err) return } pubkeyBytes, err := hex.DecodeString(pubkeyHex) if err != nil { - log.Printf("ingest[%s]: invalid pubkey hex from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("invalid pubkey hex from %s/%s", iata, pubkeyHex), "error", err) return } id, observerName, err := w.db.UpsertObserver(ctx, pubkeyBytes) if err != nil { - log.Printf("ingest[%s]: db: upsert observer failed with packet from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert observer failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName) if err != nil { - log.Printf("ingest[%s]: db: update observer broker failed with packet from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: update observer broker failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } err = w.db.UpsertIATA(ctx, iata) if err != nil { - log.Printf("ingest[%s]: db: upsert IATA failed with packet from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert IATA failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } packetHash := packet.PacketHash() @@ -584,7 +583,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if len(prevEntries) == 1 && len(currEntries) == 1 { snr := snrValues[i] if err := w.db.UpsertNodeNeighbor(ctx, currEntries[0].NodeID, prevEntries[0].NodeID, iata, &snr, nil); err != nil { - log.Printf("ingest[%s]: failed to upsert trace neighbor: %v", w.cfg.BrokerName, err) + w.log.Error("failed to upsert trace neighbor", "error", err) } } } @@ -664,7 +663,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if oErr == nil && rErr == nil && observerNodeID != responderNodeID { rxSNR := float32(parseNumber(envelope.SNR)) if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, responderNodeID, iata, &rxSNR, nil); err != nil { - log.Printf("ingest[%s]: failed to upsert observer-discover neighbor: %v", w.cfg.BrokerName, err) + w.log.Error("failed to upsert observer-discover neighbor", "error", err) } } } @@ -703,7 +702,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if matchedScope != nil { id, err := w.db.GetTransportScopeByName(ctx, *matchedScope) if err != nil { - log.Printf("ingest[%s]: failed to get scope ID for %s: %v", w.cfg.BrokerName, *matchedScope, err) + w.log.Error(fmt.Sprintf("failed to get scope ID for %s", *matchedScope), "error", err) } else { scopeID = &id } @@ -728,7 +727,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } isNew, err := w.db.UpsertPacket(ctx, pParams) if err != nil { - log.Printf("ingest[%s]: db: upsert packet failed from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert packet failed from %s/%s", iata, pubkeyHex), "error", err) return } // Try parsing with timezone offset first @@ -746,21 +745,21 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ heardAt, err = time.Parse("2006-01-02T15:04:05Z", envelope.Timestamp) } if err != nil { - log.Printf("ingest[%s]: failed to parse timestamp %q: %v", w.cfg.BrokerName, envelope.Timestamp, err) + w.log.Warn(fmt.Sprintf("failed to parse timestamp %q", envelope.Timestamp), "error", err) heardAt = time.Now().UTC() } else { // clamp to server time if offset is suspicious (> 30 min drift) now := time.Now().UTC() diff := heardAt.UTC().Sub(now) if diff > 30*time.Minute || diff < -30*time.Minute { - log.Printf("ingest[%s]: clamping suspicious timestamp %s (diff %v) for pubkey %s", w.cfg.BrokerName, envelope.Timestamp, diff, pubkeyHex[:8]) + w.log.Warn(fmt.Sprintf("clamping suspicious timestamp %s (diff %v) for pubkey %s", envelope.Timestamp, diff, pubkeyHex[:8])) heardAt = now } } radio, err := w.db.GetObserverRadio(ctx, id) if err != nil { - log.Printf("ingest[%s]: db: get observer radio failed for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: get observer radio failed for %s", pubkeyHex), "error", err) } // Save the same endpoint resolution used by the live event in the observation INSERT. var resolvedSource, resolvedDestination *api.ResolvedHop @@ -790,7 +789,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if snapshot.HasResolvedNodes() { resolvedEndpoints, err = json.Marshal(snapshot) if err != nil { - log.Printf("ingest[%s]: endpoint snapshot encoding failed: %v", w.cfg.BrokerName, err) + w.log.Error("endpoint snapshot encoding failed", "error", err) resolvedEndpoints = nil // optional enrichment must not discard the observation } } @@ -824,27 +823,27 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } inserted, err := w.db.InsertObservation(ctx, oParams) if err != nil { - log.Printf("ingest[%s]: db: insert observation failed from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: insert observation failed from %s/%s", iata, pubkeyHex), "error", err) return } if scopeID != nil && inserted { if err := w.db.UpsertObserverScope(ctx, id, *scopeID); err != nil { - log.Printf("ingest[%s]: failed to upsert observer scope for %s: %v", w.cfg.BrokerName, id, err) + w.log.Error(fmt.Sprintf("failed to upsert observer scope for %s", id), "error", err) } } // Runs on duplicate observations too; the upsert only writes when the row is >1h stale. if channelHash != nil { if err := w.db.UpsertChannelIATA(ctx, channelHash, iata, heardAt); err != nil { - log.Printf("ingest[%s]: db: upsert channel IATA failed from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert channel IATA failed from %s/%s", iata, pubkeyHex), "error", err) } } // Runs on duplicate observations too; the upsert only writes when the row is >1h stale. if traceTag != nil { if err := w.db.UpsertTraceIATA(ctx, traceTag, iata, heardAt); err != nil { - log.Printf("ingest[%s]: db: upsert trace IATA failed from %s/%s: %v", w.cfg.BrokerName, iata, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert trace IATA failed from %s/%s", iata, pubkeyHex), "error", err) } } @@ -858,7 +857,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } resolved, err := w.db.ResolvePathHashes(ctx, iata, hashes) if err != nil { - log.Printf("ingest[%s]: path resolution failed: %v", w.cfg.BrokerName, err) + w.log.Error("path resolution failed", "error", err) } var resolvedIDs []uuid.UUID for _, entries := range resolved { @@ -882,7 +881,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } if allHigh && len(nodeIDs) > 1 { if err := w.db.UpsertKnownRoute(ctx, nodeIDs, hashPrefixes, iata, int32(len(nodeIDs))); err != nil { - log.Printf("ingest[%s]: failed to upsert known route: %v", w.cfg.BrokerName, err) + w.log.Error("failed to upsert known route", "error", err) } } } @@ -912,7 +911,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ evt.Observation.PropagationTimeMs = 0 // not yet calculated count, err := w.db.GetPacketObservationCount(ctx, packetHash[:]) if err != nil { - log.Printf("ingest[%s]: failed to get observation count: %v", w.cfg.BrokerName, err) + w.log.Error("failed to get observation count", "error", err) count = 0 } evt.Packet.ObservationCount = count diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index f465c99d..26d01849 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -7,7 +7,6 @@ import ( "context" "encoding/hex" "fmt" - "log" "strings" "time" @@ -72,11 +71,11 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc if packet.PayloadType() == meshcore.PayloadTypeAdvert { advert, err := meshcore.AdvertFromBytes(packet.Payload) if err != nil { - log.Printf("ingest[%s]: error decoding advert payload: %v", w.cfg.BrokerName, err) + w.log.Warn("error decoding advert payload", "error", err) return } if !advert.Verify() { - log.Printf("ingest[%s]: dropped advert with invalid signature from pubkey %s", w.cfg.BrokerName, hex.EncodeToString(advert.PublicKey.PublicKeyBytes())) + w.log.Warn(fmt.Sprintf("dropped advert with invalid signature from pubkey %s", hex.EncodeToString(advert.PublicKey.PublicKeyBytes()))) return } var lat, lon *float64 @@ -101,7 +100,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc } nodeID, err := w.db.UpsertNode(ctx, params, nodeRadio) if err != nil { - log.Printf("ingest[%s]: db: upsert node failed: %v", w.cfg.BrokerName, err) + w.log.Error("db: upsert node failed", "error", err) return } // invalidate cache for this node @@ -118,7 +117,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc key := hex.EncodeToString(firstHop[0]) if entries := resolved[key]; len(entries) == 1 { if err := w.db.UpsertNodeNeighbor(ctx, nodeID, entries[0].NodeID, iata, nil, nil); err != nil { - log.Printf("ingest[%s]: failed to upsert node neighbor: %v", w.cfg.BrokerName, err) + w.log.Error("failed to upsert node neighbor", "error", err) } } } @@ -133,21 +132,21 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc if oErr == nil && observerNodeID != nodeID { snr := rxSNR if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, nodeID, iata, &snr, nil); err != nil { - log.Printf("ingest[%s]: failed to upsert observer-advert neighbor: %v", w.cfg.BrokerName, err) + w.log.Error("failed to upsert observer-advert neighbor", "error", err) } } } if err := w.db.UpsertNodeIATA(ctx, nodeID, iata); err != nil { - log.Printf("ingest[%s]: db: upsert node IATA failed: %v", w.cfg.BrokerName, err) + w.log.Error("db: upsert node IATA failed", "error", err) } if scopeID != nil && (packet.RouteType() == meshcore.RouteTypeTransportFlood || packet.RouteType() == meshcore.RouteTypeTransportDirect) { if err := w.db.SetNodeDefaultScope(ctx, nodeID, *scopeID); err != nil { - log.Printf("ingest[%s]: failed to set default scope for node %s: %v", w.cfg.BrokerName, hex.EncodeToString(advert.PublicKey.PublicKeyBytes()), err) + w.log.Error(fmt.Sprintf("failed to set default scope for node %s", hex.EncodeToString(advert.PublicKey.PublicKeyBytes())), "error", err) } } prefix4 := advert.PublicKey.PublicKeyBytes()[:4] if err := w.db.UpsertNodeShortID(ctx, nodeID, iata, prefix4); err != nil { - log.Printf("ingest[%s]: failed to upsert node short ID for %s: %v", w.cfg.BrokerName, hex.EncodeToString(prefix4), err) + w.log.Error(fmt.Sprintf("failed to upsert node short ID for %s", hex.EncodeToString(prefix4)), "error", err) } pubkeyHex := hex.EncodeToString(advert.PublicKey.PublicKeyBytes()) isObserver := w.db.IsObserverByPubkey(ctx, advert.PublicKey.PublicKeyBytes()) @@ -180,14 +179,14 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc if packet.PayloadType() == meshcore.PayloadTypeGrpTxt { grpTxt, err := meshcore.GroupTextFromBytes(packet.Payload) if err != nil { - log.Printf("ingest[%s]: error decoding group text payload: %v", w.cfg.BrokerName, err) + w.log.Warn("error decoding group text payload", "error", err) return } channelHashBytes := []byte{grpTxt.ChannelHash} result, err := DecryptGroupText(ctx, w.db, w.keys, packetHash, packet.Payload) if err != nil { - log.Printf("ingest[%s]: decrypt group text failed: %v", w.cfg.BrokerName, err) + w.log.Error("decrypt group text failed", "error", err) return } if result == nil { diff --git a/internal/ingest/status.go b/internal/ingest/status.go index fb540f3c..31c80647 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -8,7 +8,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "log" "strconv" "strings" "time" @@ -71,17 +70,17 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) } `json:"stats"` } if err := json.Unmarshal(raw, &envelope); err != nil { - log.Printf("ingest[%s]: malformed status envelope from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("malformed status envelope from %s", pubkeyHex), "error", err) return } pubkey, err := hex.DecodeString(pubkeyHex) if err != nil { - log.Printf("ingest[%s]: invalid pubkey hex in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("invalid pubkey hex in status from %s", pubkeyHex), "error", err) return } id, _, err := w.db.UpsertObserver(ctx, pubkey) if err != nil { - log.Printf("ingest[%s]: db: upsert observer failed in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert observer failed in status from %s", pubkeyHex), "error", err) return } // invalidate cache for observer details @@ -89,7 +88,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) w.onObserverUpsert(ctx, id) } if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName); err != nil { - log.Printf("ingest[%s]: db: upsert observer broker failed in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: upsert observer broker failed in status from %s", pubkeyHex), "error", err) } params := UpdateObserverStatusParams{ PublicKey: pubkey, @@ -122,29 +121,29 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) radio := strings.Split(strings.TrimSpace(envelope.RadioString), ",") if len(radio) != 4 { - log.Printf("ingest[%s]: missing or malformed radio params in status from %s, skipping radio fields", w.cfg.BrokerName, pubkeyHex) + w.log.Warn(fmt.Sprintf("missing or malformed radio params in status from %s, skipping radio fields", pubkeyHex)) } else { freq, err := strconv.ParseFloat(radio[0], 32) if err != nil { - log.Printf("ingest[%s]: error parsing radio freq in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error parsing radio freq in status from %s", pubkeyHex), "error", err) } else { params.RadioFreqMHz = float32(freq) } bw, err := strconv.ParseFloat(radio[1], 32) if err != nil { - log.Printf("ingest[%s]: error parsing radio bw in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error parsing radio bw in status from %s", pubkeyHex), "error", err) } else { params.RadioBWKHz = float32(bw) } sf, err := strconv.ParseInt(radio[2], 10, 16) if err != nil { - log.Printf("ingest[%s]: error parsing radio sf in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error parsing radio sf in status from %s", pubkeyHex), "error", err) } else { params.RadioSF = int16(sf) } cr, err := strconv.ParseInt(radio[3], 10, 16) if err != nil { - log.Printf("ingest[%s]: error parsing radio cr in status from %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Warn(fmt.Sprintf("error parsing radio cr in status from %s", pubkeyHex), "error", err) } else { params.RadioCR = int16(cr) } @@ -152,7 +151,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) observerID, err := w.db.UpdateObserverStatus(ctx, params) if err != nil { - log.Printf("ingest[%s]: db: update observer status failed for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: update observer status failed for %s", pubkeyHex), "error", err) return } // Store a telemetry snapshot at the configured resolution. @@ -161,7 +160,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) // stats — skip the insert rather than writing an all-zero row that would win // the hourly dedup and pollute the telemetry aggregates. if envelope.Stats.UptimeSeconds == 0 { - log.Printf("ingest[%s]: status from %s has no usable stats (uptime_secs missing or zero), skipping telemetry insert", w.cfg.BrokerName, pubkeyHex) + w.log.Debug("status has no usable stats; skipping telemetry insert") } else { resolution := w.cfg.TelemetryResolution if resolution == 0 { @@ -180,7 +179,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) envelope.Stats.NoiseFloor, envelope.Stats.UptimeSeconds, &queueLen, &debugFlags, &recvErrors, ); err != nil { - log.Printf("ingest[%s]: db: insert telemetry failed for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("db: insert telemetry failed for %s", pubkeyHex), "error", err) } } @@ -190,7 +189,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) } scopes, err := w.db.GetObserverScopes(ctx, observerID) if err != nil { - log.Printf("ingest[%s]: failed to get observer scopes for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("failed to get observer scopes for %s", pubkeyHex), "error", err) scopes = []string{} } var radioStr *string @@ -216,7 +215,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) } payload, err := json.Marshal(evt) if err != nil { - log.Printf("ingest[%s]: failed to marshal status event payload for %s: %v", w.cfg.BrokerName, pubkeyHex, err) + w.log.Error(fmt.Sprintf("failed to marshal status event payload for %s", pubkeyHex), "error", err) return } w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus, Payload: payload, IATA: iata}) diff --git a/internal/logging/logging.go b/internal/logging/logging.go new file mode 100644 index 00000000..e7e738cb --- /dev/null +++ b/internal/logging/logging.go @@ -0,0 +1,47 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package logging + +import ( + "fmt" + "io" + "log/slog" + "os" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +// New resolves environment overrides, validates options and writes to out. +// Empty settings default to info/text; callers install the logger before starting workers. +func New(out io.Writer, cfg config.LogConfig) (*slog.Logger, error) { + if value := os.Getenv("LOG_LEVEL"); value != "" { + cfg.Level = value + } + if value := os.Getenv("LOG_FORMAT"); value != "" { + cfg.Format = value + } + var level slog.Level + switch strings.ToLower(strings.TrimSpace(cfg.Level)) { + case "", "info": + level = slog.LevelInfo + case "debug": + level = slog.LevelDebug + case "warn": + level = slog.LevelWarn + case "error": + level = slog.LevelError + default: + return nil, fmt.Errorf("log level must be debug, info, warn or error") + } + opts := &slog.HandlerOptions{Level: level} + switch strings.ToLower(strings.TrimSpace(cfg.Format)) { + case "", "text": + return slog.New(slog.NewTextHandler(out, opts)), nil + case "json": + return slog.New(slog.NewJSONHandler(out, opts)), nil + default: + return nil, fmt.Errorf("log format must be text or json") + } +} diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go new file mode 100644 index 00000000..196ddaba --- /dev/null +++ b/internal/logging/logging_test.go @@ -0,0 +1,97 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package logging + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestLevelsAndFormats(t *testing.T) { + for _, format := range []string{"text", "json"} { + for i, level := range []string{"debug", "info", "warn", "error"} { + t.Run(format+"/"+level, func(t *testing.T) { + t.Setenv("LOG_LEVEL", "") + t.Setenv("LOG_FORMAT", "") + var out bytes.Buffer + logger, err := New(&out, config.LogConfig{Level: level, Format: format}) + if err != nil { + t.Fatal(err) + } + logger = logger.With("component", "ingest", "broker", "test") + logger.Debug("debug message") + logger.Info("info message") + logger.Warn("warn message") + logger.Error("error message") + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + if len(lines) != 4-i { + t.Fatalf("got %d records, want %d: %s", len(lines), 4-i, out.String()) + } + for _, line := range lines { + if format == "json" { + var record map[string]any + if err := json.Unmarshal([]byte(line), &record); err != nil { + t.Fatal(err) + } + if record["component"] != "ingest" || record["broker"] != "test" || record["time"] == nil { + t.Fatalf("missing fields: %v", record) + } + } else if !strings.Contains(line, "component=ingest") || !strings.Contains(line, "broker=test") { + t.Fatalf("missing fields: %s", line) + } + } + if !strings.Contains(out.String(), "error message") { + t.Fatal("error suppressed") + } + }) + } + } +} + +func TestDefaultsAndEnvironment(t *testing.T) { + t.Setenv("LOG_LEVEL", "") + t.Setenv("LOG_FORMAT", "") + var out bytes.Buffer + logger, err := New(&out, config.LogConfig{}) + if err != nil { + t.Fatal(err) + } + if logger.Enabled(t.Context(), slog.LevelDebug) || !logger.Enabled(t.Context(), slog.LevelInfo) { + t.Fatal("default must be info") + } + t.Setenv("LOG_LEVEL", " WARN ") + t.Setenv("LOG_FORMAT", "JSON") + logger, err = New(&out, config.LogConfig{Level: "debug", Format: "text"}) + if err != nil { + t.Fatal(err) + } + logger.Info("hidden") + logger.Error("visible") + var record map[string]any + if err = json.Unmarshal(bytes.TrimSpace(out.Bytes()), &record); err != nil { + t.Fatal(err) + } + if record["level"] != "ERROR" || record["msg"] != "visible" { + t.Fatalf("env overrides ignored: %v", record) + } +} + +func TestInvalidOptions(t *testing.T) { + for _, cfg := range []config.LogConfig{{Level: "verbose"}, {Level: "off"}, {Level: "INFO+2"}, {Format: "xml"}} { + t.Setenv("LOG_LEVEL", "") + t.Setenv("LOG_FORMAT", "") + if _, err := New(&bytes.Buffer{}, cfg); err == nil { + t.Fatalf("accepted invalid options: %+v", cfg) + } + } + t.Setenv("LOG_LEVEL", "invalid") + if _, err := New(&bytes.Buffer{}, config.LogConfig{Level: "info"}); err == nil { + t.Fatal("invalid environment override accepted") + } +} diff --git a/internal/presence/coalescer.go b/internal/presence/coalescer.go index c8e3544a..9315f433 100644 --- a/internal/presence/coalescer.go +++ b/internal/presence/coalescer.go @@ -11,7 +11,8 @@ package presence import ( "context" - "log" + "fmt" + "log/slog" "sync" "time" @@ -258,7 +259,7 @@ func (c *Coalescer) Flush(ctx context.Context) { c.mu.Unlock() if err := c.flushObservers(ctx, observers); err != nil { - log.Printf("presence: flush observers failed (%d rows dropped): %v", len(observers), err) + slog.Error(fmt.Sprintf("presence: flush observers failed (%d rows dropped)", len(observers)), "component", "presence", "error", err) } if len(brokers) > 0 { @@ -271,7 +272,7 @@ func (c *Coalescer) Flush(ctx context.Context) { seen = append(seen, ts) } if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen); err != nil { - log.Printf("presence: flush observer brokers failed (%d rows dropped): %v", len(ids), err) + slog.Error(fmt.Sprintf("presence: flush observer brokers failed (%d rows dropped)", len(ids)), "component", "presence", "error", err) } } @@ -283,7 +284,7 @@ func (c *Coalescer) Flush(ctx context.Context) { heard = append(heard, ts) } if err := c.Store.TouchPackets(ctx, hashes, heard); err != nil { - log.Printf("presence: flush packets failed (%d rows dropped): %v", len(hashes), err) + slog.Error(fmt.Sprintf("presence: flush packets failed (%d rows dropped)", len(hashes)), "component", "presence", "error", err) } } } diff --git a/internal/ws/handler.go b/internal/ws/handler.go index c21826e4..1e4772e0 100644 --- a/internal/ws/handler.go +++ b/internal/ws/handler.go @@ -31,7 +31,8 @@ package ws import ( "context" "encoding/json" - "log" + "fmt" + "log/slog" "net" "net/http" "strconv" @@ -59,14 +60,14 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc ip = host } if !limiter.acquire(ip) { - log.Printf("ws: connection limit reached for IP %s", ip) + slog.Warn(fmt.Sprintf("ws: connection limit reached for IP %s", ip), "component", "ws") http.Error(w, "too many connections from this IP", http.StatusTooManyRequests) return } defer limiter.release(ip) conn, err := websocket.Accept(w, r, nil) if err != nil { - log.Printf("ws: failed to accept connection: %v", err) + slog.Warn("ws: failed to accept connection", "component", "ws", "error", err) return } @@ -85,10 +86,10 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc "connectionId": connID, } helloBytes, _ := json.Marshal(hello) - log.Printf("ws[%s]: connected, hello: %s", connID, helloBytes) + slog.Debug("connected", "component", "ws", "connection_id", connID) err = conn.Write(ctx, websocket.MessageText, helloBytes) if err != nil { - log.Printf("ws[%s]: failed to send hello: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to send hello", connID), "component", "ws", "error", err) return } @@ -109,7 +110,7 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc msgBytes, _ := json.Marshal(msg) err = conn.Write(ctx, websocket.MessageText, msgBytes) if err != nil { - log.Printf("ws[%s]: failed to write hub event: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to write hub event", connID), "component", "ws", "error", err) cancel() return } @@ -126,7 +127,7 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc } lagBytes, _ := json.Marshal(lagged) if err := conn.Write(ctx, websocket.MessageText, lagBytes); err != nil { - log.Printf("ws[%s]: failed to write lagged notice: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to write lagged notice", connID), "component", "ws", "error", err) cancel() return } @@ -144,7 +145,7 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc _, msgBytes, err := conn.Read(readCtx) readCancel() if err != nil { - log.Printf("ws[%s]: read error: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: read error", connID), "component", "ws", "error", err) return } handleClientMessage(ctx, client, reader, h, conn, connID, msgBytes) @@ -182,7 +183,7 @@ type subscribeScope struct { func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Reader, h *hub.Hub, conn *websocket.Conn, connID string, raw []byte) { var msg clientMessage if err := json.Unmarshal(raw, &msg); err != nil { - log.Printf("ws[%s]: bad message: %v", connID, err) + slog.Warn("bad message", "component", "ws", "connection_id", connID, "error", err) return } @@ -195,12 +196,12 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea for _, ridStr := range msg.Scope.RegionIDs { rid, err := strconv.ParseInt(ridStr, 10, 32) if err != nil { - log.Printf("ws[%s]: invalid regionId %q, skipping", connID, ridStr) + slog.Warn(fmt.Sprintf("ws[%s]: invalid regionId %q, skipping", connID, ridStr), "component", "ws") continue } region, err := reader.GetRegion(ctx, int32(rid)) if err != nil { - log.Printf("ws[%s]: region %d not found, skipping: %v", connID, rid, err) + slog.Warn(fmt.Sprintf("ws[%s]: region %d not found, skipping", connID, rid), "component", "ws", "error", err) continue } iatas = append(iatas, region.IATAs...) @@ -208,7 +209,7 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea for _, slug := range msg.Scope.RegionSlugs { region, err := reader.GetRegionBySlug(ctx, slug) if err != nil { - log.Printf("ws[%s]: region slug %q not found, skipping: %v", connID, slug, err) + slog.Warn(fmt.Sprintf("ws[%s]: region slug %q not found, skipping", connID, slug), "component", "ws", "error", err) continue } iatas = append(iatas, region.IATAs...) @@ -224,10 +225,12 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea reply, _ := json.Marshal(map[string]any{ "v": 1, "type": "subscribed", "id": msg.ID, "subscriptionId": subID, }) - log.Printf("ws[%s]: subscribed %s → %s", connID, msg.ID, subID) + if slog.Default().Enabled(context.Background(), slog.LevelDebug) { + slog.Debug(fmt.Sprintf("ws[%s]: subscribed %s → %s", connID, msg.ID, subID), "component", "ws") + } err := conn.Write(ctx, websocket.MessageText, reply) if err != nil { - log.Printf("ws[%s]: failed to send subscribed reply: %v", connID, err) + slog.Warn("failed to send subscribed reply", "component", "ws", "connection_id", connID, "error", err) } case "unsubscribe": @@ -238,9 +241,11 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea reply, _ := json.Marshal(map[string]any{ "v": 1, "type": "unsubscribed", "id": msg.ID, "subscriptionId": msg.SubscriptionID, }) - log.Printf("ws[%s]: unsubscribed %s", connID, msg.SubscriptionID) + if slog.Default().Enabled(context.Background(), slog.LevelDebug) { + slog.Debug(fmt.Sprintf("ws[%s]: unsubscribed %s", connID, msg.SubscriptionID), "component", "ws") + } if err := conn.Write(ctx, websocket.MessageText, reply); err != nil { - log.Printf("ws[%s]: failed to send unsubscribed reply: %v", connID, err) + slog.Warn("failed to send unsubscribed reply", "component", "ws", "connection_id", connID, "error", err) } case "configure": @@ -248,19 +253,21 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea reply, _ := json.Marshal(map[string]any{ "v": 1, "type": "configured", "id": msg.ID, "resolvePath": msg.ResolvePath, }) - log.Printf("ws[%s]: configured resolvePath=%t", connID, msg.ResolvePath) + if slog.Default().Enabled(context.Background(), slog.LevelDebug) { + slog.Debug(fmt.Sprintf("ws[%s]: configured resolvePath=%t", connID, msg.ResolvePath), "component", "ws") + } if err := conn.Write(ctx, websocket.MessageText, reply); err != nil { - log.Printf("ws[%s]: failed to send configured reply: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to send configured reply", connID), "component", "ws", "error", err) } case "ping": reply, _ := json.Marshal(map[string]any{"v": 1, "type": "pong", "id": msg.ID}) err := conn.Write(ctx, websocket.MessageText, reply) if err != nil { - log.Printf("ws[%s]: failed to send pong: %v", connID, err) + slog.Warn(fmt.Sprintf("ws[%s]: failed to send pong", connID), "component", "ws", "error", err) } default: - log.Printf("ws[%s]: unknown message type %q", connID, msg.Type) + slog.Warn("unknown message type", "component", "ws", "connection_id", connID) } } From 6540b9253d375e3d5adcc33cbaeb175662d2c757 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 10:36:30 -0400 Subject: [PATCH 039/141] feat(admin): expose selected startup configuration (#147) --- README.md | 10 +- cmd/beacon/main.go | 5 + docs/docs.go | 111 +++++++++++++++++++++++ docs/swagger.json | 111 +++++++++++++++++++++++ docs/swagger.yaml | 76 ++++++++++++++++ internal/api/admin.go | 32 +++++++ internal/api/handlers/admin.go | 36 ++++++++ internal/api/router/admin_config_test.go | 86 ++++++++++++++++++ internal/api/router/auth_test.go | 6 ++ internal/api/router/router.go | 18 +++- 10 files changed, 487 insertions(+), 4 deletions(-) create mode 100644 internal/api/admin.go create mode 100644 internal/api/handlers/admin.go create mode 100644 internal/api/router/admin_config_test.go diff --git a/README.md b/README.md index 0c6e26b1..1251cb09 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,15 @@ With no key, admin requests return JSON 503 while public reads and WebSockets continue normally. With a key, missing, incorrect or duplicate Authorization headers return JSON 401 with `WWW-Authenticate: Bearer`. -Admin operations are not implemented yet: a valid key currently reaches a 404. +`GET /api/v1/admin/config` returns selected startup settings: CORS options with +Beacon defaults applied, `auth.configured`, and `ingest.broker_count` (configured +broker workers, not connection status or a tunable processing-worker pool). +The CORS lists are the options supplied to the middleware; its normal matching +normalization still applies. The response is a startup snapshot and excludes +credential fields, broker addresses, channel material, database settings and +other configuration. Changes require a restart. Configuration writes and account +operations are not implemented; unknown admin paths return 404 and unsupported +methods on the config endpoint return 405 after authentication. Global CORS preflights remain public. Use a long, randomly generated key, keep it out of source control and logs, and send it only in the Authorization header, never the URL or request body. Require HTTPS at the reverse proxy and restrict diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index da445eb0..380f8ac4 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -55,6 +55,11 @@ var version = "dev" // @schemes http https +// @securityDefinitions.apikey AdminKey +// @in header +// @name Authorization +// @description Enter Bearer followed by the configured operator key. Use HTTPS. + // @tag.name IATAs // @tag.description Airport/location codes that group observers and packets // @tag.name Regions diff --git a/docs/docs.go b/docs/docs.go index 451cd45e..68de0002 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -22,6 +22,49 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { + "/admin/config": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns CORS startup options with Beacon defaults, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Changes require restart; this endpoint is read-only.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Inspect selected startup configuration", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/brokers": { "get": { "produces": [ @@ -2423,6 +2466,66 @@ const docTemplate = `{ } }, "definitions": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig": { + "type": "object", + "properties": { + "configured": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig": { + "type": "object", + "properties": { + "allow_credentials": { + "type": "boolean" + }, + "allowed_headers": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowed_methods": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowed_origins": { + "type": "array", + "items": { + "type": "string" + } + }, + "max_age": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig": { + "type": "object", + "properties": { + "auth": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig" + }, + "cors": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig" + }, + "ingest": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig": { + "type": "object", + "properties": { + "broker_count": { + "description": "BrokerCount counts configured broker workers, not active MQTT connections\nor a configurable processing-worker pool.", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.AdvertObservation": { "type": "object", "properties": { @@ -4102,6 +4205,14 @@ const docTemplate = `{ } } }, + "securityDefinitions": { + "AdminKey": { + "description": "Enter Bearer followed by the configured operator key. Use HTTPS.", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + }, "tags": [ { "description": "Airport/location codes that group observers and packets", diff --git a/docs/swagger.json b/docs/swagger.json index 90acfb29..457c4cb7 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -20,6 +20,49 @@ "host": "localhost:8080", "basePath": "/api/v1", "paths": { + "/admin/config": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns CORS startup options with Beacon defaults, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Changes require restart; this endpoint is read-only.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Inspect selected startup configuration", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/brokers": { "get": { "produces": [ @@ -2421,6 +2464,66 @@ } }, "definitions": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig": { + "type": "object", + "properties": { + "configured": { + "type": "boolean" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig": { + "type": "object", + "properties": { + "allow_credentials": { + "type": "boolean" + }, + "allowed_headers": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowed_methods": { + "type": "array", + "items": { + "type": "string" + } + }, + "allowed_origins": { + "type": "array", + "items": { + "type": "string" + } + }, + "max_age": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig": { + "type": "object", + "properties": { + "auth": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig" + }, + "cors": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig" + }, + "ingest": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig": { + "type": "object", + "properties": { + "broker_count": { + "description": "BrokerCount counts configured broker workers, not active MQTT connections\nor a configurable processing-worker pool.", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.AdvertObservation": { "type": "object", "properties": { @@ -4100,6 +4203,14 @@ } } }, + "securityDefinitions": { + "AdminKey": { + "description": "Enter Bearer followed by the configured operator key. Use HTTPS.", + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + }, "tags": [ { "description": "Airport/location codes that group observers and packets", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index cb0b716e..416384e7 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,5 +1,46 @@ basePath: /api/v1 definitions: + github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig: + properties: + configured: + type: boolean + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig: + properties: + allow_credentials: + type: boolean + allowed_headers: + items: + type: string + type: array + allowed_methods: + items: + type: string + type: array + allowed_origins: + items: + type: string + type: array + max_age: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig: + properties: + auth: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig' + cors: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminCORSConfig' + ingest: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig' + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AdminIngestConfig: + properties: + broker_count: + description: |- + BrokerCount counts configured broker workers, not active MQTT connections + or a configurable processing-worker pool. + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.AdvertObservation: properties: heardAt: @@ -1191,6 +1232,35 @@ info: title: MeshCore Beacon API version: 1.6.0 paths: + /admin/config: + get: + description: Returns CORS startup options with Beacon defaults, auth configuration + status and configured broker count. Credential fields and other configuration + are excluded. Changes require restart; this endpoint is read-only. + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig' + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Inspect selected startup configuration + tags: + - Admin /brokers: get: produces: @@ -2808,6 +2878,12 @@ paths: schemes: - http - https +securityDefinitions: + AdminKey: + description: Enter Bearer followed by the configured operator key. Use HTTPS. + in: header + name: Authorization + type: apiKey swagger: "2.0" tags: - description: Airport/location codes that group observers and packets diff --git a/internal/api/admin.go b/internal/api/admin.go new file mode 100644 index 00000000..70cfc3a7 --- /dev/null +++ b/internal/api/admin.go @@ -0,0 +1,32 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +// AdminConfig is a whitelist of nonsecret settings captured when the router starts. +// It is not a serialization of the full file or environment configuration. +type AdminConfig struct { + Auth AdminAuthConfig `json:"auth"` + CORS AdminCORSConfig `json:"cors"` + Ingest AdminIngestConfig `json:"ingest"` +} + +type AdminAuthConfig struct { + Configured bool `json:"configured"` +} + +// AdminCORSConfig reports the startup options supplied to the CORS middleware, +// with Beacon defaults applied, before that library's matching normalization. +type AdminCORSConfig struct { + AllowedOrigins []string `json:"allowed_origins"` + AllowedMethods []string `json:"allowed_methods"` + AllowedHeaders []string `json:"allowed_headers"` + AllowCredentials bool `json:"allow_credentials"` + MaxAge int `json:"max_age"` +} + +type AdminIngestConfig struct { + // BrokerCount counts configured broker workers, not active MQTT connections + // or a configurable processing-worker pool. + BrokerCount int `json:"broker_count"` +} diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go new file mode 100644 index 00000000..4ec31d33 --- /dev/null +++ b/internal/api/handlers/admin.go @@ -0,0 +1,36 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "net/http" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" +) + +// AdminRouter mounts read-only operator endpoints. Its caller must wrap the +// entire subrouter with BearerAuth, including unknown paths and methods. +func AdminRouter(snapshot api.AdminConfig) http.Handler { + r := chi.NewRouter() + r.Get("/config", getAdminConfig(snapshot)) + return r +} + +// getAdminConfig godoc +// +// @Summary Inspect selected startup configuration +// @Description Returns CORS startup options with Beacon defaults, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Changes require restart; this endpoint is read-only. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Success 200 {object} api.AdminConfig +// @Failure 401 {object} map[string]APIError +// @Failure 503 {object} map[string]APIError +// @Router /admin/config [get] +func getAdminConfig(snapshot api.AdminConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + respond(w, http.StatusOK, snapshot) + } +} diff --git a/internal/api/router/admin_config_test.go b/internal/api/router/admin_config_test.go new file mode 100644 index 00000000..544c8a61 --- /dev/null +++ b/internal/api/router/admin_config_test.go @@ -0,0 +1,86 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" +) + +func TestAdminConfig(t *testing.T) { + const key = "admin-key-sentinel" + for _, variable := range []string{"POSTGRES_DSN", "MQTT_BROKER_1_URL", "MQTT_BROKER_1_USERNAME", "MQTT_BROKER_1_PASSWORD", "REDIS_PASSWORD"} { + t.Setenv(variable, "environment-secret-sentinel") + } + for _, custom := range []bool{false, true} { + cfg := config.CORSConfig{} + want := map[string]any{ + "allowed_origins": []any{"*"}, "allowed_methods": []any{"GET", "HEAD", "OPTIONS"}, + "allowed_headers": []any{"Accept", "Authorization", "Content-Type"}, "allow_credentials": false, "max_age": float64(300), + } + if custom { + cfg = config.CORSConfig{AllowedOrigins: []string{"https://example.test"}, AllowedMethods: []string{"GET"}, + AllowedHeaders: []string{"Authorization", "X-Operator"}, AllowCredentials: true, MaxAge: 600} + want = map[string]any{"allowed_origins": []any{"https://example.test"}, "allowed_methods": []any{"GET"}, + "allowed_headers": []any{"Authorization", "X-Operator"}, "allow_credentials": true, "max_age": float64(600)} + } + handler := New(nil, nil, []*ingest.Worker{{}, {}}, 5, cfg, config.ServerConfig{}, config.AuthConfig{APIKey: key}, config.ResolvedRateLimitConfig{}) + if custom { + cfg.AllowedOrigins[0] = "https://changed.test" + cfg.AllowedMethods[0] = "DELETE" + cfg.AllowedHeaders[0] = "X-Changed" + } + request := func(method, path string, headers map[string]string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, nil) + for name, value := range headers { + req.Header.Set(name, value) + } + response := httptest.NewRecorder() + handler.ServeHTTP(response, req) + return response + } + headers := map[string]string{"Authorization": "Bearer " + key} + response := request("GET", "/api/v1/admin/config", headers) + if response.Code != 200 || response.Header().Get("Content-Type") != "application/json" || response.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("config read status/headers: %d %v", response.Code, response.Header()) + } + var body map[string]any + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + expected := map[string]any{"cors": want, "auth": map[string]any{"configured": true}, "ingest": map[string]any{"broker_count": float64(2)}} + if !reflect.DeepEqual(body, expected) { + t.Fatalf("unexpected config shape or values: %v", body) + } + if strings.Contains(response.Body.String(), "sentinel") { + t.Fatal("credential setting was exposed") + } + for _, auth := range []string{"", "Bearer wrong"} { + denied := request("GET", "/api/v1/admin/config", map[string]string{"Authorization": auth}) + if denied.Code != 401 || strings.Contains(denied.Body.String(), "allowed_origins") { + t.Fatal("config escaped authentication") + } + } + if request("PUT", "/api/v1/admin/config", headers).Code != http.StatusMethodNotAllowed { + t.Fatal("config writes were accepted") + } + again := request("GET", "/api/v1/admin/config", headers) + if again.Body.String() != response.Body.String() { + t.Fatal("startup snapshot changed") + } + preflight := request("OPTIONS", "/api/v1/admin/config", map[string]string{ + "Origin": "https://example.test", "Access-Control-Request-Method": "GET", "Access-Control-Request-Headers": "Authorization", + }) + if preflight.Code != 200 || preflight.Header().Get("Access-Control-Allow-Origin") == "" || preflight.Header().Get("Access-Control-Max-Age") != map[bool]string{false: "300", true: "600"}[custom] { + t.Fatal("CORS behavior does not match reported options") + } + } +} diff --git a/internal/api/router/auth_test.go b/internal/api/router/auth_test.go index 25cef5ed..57bc5ac7 100644 --- a/internal/api/router/auth_test.go +++ b/internal/api/router/auth_test.go @@ -28,6 +28,12 @@ func TestAdminAuthBoundary(t *testing.T) { want = 401 if token == key { want = 404 + if path == "/api/v1/admin/config" || path == "/api/v1/admin//config" { + want = 405 + if method == "GET" { + want = 200 + } + } } } if w.Code != want { diff --git a/internal/api/router/router.go b/internal/api/router/router.go index 7fe35ba4..2e91ab81 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -38,7 +38,7 @@ import ( // /regions → regions subrouter // /stats → stats subrouter // -// Admin handlers are added separately; the reserved subtree is protected now. +// Admin endpoints require a configured bearer key. func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP int, corsCfg config.CORSConfig, serverCfg config.ServerConfig, authCfg config.AuthConfig, rateLimitCfg config.ResolvedRateLimitConfig) http.Handler { r := chi.NewRouter() @@ -67,6 +67,19 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP AllowCredentials: corsCfg.AllowCredentials, MaxAge: maxAge, })) + // Capture only values used by this router. Do not retain Config, credentials + // or caller-owned slices in the admin response. + adminConfig := api.AdminConfig{ + Auth: api.AdminAuthConfig{Configured: authCfg.APIKey != ""}, + CORS: api.AdminCORSConfig{ + AllowedOrigins: append([]string{}, allowedOrigins...), + AllowedMethods: append([]string{}, allowedMethods...), + AllowedHeaders: append([]string{}, allowedHeaders...), + AllowCredentials: corsCfg.AllowCredentials, + MaxAge: maxAge, + }, + Ingest: api.AdminIngestConfig{BrokerCount: len(workers)}, + } // ── Global middleware ──────────────────────────────────────────────────── r.Use(middleware.RequestID) @@ -108,8 +121,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP }) // Protect the entire subtree, including its root and unknown paths. - // Replace the empty router with the admin handlers when they are added. - r.Mount("/admin", mw.BearerAuth(authCfg.APIKey, chi.NewRouter())) + r.Mount("/admin", mw.BearerAuth(authCfg.APIKey, handlers.AdminRouter(adminConfig))) }) return r From cf5ff7a974ad5a815c1d9d2664613c480f873d71 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 10:36:39 -0400 Subject: [PATCH 040/141] feat(stats): compare flood packets reported by two observers (#152) * feat(stats): compare distinct flood packets across observers * test(stats): use the migrated observation deduplication key --- db/observer_comparison.go | 34 +++++ db/observer_comparison_integration_test.go | 109 ++++++++++++++++ db/queries/observer_comparison.sql | 23 ++++ db/sqlc/mock/querier.go | 15 +++ db/sqlc/observer_comparison.sql.go | 75 +++++++++++ db/sqlc/querier.go | 3 + docs/docs.go | 123 ++++++++++++++++++ docs/swagger.json | 123 ++++++++++++++++++ docs/swagger.yaml | 86 ++++++++++++ internal/api/handlers/observer_comparison.go | 92 +++++++++++++ .../api/handlers/observer_comparison_test.go | 98 ++++++++++++++ internal/api/handlers/stats.go | 1 + internal/api/handlers/stub_reader_test.go | 8 ++ internal/api/reader.go | 4 + internal/api/stats.go | 14 ++ internal/cache/cache_test.go | 17 +++ internal/cache/reader.go | 6 + 17 files changed, 831 insertions(+) create mode 100644 db/observer_comparison.go create mode 100644 db/observer_comparison_integration_test.go create mode 100644 db/queries/observer_comparison.sql create mode 100644 db/sqlc/observer_comparison.sql.go create mode 100644 internal/api/handlers/observer_comparison.go create mode 100644 internal/api/handlers/observer_comparison_test.go diff --git a/db/observer_comparison.go b/db/observer_comparison.go new file mode 100644 index 00000000..5186589f --- /dev/null +++ b/db/observer_comparison.go @@ -0,0 +1,34 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetObserverComparison(ctx context.Context, a, b uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { + row, err := s.q.GetObserverComparison(ctx, sqlc.GetObserverComparisonParams{ + ObserverA: a, ObserverB: b, + Since: pgtype.Timestamptz{Time: since, Valid: true}, + Until: pgtype.Timestamptz{Time: until, Valid: true}, + Iatas: iatas, + }) + if err != nil { + return nil, err + } + if !row.KnownA || !row.KnownB { + return nil, pgx.ErrNoRows + } + return &api.ObserverComparison{ + ObserverA: a, ObserverB: b, Since: since.UnixMilli(), Until: until.UnixMilli(), + TotalPackets: row.TotalPackets, OnlyA: row.OnlyA, OnlyB: row.OnlyB, Both: row.Both, + }, nil +} diff --git a/db/observer_comparison_integration_test.go b/db/observer_comparison_integration_test.go new file mode 100644 index 00000000..103f029b --- /dev/null +++ b/db/observer_comparison_integration_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestObserverComparisonPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + // Temporary tables shadow the private database's tables; no retained rows + // or sequences are changed, and the fixture is rolled back on every exit. + for _, table := range []string{"observers", "packets", "packet_observations"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO observers (id, public_key) +SELECT ('00000000-0000-0000-0000-' || lpad(i::text,12,'0'))::uuid, + decode(lpad(to_hex(i),64,'0'),'hex') FROM generate_series(1,3) i; +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),4,0,route,'\x00','\x00',NOW(),NOW() +FROM (VALUES (1,1),(2,0),(3,1),(4,1),(5,0),(6,1),(7,2),(8,3),(9,1),(10,1)) v(i,route); +INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,source_broker) +SELECT id,decode(lpad(to_hex(packet),2,'0'),'hex'), + ('00000000-0000-0000-0000-' || lpad(observer::text,12,'0'))::uuid, + iata,'2026-01-01'::timestamptz+second*interval '1 second',0,1,0,broker +FROM (VALUES + (1,1,1,'YVR',1,'a'),(2,1,1,'YVR',2,'b'), + (3,2,2,'YVR',2,'a'),(4,3,1,'YVR',3,'a'),(5,3,2,'YYJ',4,'a'), + (6,4,1,'YVR',-1,'a'),(7,4,2,'YVR',5,'a'),(8,5,1,'YVR',0,'a'), + (9,6,2,'YVR',10,'a'),(10,7,1,'YVR',1,'a'),(11,7,2,'YVR',2,'a'), + (12,8,2,'YVR',1,'a'),(13,9,3,'YVR',1,'a'), + (14,10,1,'YVR',3,'a'),(15,10,2,'YVR',9,'a')) v(id,packet,observer,iata,second,broker) +ORDER BY id +ON CONFLICT (packet_hash,observer_id) DO NOTHING; +`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + a := uuid.MustParse("00000000-0000-0000-0000-000000000001") + b := uuid.MustParse("00000000-0000-0000-0000-000000000002") + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + iatas []string + from, to int + reverse bool + counts [4]int64 // union, only A, only B, both + }{ + {"global distinct floods", nil, 0, 10, false, [4]int64{6, 2, 2, 2}}, + {"empty filter", []string{}, 0, 10, false, [4]int64{6, 2, 2, 2}}, + {"YVR receptions", []string{"YVR"}, 0, 10, false, [4]int64{6, 3, 2, 1}}, + {"reversed observers", []string{"YVR"}, 0, 10, true, [4]int64{6, 2, 3, 1}}, + {"YYJ receptions", []string{"YYJ"}, 0, 10, false, [4]int64{1, 0, 1, 0}}, + {"region union", []string{"YYJ", "YVR", "YYJ"}, 0, 10, false, [4]int64{6, 2, 2, 2}}, + {"unknown IATA", []string{"ZZZ"}, 0, 10, false, [4]int64{}}, + {"inclusive start exclusive end", nil, 0, 1, false, [4]int64{1, 1, 0, 0}}, + {"next interval", nil, 1, 3, false, [4]int64{2, 1, 1, 0}}, + {"known observers empty interval", nil, 20, 30, false, [4]int64{}}, + } { + t.Run(tc.name, func(t *testing.T) { + first, second := a, b + if tc.reverse { + first, second = b, a + } + since, until := start.Add(time.Duration(tc.from)*time.Second), start.Add(time.Duration(tc.to)*time.Second) + got, err := store.GetObserverComparison(ctx, first, second, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + if got.ObserverA != first || got.ObserverB != second || got.Since != since.UnixMilli() || got.Until != until.UnixMilli() { + t.Fatalf("comparison context = %+v", got) + } + if counts := [4]int64{got.TotalPackets, got.OnlyA, got.OnlyB, got.Both}; counts != tc.counts { + t.Fatalf("counts = %v; want %v", counts, tc.counts) + } + }) + } + if _, err := store.GetObserverComparison(ctx, a, uuid.New(), start, start.Add(time.Hour), nil); !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("unknown observer: %v", err) + } +} diff --git a/db/queries/observer_comparison.sql b/db/queries/observer_comparison.sql new file mode 100644 index 00000000..dab05da1 --- /dev/null +++ b/db/queries/observer_comparison.sql @@ -0,0 +1,23 @@ +-- name: GetObserverComparison :one +-- Count packet identities, not reception rows: a broker repeat or another +-- reception by the same observer must not inflate the comparison. +WITH heard AS ( + SELECT po.packet_hash, + bool_or(po.observer_id = @observer_a::uuid) AS seen_a, + bool_or(po.observer_id = @observer_b::uuid) AS seen_b + FROM packet_observations po + JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.observer_id IN (@observer_a::uuid, @observer_b::uuid) + AND po.heard_at >= @since::timestamptz + AND po.heard_at < @until::timestamptz + AND p.route_type IN (0, 1) + AND (COALESCE(cardinality(@iatas::text[]), 0) = 0 OR po.iata = ANY(@iatas::text[])) + GROUP BY po.packet_hash +) +SELECT count(*) AS total_packets, + count(*) FILTER (WHERE seen_a AND NOT seen_b) AS only_a, + count(*) FILTER (WHERE seen_b AND NOT seen_a) AS only_b, + count(*) FILTER (WHERE seen_a AND seen_b) AS both, + EXISTS (SELECT 1 FROM observers WHERE id = @observer_a::uuid) AS known_a, + EXISTS (SELECT 1 FROM observers WHERE id = @observer_b::uuid) AS known_b +FROM heard; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index d11eafc5..bb46595c 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -442,6 +442,21 @@ func (mr *MockQuerierMockRecorder) GetObserverByPubkey(ctx, publicKey any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverByPubkey", reflect.TypeOf((*MockQuerier)(nil).GetObserverByPubkey), ctx, publicKey) } +// GetObserverComparison mocks base method. +func (m *MockQuerier) GetObserverComparison(ctx context.Context, arg db.GetObserverComparisonParams) (db.GetObserverComparisonRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverComparison", ctx, arg) + ret0, _ := ret[0].(db.GetObserverComparisonRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverComparison indicates an expected call of GetObserverComparison. +func (mr *MockQuerierMockRecorder) GetObserverComparison(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverComparison", reflect.TypeOf((*MockQuerier)(nil).GetObserverComparison), ctx, arg) +} + // GetObserverLastIATA mocks base method. func (m *MockQuerier) GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/observer_comparison.sql.go b/db/sqlc/observer_comparison.sql.go new file mode 100644 index 00000000..00c0769e --- /dev/null +++ b/db/sqlc/observer_comparison.sql.go @@ -0,0 +1,75 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: observer_comparison.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const getObserverComparison = `-- name: GetObserverComparison :one +WITH heard AS ( + SELECT po.packet_hash, + bool_or(po.observer_id = $1::uuid) AS seen_a, + bool_or(po.observer_id = $2::uuid) AS seen_b + FROM packet_observations po + JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.observer_id IN ($1::uuid, $2::uuid) + AND po.heard_at >= $3::timestamptz + AND po.heard_at < $4::timestamptz + AND p.route_type IN (0, 1) + AND (COALESCE(cardinality($5::text[]), 0) = 0 OR po.iata = ANY($5::text[])) + GROUP BY po.packet_hash +) +SELECT count(*) AS total_packets, + count(*) FILTER (WHERE seen_a AND NOT seen_b) AS only_a, + count(*) FILTER (WHERE seen_b AND NOT seen_a) AS only_b, + count(*) FILTER (WHERE seen_a AND seen_b) AS both, + EXISTS (SELECT 1 FROM observers WHERE id = $1::uuid) AS known_a, + EXISTS (SELECT 1 FROM observers WHERE id = $2::uuid) AS known_b +FROM heard +` + +type GetObserverComparisonParams struct { + ObserverA uuid.UUID `json:"observer_a"` + ObserverB uuid.UUID `json:"observer_b"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetObserverComparisonRow struct { + TotalPackets int64 `json:"total_packets"` + OnlyA int64 `json:"only_a"` + OnlyB int64 `json:"only_b"` + Both int64 `json:"both"` + KnownA bool `json:"known_a"` + KnownB bool `json:"known_b"` +} + +// Count packet identities, not reception rows: a broker repeat or another +// reception by the same observer must not inflate the comparison. +func (q *Queries) GetObserverComparison(ctx context.Context, arg GetObserverComparisonParams) (GetObserverComparisonRow, error) { + row := q.db.QueryRow(ctx, getObserverComparison, + arg.ObserverA, + arg.ObserverB, + arg.Since, + arg.Until, + arg.Iatas, + ) + var i GetObserverComparisonRow + err := row.Scan( + &i.TotalPackets, + &i.OnlyA, + &i.OnlyB, + &i.Both, + &i.KnownA, + &i.KnownB, + ) + return i, err +} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index a6e119cc..7496e554 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -66,6 +66,9 @@ type Querier interface { GetObserverBrokers(ctx context.Context, observerID uuid.UUID) ([]GetObserverBrokersRow, error) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, error) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Observer, error) + // Count packet identities, not reception rows: a broker repeat or another + // reception by the same observer must not inflate the comparison. + GetObserverComparison(ctx context.Context, arg GetObserverComparisonParams) (GetObserverComparisonRow, error) GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) GetObserverRadio(ctx context.Context, id uuid.UUID) (GetObserverRadioRow, error) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) diff --git a/docs/docs.go b/docs/docs.go index 68de0002..7e443c37 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1876,6 +1876,98 @@ const docTemplate = `{ } } }, + "/stats/observer-comparison": { + "get": { + "description": "Counts distinct retained packet hashes with route type 0 or 1, using reception time in [since, until). onlyA, onlyB and both partition totalPackets (the union). Repeated receptions count once. These are reported receptions, not a measure of radio packet loss or continuous observer coverage.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Compare flood packets reported by two observers", + "parameters": [ + { + "type": "string", + "description": "First observer UUID", + "name": "observerA", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Second, distinct observer UUID", + "name": "observerB", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; later than since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/overview": { "get": { "produces": [ @@ -3285,6 +3377,37 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison": { + "type": "object", + "properties": { + "both": { + "type": "integer" + }, + "observerA": { + "type": "string" + }, + "observerB": { + "type": "string" + }, + "onlyA": { + "type": "integer" + }, + "onlyB": { + "type": "integer" + }, + "since": { + "description": "inclusive, epoch milliseconds", + "type": "integer" + }, + "totalPackets": { + "type": "integer" + }, + "until": { + "description": "exclusive, epoch milliseconds", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 457c4cb7..aa2eba99 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1874,6 +1874,98 @@ } } }, + "/stats/observer-comparison": { + "get": { + "description": "Counts distinct retained packet hashes with route type 0 or 1, using reception time in [since, until). onlyA, onlyB and both partition totalPackets (the union). Repeated receptions count once. These are reported receptions, not a measure of radio packet loss or continuous observer coverage.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Compare flood packets reported by two observers", + "parameters": [ + { + "type": "string", + "description": "First observer UUID", + "name": "observerA", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Second, distinct observer UUID", + "name": "observerB", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; later than since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/overview": { "get": { "produces": [ @@ -3283,6 +3375,37 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison": { + "type": "object", + "properties": { + "both": { + "type": "integer" + }, + "observerA": { + "type": "string" + }, + "observerB": { + "type": "string" + }, + "onlyA": { + "type": "integer" + }, + "onlyB": { + "type": "integer" + }, + "since": { + "description": "inclusive, epoch milliseconds", + "type": "integer" + }, + "totalPackets": { + "type": "integer" + }, + "until": { + "description": "exclusive, epoch milliseconds", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 416384e7..d10fb48b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -579,6 +579,27 @@ definitions: description: broker name e.g. "mqtt1" type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison: + properties: + both: + type: integer + observerA: + type: string + observerB: + type: string + onlyA: + type: integer + onlyB: + type: integer + since: + description: inclusive, epoch milliseconds + type: integer + totalPackets: + type: integer + until: + description: exclusive, epoch milliseconds + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverSummary: properties: displayName: @@ -2479,6 +2500,71 @@ paths: summary: Hourly observation time series tags: - Stats + /stats/observer-comparison: + get: + description: Counts distinct retained packet hashes with route type 0 or 1, + using reception time in [since, until). onlyA, onlyB and both partition totalPackets + (the union). Repeated receptions count once. These are reported receptions, + not a measure of radio packet loss or continuous observer coverage. + parameters: + - description: First observer UUID + in: query + name: observerA + required: true + type: string + - description: Second, distinct observer UUID + in: query + name: observerB + required: true + type: string + - description: Inclusive start, epoch milliseconds (0 through 253402300799999) + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; later than since + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverComparison' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Compare flood packets reported by two observers + tags: + - Stats /stats/overview: get: parameters: diff --git a/internal/api/handlers/observer_comparison.go b/internal/api/handlers/observer_comparison.go new file mode 100644 index 00000000..6988d50a --- /dev/null +++ b/internal/api/handlers/observer_comparison.go @@ -0,0 +1,92 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strconv" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// getObserverComparison godoc +// +// @Summary Compare flood packets reported by two observers +// @Description Counts distinct retained packet hashes with route type 0 or 1, using reception time in [since, until). onlyA, onlyB and both partition totalPackets (the union). Repeated receptions count once. These are reported receptions, not a measure of radio packet loss or continuous observer coverage. +// @Tags Stats +// @Produce json +// @Param observerA query string true "First observer UUID" +// @Param observerB query string true "Second, distinct observer UUID" +// @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" +// @Param until query int true "Exclusive end, epoch milliseconds; later than since" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.ObserverComparison +// @Failure 400 {object} handlers.APIError +// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/observer-comparison [get] +func getObserverComparison(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + for _, key := range []string{"observerA", "observerB", "since", "until"} { + if len(q[key]) != 1 || q.Get(key) == "" { + respondError(w, http.StatusBadRequest, key+" must be supplied exactly once") + return + } + } + a, errA := uuid.Parse(q.Get("observerA")) + b, errB := uuid.Parse(q.Get("observerB")) + if errA != nil || errB != nil || a == uuid.Nil || b == uuid.Nil || a == b { + respondError(w, http.StatusBadRequest, "observerA and observerB must be distinct, nonzero UUIDs") + return + } + since, errSince := strconv.ParseInt(q.Get("since"), 10, 64) + until, errUntil := strconv.ParseInt(q.Get("until"), 10, 64) + if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 { + respondError(w, http.StatusBadRequest, "since and until must be epoch milliseconds between 0 and 253402300799999, with until later than since") + return + } + // Bound database work even for arbitrary historical windows. Cancellation + // also propagates when a user changes filters or leaves the page. + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondError(w, http.StatusBadRequest, "region not found") + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + // An empty selected region must not silently become a global query. + iatas = []string{""} + } + } + comparison, err := reader.GetObserverComparison(ctx, a, b, time.UnixMilli(since), time.UnixMilli(until), iatas) + switch { + case errors.Is(err, pgx.ErrNoRows): + respondError(w, http.StatusNotFound, "observer not found") + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "comparison timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("observer comparison failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, comparison) + } + } +} diff --git a/internal/api/handlers/observer_comparison_test.go b/internal/api/handlers/observer_comparison_test.go new file mode 100644 index 00000000..3a46da7e --- /dev/null +++ b/internal/api/handlers/observer_comparison_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestObserverComparisonRejectsInvalidQuery(t *testing.T) { + for _, query := range []string{ + "", + "observerA=bad&observerB=bad&since=0&until=1", + "observerA=11111111-1111-1111-1111-111111111111&observerB=11111111-1111-1111-1111-111111111111&since=0&until=1", + "observerA=11111111-1111-1111-1111-111111111111&observerB=22222222-2222-2222-2222-222222222222&since=2&until=1", + "observerA=11111111-1111-1111-1111-111111111111&observerB=22222222-2222-2222-2222-222222222222&since=no&until=1", + "observerA=11111111-1111-1111-1111-111111111111&observerB=22222222-2222-2222-2222-222222222222&since=-1&until=1", + } { + t.Run(query, func(t *testing.T) { + w := httptest.NewRecorder() + StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) + if w.Code != 400 { + t.Fatalf("status %d; wanted validation error", w.Code) + } + }) + } +} + +func TestObserverComparisonHTTP(t *testing.T) { + a := uuid.MustParse("11111111-1111-1111-1111-111111111111") + b := uuid.MustParse("22222222-2222-2222-2222-222222222222") + query := "observerA=" + a.String() + "&observerB=" + b.String() + "&since=0&until=1000" + for _, suffix := range []string{"&since=2", "&observerA=" + b.String(), "&until=253402300800000", "&observerB=00000000-0000-0000-0000-000000000000"} { + w := httptest.NewRecorder() + StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) + if w.Code != 400 { + t.Fatalf("invalid query returned %d", w.Code) + } + } + for _, tc := range []struct { + name, suffix string + err error + status int + iatas []string + }{ + {"global", "", nil, 200, nil}, + {"IATA", "&iatas=yvr,yyj", nil, 200, []string{"YVR", "YYJ"}}, + {"region", "®ion=coast", nil, 200, []string{"YVR"}}, + {"empty region", "®ion=empty", nil, 200, []string{""}}, + {"unknown observer", "", pgx.ErrNoRows, 404, nil}, + {"timeout", "", context.DeadlineExceeded, 503, nil}, + {"database failure", "", errors.New("private database detail"), 500, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := stubReader{ + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }, + getObserverComparison: func(ctx context.Context, first, second uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { + t.Fatal("query has no bounded deadline") + } + if first != a || second != b || since.UnixMilli() != 0 || until.UnixMilli() != 1000 || !reflect.DeepEqual(iatas, tc.iatas) { + t.Fatalf("unexpected arguments: %s %s %v %v %v", first, second, since, until, iatas) + } + return &api.ObserverComparison{ObserverA: a, ObserverB: b, Until: 1000, TotalPackets: 4, OnlyA: 1, OnlyB: 1, Both: 2}, tc.err + }, + } + w := httptest.NewRecorder() + StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) + if w.Code != tc.status { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + if tc.status == 200 { + var got api.ObserverComparison + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.TotalPackets != 4 || got.ObserverA != a || got.Both != 2 { + t.Fatalf("response = %+v", got) + } + } + }) + } +} diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index 71828d82..f65bdc04 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -30,6 +30,7 @@ func StatsRouter(reader api.Reader) http.Handler { r := chi.NewRouter() r.Get("/overview", getStatsOverview(reader)) r.Get("/observations", getStatsObservations(reader)) + r.Get("/observer-comparison", getObserverComparison(reader)) r.Get("/payload-breakdown", getStatsPayloadBreakdown(reader)) r.Get("/top-nodes", getStatsTopNodes(reader)) r.Get("/top-observers", getStatsTopObservers(reader)) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index d87076e6..559200a0 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -16,6 +16,7 @@ import ( // Unset fields return zero values. Use it for both validation tests // (leave all fields nil) and happy path tests (set only what you need). type stubReader struct { + getObserverComparison func(context.Context, uuid.UUID, uuid.UUID, time.Time, time.Time, []string) (*api.ObserverComparison, error) listIATAs func(ctx context.Context) ([]api.IATA, error) getIATA func(ctx context.Context, iata string) (*api.IATA, error) getIATABorder func(ctx context.Context, iata string) (json.RawMessage, error) @@ -65,6 +66,13 @@ type stubReader struct { getNodesByIDs func(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]*api.ResolvedNode, error) } +func (s stubReader) GetObserverComparison(ctx context.Context, a, b uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { + if s.getObserverComparison != nil { + return s.getObserverComparison(ctx, a, b, since, until, iatas) + } + return nil, nil +} + func (s stubReader) ListIATAs(ctx context.Context) ([]api.IATA, error) { if s.listIATAs != nil { return s.listIATAs(ctx) diff --git a/internal/api/reader.go b/internal/api/reader.go index 75febdc7..1b90b75c 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -148,6 +148,10 @@ type Reader interface { // Pass empty string for preset or nil iatas to skip those filters. GetRadioPresets(ctx context.Context, preset string, iatas []string) ([]RadioPreset, error) + // GetObserverComparison compares distinct flood packets reported by two observers + // during [since, until), optionally restricted to reception IATAs. + GetObserverComparison(ctx context.Context, observerA, observerB uuid.UUID, since, until time.Time, iatas []string) (*ObserverComparison, error) + // GetStatsOverview returns top-line network figures for the last 24 hours. // Pass nil iatas to return stats across all IATAs. GetStatsOverview(ctx context.Context, iatas []string) (*StatsOverview, error) diff --git a/internal/api/stats.go b/internal/api/stats.go index 09e954a9..1485639d 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -5,6 +5,20 @@ package api import "github.com/google/uuid" +// ObserverComparison counts distinct flood packet hashes in [Since, Until). +// The three groups partition TotalPackets, the union heard by either observer. +// Counts describe retained reported receptions, not radio packet loss. +type ObserverComparison struct { + ObserverA uuid.UUID `json:"observerA"` + ObserverB uuid.UUID `json:"observerB"` + Since int64 `json:"since"` // inclusive, epoch milliseconds + Until int64 `json:"until"` // exclusive, epoch milliseconds + TotalPackets int64 `json:"totalPackets"` + OnlyA int64 `json:"onlyA"` + OnlyB int64 `json:"onlyB"` + Both int64 `json:"both"` +} + // RadioPreset represents a unique radio configuration observed in a given IATA, // aggregated from both observer status messages and node adverts. type RadioPreset struct { diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index cada469b..e39ccc03 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -37,6 +37,23 @@ func (s *stubReader) ListIATAs(_ context.Context) ([]api.IATA, error) { } // implement remaining api.Reader methods as no-ops +func (s *stubReader) GetObserverComparison(_ context.Context, _, _ uuid.UUID, _, _ time.Time, _ []string) (*api.ObserverComparison, error) { + s.calls++ + return &api.ObserverComparison{TotalPackets: int64(s.calls)}, s.err +} + +func TestObserverComparisonPassThrough(t *testing.T) { + inner := &stubReader{} + reader := &CachedReader{inner: inner} + a, b, since, until := uuid.New(), uuid.New(), time.Now().Add(-time.Hour), time.Now() + for i := int64(1); i <= 2; i++ { + got, err := reader.GetObserverComparison(context.Background(), a, b, since, until, nil) + if err != nil || got.TotalPackets != i { + t.Fatalf("comparison unexpectedly cached: %+v, %v", got, err) + } + } +} + func (s *stubReader) GetIATA(_ context.Context, _ string) (*api.IATA, error) { return nil, nil } func (s *stubReader) GetIATABorder(_ context.Context, _ string) (json.RawMessage, error) { diff --git a/internal/cache/reader.go b/internal/cache/reader.go index ba5ce8e2..aa4f402c 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -141,6 +141,12 @@ func (cr *CachedReader) GetScopeNames(ctx context.Context) ([]string, error) { }) } +// GetObserverComparison passes arbitrary observer/time combinations through; +// these explicit queries should not fill the shared statistics cache. +func (cr *CachedReader) GetObserverComparison(ctx context.Context, a, b uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { + return cr.inner.GetObserverComparison(ctx, a, b, since, until, iatas) +} + // GetScopeStats implements [api.Reader]. func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { segment := "all" From c7ace0ce0ec670413c73fabb7811c6d99846ed77 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 13:28:47 -0400 Subject: [PATCH 041/141] feat(ws): limit connection attempts and signal capacity shedding (#141) --- cmd/beacon/main.go | 3 +- config.yaml.example | 6 + docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 1 + internal/api/router/admin_config_test.go | 2 +- internal/api/router/auth_test.go | 2 +- internal/api/router/rate_limit_test.go | 12 +- internal/api/router/router.go | 4 +- internal/api/router/router_test.go | 30 ++-- internal/api/router/ws_attempt_test.go | 53 +++++++ internal/config/config.go | 15 +- internal/config/ws_connect_test.go | 37 +++++ internal/ws/connect_limit_test.go | 174 +++++++++++++++++++++++ internal/ws/handler.go | 21 ++- 15 files changed, 335 insertions(+), 29 deletions(-) create mode 100644 internal/api/router/ws_attempt_test.go create mode 100644 internal/config/ws_connect_test.go create mode 100644 internal/ws/connect_limit_test.go diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 380f8ac4..219b1390 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -43,6 +43,7 @@ var version = "dev" // @version 1.6.0 // @description MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket. // @description REST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget. +// @description WebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open. // @termsOfService https://github.com/MeshCore-Beacon/beacon-server // @contact.name MeshCore Beacon @@ -292,7 +293,7 @@ func main() { go scheduler.Start(ctx) // ── HTTP server ────────────────────────────────────────────────────────── - r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, cfg.CORS, cfg.Server, cfg.Auth, resolved.RateLimit) + r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, resolved.MaxConnectsPerMinute, cfg.CORS, cfg.Server, cfg.Auth, resolved.RateLimit) srv := &http.Server{ Addr: addr, diff --git a/config.yaml.example b/config.yaml.example index a805495e..9686dcdd 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -136,6 +136,12 @@ routes: websocket: max_connections_per_ip: 5 # default: 5 + # Upgrade attempts, including failed handshakes; zero/omitted defaults to 10. + # IPv6 clients share a /64 attempt budget. Negative values prevent startup. + # Rate exhaustion rejects the handshake with 429 and Retry-After: 60. + # Within this rate budget, a full concurrent cap accepts then closes with 1013 + # before hello, allowing browsers to back off without evicting existing clients. + max_connects_per_minute: 10 # Node staleness, deletion, and clock-drift thresholds. #nodes: diff --git a/docs/docs.go b/docs/docs.go index 7e443c37..efc4bd1b 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4383,7 +4383,7 @@ var SwaggerInfo = &swag.Spec{ BasePath: "/api/v1", Schemes: []string{"http", "https"}, Title: "MeshCore Beacon API", - Description: "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.\nREST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget.", + Description: "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.\nREST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget.\nWebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open.", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, LeftDelim: "{{", diff --git a/docs/swagger.json b/docs/swagger.json index aa2eba99..018641ce 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -5,7 +5,7 @@ ], "swagger": "2.0", "info": { - "description": "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.\nREST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget.", + "description": "MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket.\nREST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget.\nWebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open.", "title": "MeshCore Beacon API", "termsOfService": "https://github.com/MeshCore-Beacon/beacon-server", "contact": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index d10fb48b..6e7923e9 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1247,6 +1247,7 @@ info: description: |- MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket. REST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget. + WebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open. license: name: AGPL-3-or-later termsOfService: https://github.com/MeshCore-Beacon/beacon-server diff --git a/internal/api/router/admin_config_test.go b/internal/api/router/admin_config_test.go index 544c8a61..fd242862 100644 --- a/internal/api/router/admin_config_test.go +++ b/internal/api/router/admin_config_test.go @@ -32,7 +32,7 @@ func TestAdminConfig(t *testing.T) { want = map[string]any{"allowed_origins": []any{"https://example.test"}, "allowed_methods": []any{"GET"}, "allowed_headers": []any{"Authorization", "X-Operator"}, "allow_credentials": true, "max_age": float64(600)} } - handler := New(nil, nil, []*ingest.Worker{{}, {}}, 5, cfg, config.ServerConfig{}, config.AuthConfig{APIKey: key}, config.ResolvedRateLimitConfig{}) + handler := New(nil, nil, []*ingest.Worker{{}, {}}, 5, 1000, cfg, config.ServerConfig{}, config.AuthConfig{APIKey: key}, config.ResolvedRateLimitConfig{}) if custom { cfg.AllowedOrigins[0] = "https://changed.test" cfg.AllowedMethods[0] = "DELETE" diff --git a/internal/api/router/auth_test.go b/internal/api/router/auth_test.go index 57bc5ac7..6c7b1967 100644 --- a/internal/api/router/auth_test.go +++ b/internal/api/router/auth_test.go @@ -13,7 +13,7 @@ import ( func TestAdminAuthBoundary(t *testing.T) { for _, key := range []string{"", "synthetic-test-key"} { - handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{APIKey: key}, config.ResolvedRateLimitConfig{}) + handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{APIKey: key}, config.ResolvedRateLimitConfig{}) for _, path := range []string{"/api/v1/admin", "/api/v1/admin/", "/api/v1/admin/config", "/api/v1/admin//config"} { for _, method := range []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"} { for _, token := range []string{"", "wrong", "synthetic-test-key"} { diff --git a/internal/api/router/rate_limit_test.go b/internal/api/router/rate_limit_test.go index ac7f7aec..95ab42f2 100644 --- a/internal/api/router/rate_limit_test.go +++ b/internal/api/router/rate_limit_test.go @@ -22,7 +22,7 @@ import ( ) func TestDefaultAPIRateLimit(t *testing.T) { - handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.Resolve(&config.Config{}).RateLimit) + handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.Resolve(&config.Config{}).RateLimit) for i := 0; i <= 300; i++ { request := httptest.NewRequest(http.MethodGet, "/api/v1/brokers", nil) request.RemoteAddr = "198.51.100.1:1234" @@ -56,7 +56,7 @@ func TestAPIRateLimitContractAndLogging(t *testing.T) { slog.SetDefault(slog.New(slog.NewJSONHandler(&logs, nil))) t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) proxy := config.ServerConfig{TrustedProxies: []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}} - handler := New(nil, nil, nil, 5, config.CORSConfig{}, proxy, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 2, Burst: 10}) + handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, proxy, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 2, Burst: 10}) for _, path := range []string{"/api/v1/brokers", "/api/v1/packets?limit=0"} { response := rateRequest(handler, path, "127.0.0.1:1234", "198.51.100.25") if response.Code != http.StatusOK && response.Code != http.StatusBadRequest { @@ -111,7 +111,7 @@ func TestAPIRateLimitClientIdentity(t *testing.T) { if tc.trusted { proxy.TrustedProxies = []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")} } - handler := New(nil, nil, nil, 5, config.CORSConfig{}, proxy, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1}) + handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, proxy, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1}) if response := rateRequest(handler, "/api/v1/brokers", tc.firstPeer, tc.firstHeader); response.Code != http.StatusOK { t.Fatalf("first client: %d", response.Code) } @@ -128,7 +128,7 @@ func TestAPIRateLimitClientIdentity(t *testing.T) { func TestAPIRateLimitWindowsAndExclusions(t *testing.T) { synctest.Test(t, func(t *testing.T) { - handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 4, Burst: 2}) + handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 4, Burst: 2}) request := func() *httptest.ResponseRecorder { return rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", "") } @@ -164,7 +164,7 @@ func TestAPIRateLimitWindowsAndExclusions(t *testing.T) { t.Fatal("minute budget did not recover") } }) - handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: false, RequestsPerMinute: 1, Burst: 1}) + handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: false, RequestsPerMinute: 1, Burst: 1}) for range 5 { if response := rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", ""); response.Code != http.StatusOK || response.Header().Get("Retry-After") != "" { t.Fatal("disabled limiter still applied") @@ -173,7 +173,7 @@ func TestAPIRateLimitWindowsAndExclusions(t *testing.T) { } func TestAPIRateLimitConcurrentRequests(t *testing.T) { - handler := New(nil, nil, nil, 5, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 5, Burst: 5}) + handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 5, Burst: 5}) var accepted, rejected atomic.Int32 var group sync.WaitGroup for range 20 { diff --git a/internal/api/router/router.go b/internal/api/router/router.go index 2e91ab81..34db94a5 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -39,7 +39,7 @@ import ( // /stats → stats subrouter // // Admin endpoints require a configured bearer key. -func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP int, corsCfg config.CORSConfig, serverCfg config.ServerConfig, authCfg config.AuthConfig, rateLimitCfg config.ResolvedRateLimitConfig) http.Handler { +func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP, maxConnectsPerMinute int, corsCfg config.CORSConfig, serverCfg config.ServerConfig, authCfg config.AuthConfig, rateLimitCfg config.ResolvedRateLimitConfig) http.Handler { r := chi.NewRouter() // ── CORS ───────────────────────────────────────────────────────────────── @@ -99,7 +99,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP )) // ── WebSocket ──────────────────────────────────────────────────────────── - r.Get("/ws", ws.Handler(h, reader, maxConnsPerIP)) + r.Get("/ws", ws.Handler(h, reader, maxConnsPerIP, maxConnectsPerMinute)) // ── Public REST API (v1) ───────────────────────────────────────────────── r.Route("/api/v1", func(r chi.Router) { diff --git a/internal/api/router/router_test.go b/internal/api/router/router_test.go index d3565bca..88927e29 100644 --- a/internal/api/router/router_test.go +++ b/internal/api/router/router_test.go @@ -19,7 +19,7 @@ import ( ) func TestWebSocketLimitIgnoresUntrustedHeaders(t *testing.T) { - checkWebSocketLimit(t, config.ServerConfig{}, "198.51.100.2", http.StatusTooManyRequests) + checkWebSocketLimit(t, config.ServerConfig{}, "198.51.100.2", true) } func TestWebSocketLimitUsesTrustedClientIP(t *testing.T) { @@ -27,16 +27,16 @@ func TestWebSocketLimitUsesTrustedClientIP(t *testing.T) { netip.MustParsePrefix("127.0.0.1/8"), netip.MustParsePrefix("::1/128"), }} t.Run("different clients", func(t *testing.T) { - checkWebSocketLimit(t, cfg, "198.51.100.2", http.StatusSwitchingProtocols) + checkWebSocketLimit(t, cfg, "198.51.100.2", false) }) t.Run("same client", func(t *testing.T) { - checkWebSocketLimit(t, cfg, "198.51.100.1", http.StatusTooManyRequests) + checkWebSocketLimit(t, cfg, "198.51.100.1", true) }) } -func checkWebSocketLimit(t *testing.T, cfg config.ServerConfig, secondIP string, wantStatus int) { +func checkWebSocketLimit(t *testing.T, cfg config.ServerConfig, secondIP string, wantShed bool) { t.Helper() - server := httptest.NewServer(New(hub.New(), nil, nil, 1, config.CORSConfig{}, cfg, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1})) + server := httptest.NewServer(New(hub.New(), nil, nil, 1, 10, config.CORSConfig{}, cfg, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1})) defer server.Close() // Exhaust this client's REST budget before checking its independent WS cap. for _, want := range []int{http.StatusOK, http.StatusTooManyRequests} { @@ -71,10 +71,22 @@ func checkWebSocketLimit(t *testing.T, cfg config.ServerConfig, secondIP string, t.Fatalf("read hello: %v", err) } second, response, err := dial(secondIP) - if second != nil { - second.CloseNow() + if err != nil || response == nil || response.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("second handshake: response=%v err=%v", response, err) } - if response == nil || response.StatusCode != wantStatus || (err == nil) != (wantStatus == http.StatusSwitchingProtocols) { - t.Fatalf("second connection: response=%v err=%v, want status %d", response, err, wantStatus) + defer second.CloseNow() + _, _, err = second.Read(ctx) + if wantShed && websocket.CloseStatus(err) != websocket.StatusTryAgainLater { + t.Fatalf("expected close 1013, got %v", err) + } + if !wantShed && err != nil { + t.Fatalf("independent client did not receive hello: %v", err) + } + if err := first.Write(ctx, websocket.MessageText, []byte(`{"v":1,"type":"ping","id":"still-active"}`)); err != nil { + t.Fatal(err) + } + _, pong, err := first.Read(ctx) + if err != nil || !strings.Contains(string(pong), `"type":"pong"`) { + t.Fatalf("established client stopped responding: %s %v", pong, err) } } diff --git a/internal/api/router/ws_attempt_test.go b/internal/api/router/ws_attempt_test.go new file mode 100644 index 00000000..b0f73982 --- /dev/null +++ b/internal/api/router/ws_attempt_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "net/netip" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestWebSocketAttemptClientIdentity(t *testing.T) { + for _, tc := range []struct { + name, firstPeer, secondPeer, firstHeader, secondHeader string + trusted, shared bool + }{ + {"untrusted rotation", "198.51.100.1:1", "198.51.100.1:2", "192.0.2.1", "192.0.2.2", false, true}, + {"independent peers", "198.51.100.1:1", "198.51.100.2:1", "192.0.2.1", "192.0.2.1", false, false}, + {"trusted independent clients", "127.0.0.1:1", "127.0.0.1:2", "198.51.100.1", "198.51.100.2", true, false}, + {"trusted shared client", "127.0.0.1:1", "127.0.0.1:2", "198.51.100.1", "198.51.100.1", true, true}, + {"IPv6 shared prefix", "[2001:db8:1::1]:1", "[2001:db8:1::2]:2", "", "", false, true}, + {"IPv6 independent prefixes", "[2001:db8:1::1]:1", "[2001:db8:2::1]:2", "", "", false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := config.ServerConfig{} + if tc.trusted { + cfg.TrustedProxies = []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")} + } + handler := New(nil, nil, nil, 5, 1, config.CORSConfig{}, cfg, config.AuthConfig{}, config.ResolvedRateLimitConfig{}) + attempt := func(peer, header string) int { + request := httptest.NewRequest(http.MethodGet, "/ws", nil) + request.RemoteAddr = peer + request.Header.Set("X-Real-IP", header) + request.Header.Set("X-Forwarded-For", header) + request.Header.Set("True-Client-IP", header) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + return response.Code + } + first := attempt(tc.firstPeer, tc.firstHeader) + if first < 400 || first == http.StatusTooManyRequests { + t.Fatalf("first failed handshake: %d", first) + } + second := attempt(tc.secondPeer, tc.secondHeader) + if (second == http.StatusTooManyRequests) != tc.shared || second < 400 { + t.Fatalf("second handshake status %d; shared budget=%v", second, tc.shared) + } + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 9455a2e5..9147b871 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -85,6 +85,7 @@ type ResolvedConfig struct { RouteGrace time.Duration RouteMinObservations int MaxConnsPerIP int + MaxConnectsPerMinute int ViewRefreshInterval time.Duration ReconfirmInterval time.Duration CleanupInterval time.Duration @@ -231,6 +232,9 @@ type WebSocketConfig struct { // MaxConnectionsPerIP is the maximum number of concurrent WebSocket // connections allowed from a single IP address. Defaults to 5 if not set. MaxConnectionsPerIP int `yaml:"max_connections_per_ip"` + // MaxConnectsPerMinute limits upgrade attempts, including failed handshakes. + // Zero/omitted defaults to 10; IPv6 addresses share a /64 attempt budget. + MaxConnectsPerMinute int `yaml:"max_connects_per_minute"` } // PacketsConfig controls packet retention behaviour. @@ -382,6 +386,9 @@ func Load(path string) (*Config, error) { if cfg.RateLimit.RequestsPerMinute < 0 || cfg.RateLimit.Burst < 0 { return nil, fmt.Errorf("ratelimit.requests_per_minute and ratelimit.burst must be positive or zero for defaults") } + if cfg.WebSocket.MaxConnectsPerMinute < 0 { + return nil, fmt.Errorf("websocket.max_connects_per_minute must be positive or zero for the default") + } configDir := filepath.Dir(path) for iata, details := range cfg.IATAs { if details.BorderFile != "" && !filepath.IsAbs(details.BorderFile) { @@ -407,6 +414,7 @@ func Resolve(cfg *Config) ResolvedConfig { RouteGrace: cfg.Routes.Grace.Duration, RouteMinObservations: cfg.Routes.MinObservations, MaxConnsPerIP: cfg.WebSocket.MaxConnectionsPerIP, + MaxConnectsPerMinute: cfg.WebSocket.MaxConnectsPerMinute, ViewRefreshInterval: cfg.Background.ViewRefresh.Duration, ReconfirmInterval: cfg.Background.Reconfirm.Duration, CleanupInterval: cfg.Background.Cleanup.Duration, @@ -446,6 +454,9 @@ func Resolve(cfg *Config) ResolvedConfig { if r.MaxConnsPerIP == 0 { r.MaxConnsPerIP = 5 } + if r.MaxConnectsPerMinute == 0 { + r.MaxConnectsPerMinute = 10 + } if r.ViewRefreshInterval == 0 { r.ViewRefreshInterval = time.Hour } @@ -477,9 +488,9 @@ func Resolve(cfg *Config) ResolvedConfig { func (r ResolvedConfig) String() string { return fmt.Sprintf( - "telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d", + "telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d maxConnectsPerMinute=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d", r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, - r.MaxConnsPerIP, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, + r.MaxConnsPerIP, r.MaxConnectsPerMinute, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold, r.NodeStaleThreshold, r.NodeDeleteAfter, r.ObserverDeleteAfter, r.RateLimit.Enabled, r.RateLimit.RequestsPerMinute, r.RateLimit.Burst, diff --git a/internal/config/ws_connect_test.go b/internal/config/ws_connect_test.go new file mode 100644 index 00000000..5914707c --- /dev/null +++ b/internal/config/ws_connect_test.go @@ -0,0 +1,37 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWebSocketConnectConfig(t *testing.T) { + for _, tc := range []struct { + name, yaml string + want int + wantError bool + }{ + {"omitted", "{}", 10, false}, + {"zero default", "websocket: {max_connects_per_minute: 0}", 10, false}, + {"custom", "websocket: {max_connects_per_minute: 30}", 30, false}, + {"negative", "websocket: {max_connects_per_minute: -1}", 0, true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(tc.yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if (err != nil) != tc.wantError { + t.Fatalf("Load error = %v, want error = %v", err, tc.wantError) + } + if err == nil && (Resolve(cfg).MaxConnectsPerMinute != tc.want || Resolve(cfg).MaxConnsPerIP != 5) { + t.Fatalf("unexpected resolved WebSocket limits: %+v", Resolve(cfg)) + } + }) + } +} diff --git a/internal/ws/connect_limit_test.go b/internal/ws/connect_limit_test.go new file mode 100644 index 00000000..2e55a8b8 --- /dev/null +++ b/internal/ws/connect_limit_test.go @@ -0,0 +1,174 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ws + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/coder/websocket" +) + +func connectTestServer(t *testing.T, maxConns, maxConnects int) (*httptest.Server, <-chan struct{}) { + t.Helper() + handler := Handler(hub.New(), nil, maxConns, maxConnects) + done := make(chan struct{}, 64) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handler(w, r) + done <- struct{}{} + })) + t.Cleanup(server.Close) + return server, done +} + +func TestFailedHandshakeDoesNotUseConnectionSlot(t *testing.T) { + server, done := connectTestServer(t, 1, 10) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + response, err := server.Client().Get(server.URL) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode < 400 || response.StatusCode == http.StatusTooManyRequests { + t.Fatalf("expected a failed handshake: %d", response.StatusCode) + } + waitConnectionEnd(t, ctx, done) + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + defer conn.CloseNow() + if _, _, err := conn.Read(ctx); err != nil { + t.Fatalf("failed handshake consumed a connection slot: %v", err) + } + conn.CloseNow() + waitConnectionEnd(t, ctx, done) +} + +func TestConnectAttemptBudgetAndRecovery(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + handler := Handler(nil, nil, 1, 2) + attempt := func() *httptest.ResponseRecorder { + request := httptest.NewRequest(http.MethodGet, "/ws", nil) + request.RemoteAddr = "198.51.100.1:1234" + response := httptest.NewRecorder() + handler(response, request) + return response + } + for range 2 { + if response := attempt(); response.Code < 400 || response.Code == http.StatusTooManyRequests { + t.Fatalf("expected failed handshake within budget: %d", response.Code) + } + } + if response := attempt(); response.Code != http.StatusTooManyRequests || response.Header().Get("Retry-After") != "60" { + t.Fatalf("failed handshakes bypassed attempt budget: %d %v", response.Code, response.Header()) + } + time.Sleep(2 * time.Minute) + if response := attempt(); response.Code < 400 || response.Code == http.StatusTooManyRequests { + t.Fatalf("attempt budget did not recover: %d", response.Code) + } + }) +} + +func TestConcurrentConnectsRespectBothLimits(t *testing.T) { + for _, tc := range []struct { + name string + conns, attempts int + wantOpen, wantShed int32 + wantRejected int32 + }{ + {"concurrent cap", 2, 20, 2, 6, 0}, + {"attempt cap", 20, 2, 2, 0, 6}, + } { + t.Run(tc.name, func(t *testing.T) { + server, done := connectTestServer(t, tc.conns, tc.attempts) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var opened, shed, rejected atomic.Int32 + connections := make(chan *websocket.Conn, 8) + defer func() { + close(connections) + for conn := range connections { + conn.CloseNow() + } + for range 8 { + waitConnectionEnd(t, ctx, done) + } + }() + var group sync.WaitGroup + for range 8 { + group.Go(func() { + conn, response, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil) + if err != nil { + if response != nil && response.StatusCode == http.StatusTooManyRequests { + rejected.Add(1) + } else { + t.Errorf("unexpected handshake failure: %v", err) + } + return + } + connections <- conn // Keep accepted connections open until every attempt completes. + if _, _, err := conn.Read(ctx); err == nil { + opened.Add(1) + } else if websocket.CloseStatus(err) == websocket.StatusTryAgainLater { + shed.Add(1) + } else { + t.Errorf("unexpected WebSocket read error: %v", err) + } + }) + } + group.Wait() + if opened.Load() != tc.wantOpen || shed.Load() != tc.wantShed || rejected.Load() != tc.wantRejected { + t.Fatalf("opened=%d shed=%d rejected=%d", opened.Load(), shed.Load(), rejected.Load()) + } + }) + } +} + +func waitConnectionEnd(t *testing.T, ctx context.Context, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-ctx.Done(): + t.Fatal("connection handler did not release its slot") + } +} + +func TestConnectChurnLimited(t *testing.T) { + server, done := connectTestServer(t, 1, 10) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + url := "ws" + strings.TrimPrefix(server.URL, "http") + for range 10 { + conn, _, err := websocket.Dial(ctx, url, nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := conn.Read(ctx); err != nil { + conn.CloseNow() + t.Fatalf("read hello: %v", err) + } + conn.CloseNow() + waitConnectionEnd(t, ctx, done) + } + conn, response, err := websocket.Dial(ctx, url, nil) + if conn != nil { + conn.CloseNow() + } + if err == nil || response == nil || response.StatusCode != http.StatusTooManyRequests { + t.Fatal("eleventh connection attempt was not rejected with HTTP 429") + } + if response.Header.Get("Retry-After") != "60" { + t.Fatal("missing upgrade retry guidance") + } +} diff --git a/internal/ws/handler.go b/internal/ws/handler.go index 1e4772e0..cb632eaa 100644 --- a/internal/ws/handler.go +++ b/internal/ws/handler.go @@ -39,6 +39,7 @@ import ( "time" "github.com/coder/websocket" + "github.com/go-chi/httprate" "github.com/google/uuid" "github.com/MeshCore-Beacon/beacon-server/internal/api" @@ -52,24 +53,34 @@ const ( // Handler returns an http.HandlerFunc that requires the hub to be injected. // Wire it via router.New(h) so the hub is available at startup. -func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP int) http.HandlerFunc { +func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP, maxConnectsPerMinute int) http.HandlerFunc { limiter := newIPLimiter(maxConnsPerIP) + attempts := httprate.NewRateLimiter(maxConnectsPerMinute, time.Minute, + httprate.WithResponseHeaders(httprate.ResponseHeaders{RetryAfter: "Retry-After"})) return func(w http.ResponseWriter, r *http.Request) { ip := r.RemoteAddr if host, _, err := net.SplitHostPort(ip); err == nil { ip = host } - if !limiter.acquire(ip) { - slog.Warn(fmt.Sprintf("ws: connection limit reached for IP %s", ip), "component", "ws") - http.Error(w, "too many connections from this IP", http.StatusTooManyRequests) + // Count attempts before Accept, including failed handshakes. RemoteAddr + // was already resolved by TrustedProxyIP; never read forwarding headers here. + if attempts.RespondOnLimit(w, r, httprate.CanonicalizeIP(ip)) { return } - defer limiter.release(ip) conn, err := websocket.Accept(w, r, nil) if err != nil { slog.Warn("ws: failed to accept connection", "component", "ws", "error", err) return } + defer conn.CloseNow() + if !limiter.acquire(ip) { + // This attempt passed the rate budget. Give the accepted socket a + // browser-visible backoff signal before allocating a hub client. + slog.Warn("ws: connection limit reached", "component", "ws", "client_ip", ip) + _ = conn.Close(websocket.StatusTryAgainLater, "connection limit reached") + return + } + defer limiter.release(ip) connID := uuid.NewString() client := h.NewClient() From 775aedfd2acd5a6adf06b5af2c005432d0936771 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 15:27:25 -0400 Subject: [PATCH 042/141] feat(admin): update runtime CORS origins consistently (#150) * feat(admin): update runtime CORS origins consistently * test(admin): preserve rate-limit headers after CORS updates * test(admin): preserve router connection budget after rebase --- README.md | 24 +++- docs/docs.go | 125 ++++++++++++++++- docs/swagger.json | 125 ++++++++++++++++- docs/swagger.yaml | 91 +++++++++++- internal/api/admin.go | 18 ++- internal/api/handlers/admin.go | 74 +++++++++- internal/api/middleware/runtime_config.go | 129 ++++++++++++++++++ .../api/middleware/runtime_config_test.go | 110 +++++++++++++++ internal/api/router/admin_config_test.go | 4 +- internal/api/router/auth_test.go | 3 + internal/api/router/router.go | 13 +- internal/api/router/runtime_config_test.go | 93 +++++++++++++ 12 files changed, 776 insertions(+), 33 deletions(-) create mode 100644 internal/api/middleware/runtime_config.go create mode 100644 internal/api/middleware/runtime_config_test.go create mode 100644 internal/api/router/runtime_config_test.go diff --git a/README.md b/README.md index 1251cb09..10db9885 100644 --- a/README.md +++ b/README.md @@ -127,14 +127,14 @@ With no key, admin requests return JSON 503 while public reads and WebSockets continue normally. With a key, missing, incorrect or duplicate Authorization headers return JSON 401 with `WWW-Authenticate: Bearer`. -`GET /api/v1/admin/config` returns selected startup settings: CORS options with +`GET /api/v1/admin/config` returns selected running settings: CORS options with Beacon defaults applied, `auth.configured`, and `ingest.broker_count` (configured broker workers, not connection status or a tunable processing-worker pool). The CORS lists are the options supplied to the middleware; its normal matching -normalization still applies. The response is a startup snapshot and excludes +normalization still applies. The response excludes credential fields, broker addresses, channel material, database settings and -other configuration. Changes require a restart. Configuration writes and account -operations are not implemented; unknown admin paths return 404 and unsupported +other configuration. Account operations are not implemented; unknown admin paths +return 404 and unsupported methods on the config endpoint return 405 after authentication. Global CORS preflights remain public. Use a long, randomly generated key, keep it out of source control and logs, and send it only in the Authorization header, @@ -142,6 +142,22 @@ never the URL or request body. Require HTTPS at the reverse proxy and restrict direct access to Beacon's HTTP listener to that proxy or a private connection. Changing the key requires a restart. No API key is issued automatically. +`PUT /api/v1/admin/config` accepts only +`{"cors":{"allowed_origins":["https://example.org"]}}`. It replaces the entire +origin list immediately and updates the reported configuration with the same +policy. Requests already in progress may use the previous policy. Concurrent +valid updates are serialized; updates take effect one at a time. The response +contains `config`, `persisted: false` and `requires_restart: false`. + +Updates are **runtime-only**: no file or database is written, and restarting +reloads the saved configuration. Keep 1–32 ASCII HTTP(S) origins, at most 512 bytes +each, with an optional single hostname wildcard; a sole `*` permits all origins. +Empty/null lists, URL paths/queries/credentials, control characters and unknown +fields are rejected. Requests must be JSON, at most 16 KiB. Other CORS options, +auth/credential fields and broker count cannot be changed here; there is no +configurable `ingest.worker_count`. Cross-origin admin clients need PUT allowed +in the saved CORS methods. CORS controls browser access, not authentication. + ### Environment variables (`.env`) | Variable | Default | Description | diff --git a/docs/docs.go b/docs/docs.go index efc4bd1b..fed853eb 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -29,14 +29,14 @@ const docTemplate = `{ "AdminKey": [] } ], - "description": "Returns CORS startup options with Beacon defaults, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Changes require restart; this endpoint is read-only.", + "description": "Returns current CORS options, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Runtime origin updates are lost on restart.", "produces": [ "application/json" ], "tags": [ "Admin" ], - "summary": "Inspect selected startup configuration", + "summary": "Inspect selected running configuration", "responses": { "200": { "description": "OK", @@ -63,6 +63,88 @@ const docTemplate = `{ } } } + }, + "put": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Replaces only cors.allowed_origins immediately. Updates are serialized; concurrent valid updates are applied one at a time. Already-running requests may finish with the previous policy. Nothing is persisted; restart reloads saved configuration.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Update runtime CORS origins", + "parameters": [ + { + "description": "1-32 ASCII HTTP(S) origins, at most 512 bytes each; one hostname wildcard is supported, or a sole *. Empty/null lists and unsupported fields are rejected. Body at most 16 KiB.", + "name": "config", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } } }, "/brokers": { @@ -4303,6 +4385,45 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest": { + "type": "object", + "required": [ + "allowed_origins" + ], + "properties": { + "allowed_origins": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest": { + "type": "object", + "required": [ + "cors" + ], + "properties": { + "cors": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse": { + "type": "object", + "properties": { + "config": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig" + }, + "persisted": { + "type": "boolean" + }, + "requires_restart": { + "type": "boolean" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 018641ce..b61d26a4 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -27,14 +27,14 @@ "AdminKey": [] } ], - "description": "Returns CORS startup options with Beacon defaults, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Changes require restart; this endpoint is read-only.", + "description": "Returns current CORS options, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Runtime origin updates are lost on restart.", "produces": [ "application/json" ], "tags": [ "Admin" ], - "summary": "Inspect selected startup configuration", + "summary": "Inspect selected running configuration", "responses": { "200": { "description": "OK", @@ -61,6 +61,88 @@ } } } + }, + "put": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Replaces only cors.allowed_origins immediately. Updates are serialized; concurrent valid updates are applied one at a time. Already-running requests may finish with the previous policy. Nothing is persisted; restart reloads saved configuration.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Update runtime CORS origins", + "parameters": [ + { + "description": "1-32 ASCII HTTP(S) origins, at most 512 bytes each; one hostname wildcard is supported, or a sole *. Empty/null lists and unsupported fields are rejected. Body at most 16 KiB.", + "name": "config", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } } }, "/brokers": { @@ -4301,6 +4383,45 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest": { + "type": "object", + "required": [ + "allowed_origins" + ], + "properties": { + "allowed_origins": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest": { + "type": "object", + "required": [ + "cors" + ], + "properties": { + "cors": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse": { + "type": "object", + "properties": { + "config": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig" + }, + "persisted": { + "type": "boolean" + }, + "requires_restart": { + "type": "boolean" + } + } + }, "internal_api_handlers.APIError": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6e7923e9..26655f5d 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1223,6 +1223,31 @@ definitions: description: TRACE or PING type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest: + properties: + allowed_origins: + items: + type: string + type: array + required: + - allowed_origins + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest: + properties: + cors: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminCORSRequest' + required: + - cors + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse: + properties: + config: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AdminConfig' + persisted: + type: boolean + requires_restart: + type: boolean + type: object internal_api_handlers.APIError: properties: code: @@ -1256,9 +1281,9 @@ info: paths: /admin/config: get: - description: Returns CORS startup options with Beacon defaults, auth configuration - status and configured broker count. Credential fields and other configuration - are excluded. Changes require restart; this endpoint is read-only. + description: Returns current CORS options, auth configuration status and configured + broker count. Credential fields and other configuration are excluded. Runtime + origin updates are lost on restart. produces: - application/json responses: @@ -1280,7 +1305,65 @@ paths: type: object security: - AdminKey: [] - summary: Inspect selected startup configuration + summary: Inspect selected running configuration + tags: + - Admin + put: + consumes: + - application/json + description: Replaces only cors.allowed_origins immediately. Updates are serialized; + concurrent valid updates are applied one at a time. Already-running requests + may finish with the previous policy. Nothing is persisted; restart reloads + saved configuration. + parameters: + - description: 1-32 ASCII HTTP(S) origins, at most 512 bytes each; one hostname + wildcard is supported, or a sole *. Empty/null lists and unsupported fields + are rejected. Body at most 16 KiB. + in: body + name: config + required: true + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.UpdateAdminConfigResponse' + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "413": + description: Request Entity Too Large + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "415": + description: Unsupported Media Type + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Update runtime CORS origins tags: - Admin /brokers: diff --git a/internal/api/admin.go b/internal/api/admin.go index 70cfc3a7..e71b38c6 100644 --- a/internal/api/admin.go +++ b/internal/api/admin.go @@ -3,7 +3,7 @@ package api -// AdminConfig is a whitelist of nonsecret settings captured when the router starts. +// AdminConfig is a whitelist of selected nonsecret running settings. // It is not a serialization of the full file or environment configuration. type AdminConfig struct { Auth AdminAuthConfig `json:"auth"` @@ -15,7 +15,7 @@ type AdminAuthConfig struct { Configured bool `json:"configured"` } -// AdminCORSConfig reports the startup options supplied to the CORS middleware, +// AdminCORSConfig reports the current options supplied to the CORS middleware, // with Beacon defaults applied, before that library's matching normalization. type AdminCORSConfig struct { AllowedOrigins []string `json:"allowed_origins"` @@ -30,3 +30,17 @@ type AdminIngestConfig struct { // or a configurable processing-worker pool. BrokerCount int `json:"broker_count"` } + +type UpdateAdminConfigRequest struct { + CORS *UpdateAdminCORSRequest `json:"cors" validate:"required"` +} + +type UpdateAdminCORSRequest struct { + AllowedOrigins *[]string `json:"allowed_origins" validate:"required"` +} + +type UpdateAdminConfigResponse struct { + Config AdminConfig `json:"config"` + Persisted bool `json:"persisted"` + RequiresRestart bool `json:"requires_restart"` +} diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go index 4ec31d33..b775a515 100644 --- a/internal/api/handlers/admin.go +++ b/internal/api/handlers/admin.go @@ -4,24 +4,30 @@ package handlers import ( + "encoding/json" + "errors" + "io" + "mime" "net/http" "github.com/MeshCore-Beacon/beacon-server/internal/api" + mw "github.com/MeshCore-Beacon/beacon-server/internal/api/middleware" "github.com/go-chi/chi/v5" ) -// AdminRouter mounts read-only operator endpoints. Its caller must wrap the +// AdminRouter mounts operator endpoints. Its caller must wrap the // entire subrouter with BearerAuth, including unknown paths and methods. -func AdminRouter(snapshot api.AdminConfig) http.Handler { +func AdminRouter(runtime *mw.RuntimeConfig) http.Handler { r := chi.NewRouter() - r.Get("/config", getAdminConfig(snapshot)) + r.Get("/config", getAdminConfig(runtime)) + r.Put("/config", updateAdminConfig(runtime)) return r } // getAdminConfig godoc // -// @Summary Inspect selected startup configuration -// @Description Returns CORS startup options with Beacon defaults, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Changes require restart; this endpoint is read-only. +// @Summary Inspect selected running configuration +// @Description Returns current CORS options, auth configuration status and configured broker count. Credential fields and other configuration are excluded. Runtime origin updates are lost on restart. // @Tags Admin // @Produce json // @Security AdminKey @@ -29,8 +35,62 @@ func AdminRouter(snapshot api.AdminConfig) http.Handler { // @Failure 401 {object} map[string]APIError // @Failure 503 {object} map[string]APIError // @Router /admin/config [get] -func getAdminConfig(snapshot api.AdminConfig) http.HandlerFunc { +func getAdminConfig(runtime *mw.RuntimeConfig) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - respond(w, http.StatusOK, snapshot) + respond(w, http.StatusOK, runtime.Snapshot()) + } +} + +// updateAdminConfig godoc +// @Summary Update runtime CORS origins +// @Description Replaces only cors.allowed_origins immediately. Updates are serialized; concurrent valid updates are applied one at a time. Already-running requests may finish with the previous policy. Nothing is persisted; restart reloads saved configuration. +// @Tags Admin +// @Accept json +// @Produce json +// @Security AdminKey +// @Param config body api.UpdateAdminConfigRequest true "1-32 ASCII HTTP(S) origins, at most 512 bytes each; one hostname wildcard is supported, or a sole *. Empty/null lists and unsupported fields are rejected. Body at most 16 KiB." +// @Success 200 {object} api.UpdateAdminConfigResponse +// @Failure 400,401,413,415,503 {object} map[string]APIError +// @Router /admin/config [put] +func updateAdminConfig(runtime *mw.RuntimeConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + media, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || media != "application/json" { + respondError(w, 415, "Content-Type must be application/json") + return + } + r.Body = http.MaxBytesReader(w, r.Body, 16*1024) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + var input api.UpdateAdminConfigRequest + err = decoder.Decode(&input) + if err == nil { + var extra any + err = decoder.Decode(&extra) + if errors.Is(err, io.EOF) { + err = nil + } else if err == nil { + err = errors.New("multiple JSON values") + } + } + if err != nil { + var sizeError *http.MaxBytesError + if errors.As(err, &sizeError) { + respondError(w, 413, "request body exceeds 16 KiB") + } else { + respondError(w, 400, "invalid configuration JSON") + } + return + } + if input.CORS == nil || input.CORS.AllowedOrigins == nil { + respondError(w, 400, "cors.allowed_origins is required") + return + } + config, err := runtime.UpdateOrigins(*input.CORS.AllowedOrigins) + if err != nil { + respondError(w, 400, mw.ErrInvalidOrigins.Error()) + return + } + respond(w, 200, api.UpdateAdminConfigResponse{Config: config, Persisted: false, RequiresRestart: false}) } } diff --git a/internal/api/middleware/runtime_config.go b/internal/api/middleware/runtime_config.go new file mode 100644 index 00000000..cb6a9e60 --- /dev/null +++ b/internal/api/middleware/runtime_config.go @@ -0,0 +1,129 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "errors" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "unicode" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/cors" +) + +var ErrInvalidOrigins = errors.New("allowed_origins must contain 1-32 ASCII HTTP(S) origins (at most 512 bytes each), or a sole *; paths, credentials, queries and control characters are not allowed") + +type runtimePolicy struct { + config api.AdminConfig + handler http.Handler +} + +// RuntimeConfig owns one router's CORS handler and its nonsecret config view. +// Published policies are immutable. Updates are runtime-only, never persisted. +type RuntimeConfig struct { + mu sync.Mutex + current atomic.Pointer[runtimePolicy] + next http.Handler + exposedHeaders []string +} + +func NewRuntimeConfig(config api.AdminConfig, exposedHeaders []string) *RuntimeConfig { + s := &RuntimeConfig{exposedHeaders: slices.Clone(exposedHeaders)} + s.current.Store(&runtimePolicy{config: cloneAdminConfig(config)}) + return s +} + +// CORS is bound once as the outer middleware of its owning router. +func (s *RuntimeConfig) CORS(next http.Handler) http.Handler { + s.mu.Lock() + defer s.mu.Unlock() + if s.next != nil { + panic("runtime CORS already bound") + } + s.next = next + s.current.Store(s.policy(s.current.Load().config)) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.current.Load().handler.ServeHTTP(w, r) + }) +} + +func (s *RuntimeConfig) Snapshot() api.AdminConfig { + return cloneAdminConfig(s.current.Load().config) +} + +func (s *RuntimeConfig) UpdateOrigins(origins []string) (api.AdminConfig, error) { + validated, err := validateOrigins(origins) + if err != nil { + return api.AdminConfig{}, err + } + s.mu.Lock() + defer s.mu.Unlock() + config := cloneAdminConfig(s.current.Load().config) + config.CORS.AllowedOrigins = validated + s.current.Store(s.policy(config)) + return cloneAdminConfig(config), nil +} + +func (s *RuntimeConfig) policy(config api.AdminConfig) *runtimePolicy { + p := &runtimePolicy{config: config} + if s.next != nil { + p.handler = cors.Handler(cors.Options{ + AllowedOrigins: slices.Clone(config.CORS.AllowedOrigins), + AllowedMethods: slices.Clone(config.CORS.AllowedMethods), + AllowedHeaders: slices.Clone(config.CORS.AllowedHeaders), + AllowCredentials: config.CORS.AllowCredentials, MaxAge: config.CORS.MaxAge, + ExposedHeaders: slices.Clone(s.exposedHeaders), + })(s.next) + } + return p +} + +func cloneAdminConfig(config api.AdminConfig) api.AdminConfig { + config.CORS.AllowedOrigins = slices.Clone(config.CORS.AllowedOrigins) + config.CORS.AllowedMethods = slices.Clone(config.CORS.AllowedMethods) + config.CORS.AllowedHeaders = slices.Clone(config.CORS.AllowedHeaders) + return config +} + +func validateOrigins(origins []string) ([]string, error) { + if len(origins) == 0 || len(origins) > 32 { + return nil, ErrInvalidOrigins + } + result := make([]string, len(origins)) + for i, origin := range origins { + if len(origin) > 512 || strings.IndexFunc(origin, unicode.IsControl) >= 0 { + return nil, ErrInvalidOrigins + } + origin = strings.TrimSpace(origin) + if strings.IndexFunc(origin, func(r rune) bool { return r > unicode.MaxASCII }) >= 0 { + return nil, ErrInvalidOrigins + } + origin = strings.ToLower(origin) + if origin == "*" { + if len(origins) != 1 { + return nil, ErrInvalidOrigins + } + result[i] = origin + continue + } + u, err := url.Parse(origin) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(origin, "#") || strings.Count(u.Host, "*") > 1 || strings.Contains(u.Host, "%") || strings.HasSuffix(u.Host, ":") { + return nil, ErrInvalidOrigins + } + if port := u.Port(); port != "" { + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 { + return nil, ErrInvalidOrigins + } + } + result[i] = origin + } + return result, nil +} diff --git a/internal/api/middleware/runtime_config_test.go b/internal/api/middleware/runtime_config_test.go new file mode 100644 index 00000000..402edd2b --- /dev/null +++ b/internal/api/middleware/runtime_config_test.go @@ -0,0 +1,110 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package middleware + +import ( + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func TestValidateOrigins(t *testing.T) { + for _, tc := range []struct { + input []string + valid bool + }{ + {[]string{"*"}, true}, {[]string{" HTTPS://Example.test ", "https://*.example.test", "http://127.0.0.1:8080", "http://[::1]:8080"}, true}, + {nil, false}, {[]string{}, false}, {make([]string, 33), false}, {[]string{""}, false}, {[]string{"null"}, false}, + {[]string{"*", "https://example.test"}, false}, {[]string{"ftp://example.test"}, false}, {[]string{"https://"}, false}, + {[]string{"https://example.test/"}, false}, {[]string{"https://example.test?"}, false}, {[]string{"https://example.test#"}, false}, + {[]string{"https://user:secret@example.test"}, false}, {[]string{"https://example.test:65536"}, false}, {[]string{"https://example.test:"}, false}, + {[]string{"https://*.*.test"}, false}, {[]string{"https://example.test\n"}, false}, {[]string{"https://é.test"}, false}, {[]string{"https://K.test"}, false}, {[]string{strings.Repeat("x", 513)}, false}, + } { + _, err := validateOrigins(tc.input) + if (err == nil) != tc.valid { + t.Fatalf("origins=%q valid=%v error=%v", tc.input, tc.valid, err) + } + } + input := []string{" HTTPS://Example.test "} + got, err := validateOrigins(input) + if err != nil || got[0] != "https://example.test" || input[0] != " HTTPS://Example.test " { + t.Fatal("normalization changed caller state") + } +} + +func TestRuntimePolicyAndConcurrency(t *testing.T) { + initial := api.AdminConfig{Auth: api.AdminAuthConfig{Configured: true}, Ingest: api.AdminIngestConfig{BrokerCount: 2}, + CORS: api.AdminCORSConfig{AllowedOrigins: []string{"https://before.test"}, AllowedMethods: []string{"GET", "PUT"}, AllowedHeaders: []string{"Authorization"}, AllowCredentials: true, MaxAge: 600}} + state := NewRuntimeConfig(initial, []string{"Retry-After"}) + initial.CORS.AllowedOrigins[0] = "https://caller-change.test" + handler := state.CORS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) })) + preflight := func(origin string) *httptest.ResponseRecorder { + r := httptest.NewRequest("OPTIONS", "/api/v1/brokers", nil) + r.Header.Set("Origin", origin) + r.Header.Set("Access-Control-Request-Method", "PUT") + r.Header.Set("Access-Control-Request-Headers", "Authorization") + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + return w + } + if preflight("https://before.test").Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("startup policy changed with caller slice") + } + origins := []string{"https://*.after.test"} + updated, err := state.UpdateOrigins(origins) + if err != nil { + t.Fatal(err) + } + origins[0] = "https://caller-change.test" + updated.CORS.AllowedOrigins[0] = "https://response-change.test" + if preflight("https://before.test").Header().Get("Access-Control-Allow-Origin") != "" { + t.Fatal("old policy remains active") + } + w := preflight("https://app.after.test") + if w.Code != 200 || w.Header().Get("Access-Control-Allow-Origin") != "https://app.after.test" || w.Header().Get("Access-Control-Max-Age") != "600" || w.Header().Get("Access-Control-Allow-Credentials") != "true" { + t.Fatal("new policy/options not enforced") + } + r := httptest.NewRequest("GET", "/", nil) + r.Header.Set("Origin", "https://app.after.test") + w = httptest.NewRecorder() + handler.ServeHTTP(w, r) + if w.Header().Get("Access-Control-Expose-Headers") != "Retry-After" { + t.Fatal("exposed headers lost") + } + before := state.Snapshot() + if _, err := state.UpdateOrigins(nil); err == nil || !reflect.DeepEqual(state.Snapshot(), before) { + t.Fatal("invalid update changed config") + } + var group sync.WaitGroup + for i := 0; i < 8; i++ { + group.Add(1) + go func(i int) { + defer group.Done() + for j := 0; j < 40; j++ { + origin := "https://a.test" + if i%2 != 0 { + origin = "https://b.test" + } + if _, err := state.UpdateOrigins([]string{origin}); err != nil { + t.Error(err) + } + snapshot := state.Snapshot() + if len(snapshot.CORS.AllowedOrigins) != 1 || !snapshot.Auth.Configured || snapshot.Ingest.BrokerCount != 2 { + t.Error("torn config") + } + preflight(origin) + } + }(i) + } + group.Wait() + final := state.Snapshot() + if preflight(final.CORS.AllowedOrigins[0]).Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("reported/enforced final policy differs") + } +} diff --git a/internal/api/router/admin_config_test.go b/internal/api/router/admin_config_test.go index fd242862..43ee7cfb 100644 --- a/internal/api/router/admin_config_test.go +++ b/internal/api/router/admin_config_test.go @@ -69,8 +69,8 @@ func TestAdminConfig(t *testing.T) { t.Fatal("config escaped authentication") } } - if request("PUT", "/api/v1/admin/config", headers).Code != http.StatusMethodNotAllowed { - t.Fatal("config writes were accepted") + if request("PUT", "/api/v1/admin/config", headers).Code != http.StatusUnsupportedMediaType { + t.Fatal("untyped config write was accepted") } again := request("GET", "/api/v1/admin/config", headers) if again.Body.String() != response.Body.String() { diff --git a/internal/api/router/auth_test.go b/internal/api/router/auth_test.go index 6c7b1967..66c9614e 100644 --- a/internal/api/router/auth_test.go +++ b/internal/api/router/auth_test.go @@ -33,6 +33,9 @@ func TestAdminAuthBoundary(t *testing.T) { if method == "GET" { want = 200 } + if method == "PUT" { + want = 415 + } } } } diff --git a/internal/api/router/router.go b/internal/api/router/router.go index 34db94a5..d58e8b71 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -11,7 +11,6 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - "github.com/go-chi/cors" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" @@ -59,14 +58,6 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP, if maxAge == 0 { maxAge = 300 } - r.Use(cors.Handler(cors.Options{ - AllowedOrigins: allowedOrigins, - AllowedMethods: allowedMethods, - AllowedHeaders: allowedHeaders, - ExposedHeaders: []string{"Retry-After"}, - AllowCredentials: corsCfg.AllowCredentials, - MaxAge: maxAge, - })) // Capture only values used by this router. Do not retain Config, credentials // or caller-owned slices in the admin response. adminConfig := api.AdminConfig{ @@ -80,6 +71,8 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP, }, Ingest: api.AdminIngestConfig{BrokerCount: len(workers)}, } + runtimeConfig := mw.NewRuntimeConfig(adminConfig, []string{"Retry-After"}) + r.Use(runtimeConfig.CORS) // ── Global middleware ──────────────────────────────────────────────────── r.Use(middleware.RequestID) @@ -121,7 +114,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP, }) // Protect the entire subtree, including its root and unknown paths. - r.Mount("/admin", mw.BearerAuth(authCfg.APIKey, handlers.AdminRouter(adminConfig))) + r.Mount("/admin", mw.BearerAuth(authCfg.APIKey, handlers.AdminRouter(runtimeConfig))) }) return r diff --git a/internal/api/router/runtime_config_test.go b/internal/api/router/runtime_config_test.go new file mode 100644 index 00000000..17dc67b9 --- /dev/null +++ b/internal/api/router/runtime_config_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestRuntimeConfigUpdate(t *testing.T) { + newRouter := func() http.Handler { + return New(nil, nil, nil, 5, 1000, config.CORSConfig{AllowedOrigins: []string{"https://before.test"}}, config.ServerConfig{}, config.AuthConfig{APIKey: "test-key"}, config.ResolvedRateLimitConfig{}) + } + handler := newRouter() + request := func(method, path, body, token, media, origin string) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, path, strings.NewReader(body)) + r.Header.Set("Content-Type", media) + if token != "" { + r.Header.Set("Authorization", "Bearer "+token) + } + if origin != "" { + r.Header.Set("Origin", origin) + } + if method == "OPTIONS" { + r.Header.Set("Access-Control-Request-Method", "GET") + r.Header.Set("Access-Control-Request-Headers", "Authorization") + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + return w + } + body := `{"cors":{"allowed_origins":["https://after.test"]}}` + for _, token := range []string{"", "wrong"} { + if request("PUT", "/api/v1/admin/config", body, token, "application/json", "").Code != 401 { + t.Fatal("unauthorized update") + } + } + w := request("PUT", "/api/v1/admin/config", body, "test-key", "application/json", "") + var result api.UpdateAdminConfigResponse + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &result) != nil || result.Persisted || result.RequiresRestart || w.Header().Get("Cache-Control") != "no-store" { + t.Fatal("update contract") + } + if result.Config.CORS.AllowedOrigins[0] != "https://after.test" { + t.Fatal("wrong returned policy") + } + read := request("GET", "/api/v1/admin/config", "", "test-key", "", "") + var actual api.AdminConfig + if json.Unmarshal(read.Body.Bytes(), &actual) != nil || actual.CORS.AllowedOrigins[0] != "https://after.test" { + t.Fatal("GET still reports old config") + } + for _, tc := range []struct { + body, media string + status int + }{ + {`{}`, "application/json", 400}, {`null`, "application/json", 400}, {`{"cors":null}`, "application/json", 400}, + {`{"cors":{"allowed_origins":null}}`, "application/json", 400}, {`{"cors":{"allowed_origins":[]}}`, "application/json", 400}, + {`{"auth":{"api_key":"secret-sentinel"}}`, "application/json", 400}, {`{"ingest":{"worker_count":4}}`, "application/json", 400}, + {`{"cors":{"allowed_origins":["*"],"max_age":0}}`, "application/json", 400}, + {`{"cors":{"allowed_origins":["https://user:secret-sentinel@site.test"]}}`, "application/json", 400}, + {body + ` {}`, "application/json", 400}, {body, "text/plain", 415}, {body + strings.Repeat(" ", 16384), "application/json", 413}, + } { + bad := request("PUT", "/api/v1/admin/config", tc.body, "test-key", tc.media, "") + if bad.Code != tc.status || strings.Contains(bad.Body.String(), "secret-sentinel") { + t.Fatalf("invalid update status=%d want=%d", bad.Code, tc.status) + } + if request("GET", "/api/v1/admin/config", "", "test-key", "", "").Body.String() != read.Body.String() { + t.Fatal("invalid update changed state") + } + } + if request("OPTIONS", "/api/v1/brokers", "", "", "", "https://before.test").Header().Get("Access-Control-Allow-Origin") != "" { + t.Fatal("old origin allowed") + } + if request("OPTIONS", "/api/v1/brokers", "", "", "", "https://after.test").Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("new origin blocked") + } + if exposed := request("GET", "/api/v1/brokers", "", "", "", "https://after.test").Header().Get("Access-Control-Expose-Headers"); !strings.EqualFold(exposed, "Retry-After") { + t.Fatalf("runtime update lost rate-limit backoff header: %q", exposed) + } + if request("GET", "/api/v1/brokers", "", "", "", "").Code != 200 { + t.Fatal("public reads blocked") + } + handler = newRouter() + if request("OPTIONS", "/api/v1/brokers", "", "", "", "https://before.test").Header().Get("Access-Control-Allow-Origin") == "" { + t.Fatal("new router did not reload startup policy") + } +} From 6aba17c75ea92451a5b7c38451101bf7f008f9b4 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 15:27:34 -0400 Subject: [PATCH 043/141] feat(nodes): indicate repeaters outside configured local borders (#153) --- README.md | 28 ++++++ cmd/beacon/main.go | 10 ++ config.yaml.example | 3 + docs/docs.go | 8 ++ docs/swagger.json | 8 ++ docs/swagger.yaml | 10 ++ internal/api/local_borders.go | 50 ++++++++++ internal/api/local_borders_test.go | 124 ++++++++++++++++++++++++ internal/api/nodes.go | 3 + internal/borders/borders.go | 90 +++++++++++++++++ internal/borders/borders_test.go | 116 ++++++++++++++++++++++ internal/config/border.go | 3 + internal/config/config.go | 3 + internal/config/foreign_borders.go | 55 +++++++++++ internal/config/foreign_borders_test.go | 66 +++++++++++++ internal/ingest/foreign_nodes_test.go | 81 ++++++++++++++++ internal/ingest/ingest.go | 3 + internal/ingest/side_effects.go | 7 ++ 18 files changed, 668 insertions(+) create mode 100644 internal/api/local_borders.go create mode 100644 internal/api/local_borders_test.go create mode 100644 internal/borders/borders.go create mode 100644 internal/borders/borders_test.go create mode 100644 internal/config/foreign_borders.go create mode 100644 internal/config/foreign_borders_test.go create mode 100644 internal/ingest/foreign_nodes_test.go diff --git a/README.md b/README.md index 10db9885..2609e748 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,7 @@ websocket: # Node staleness, deletion, and clock-drift thresholds. nodes: + mark_foreign: false # optional indication for repeaters outside configured IATA borders stale_threshold: 24h # mark a node "stale" in the API after this long unseen (default: 24h) delete_after: 720h # delete a node entirely after this long unseen (default: 30 days, same default as packets.retention) clock_drift_threshold: 5m # |device clock - server clock| above which clockOutOfSync=true for a repeater/room server (default: 5m) @@ -287,6 +288,33 @@ must be defined here — they are not auto-created. --- +### Foreign repeater indication + +Set `nodes.mark_foreign: true` to expose `possiblyForeign` on repeater nodes. +The local operating area is the union of **all configured** +`iatas..borderFile` GeoJSON Polygon/MultiPolygon features. IATAs without +border files do not add an area; airport coordinates and the current API region +filter are not boundaries. Enabling this with no borders, missing files or +invalid geometry fails startup. Border changes require a restart. + +Inside any polygon (including its edges) means `false`; outside the entire union +means `true`. Hole interiors are outside; hole edges are local. Other node roles, +missing/invalid positions and the 0/0 location reset have no classification. +This is a hint based on reported position, not proof of a repeater's origin. +Packet ingestion, heard-in IATAs and route matching remain unchanged. + +Node list/detail reads apply the current geometry after cache reads, so existing +historical nodes need no backfill. A `nodeUpdate` includes `possiblyForeign` +when its advert provides a position: a boolean for known positions, `null` to +clear an unknown/reset position. Omission retains the previous value when a +repeater's advert omits its position. A change to another role also sends `null`. +The field is omitted everywhere when the feature is disabled (the default). + +Use longitude/latitude coordinate order and split antimeridian-crossing borders +into MultiPolygons as described in [RFC 7946 section 3.1.9](https://www.rfc-editor.org/rfc/rfc7946#section-3.1.9). +Classification rejects unsplit edges spanning more than 180 degrees rather than +silently treating them as the complementary global area. + ## Authentication API authentication is not yet implemented. Beacon is intended for trusted diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 219b1390..f2a9294c 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -108,6 +108,11 @@ func main() { } resolved := config.Resolve(cfg) + localBorders, err := config.LoadLocalBorders(cfg) + if err != nil { + slog.Error("invalid local border configuration", "component", "startup", "error", err) + os.Exit(1) + } slog.Info(fmt.Sprintf("config: loaded — %s", resolved), "component", "startup") @@ -242,6 +247,7 @@ func main() { broker1 := ingest.New( ingest.Config{ BrokerName: "mqtt1", + LocalBorders: localBorders, URL: getEnv("MQTT_BROKER_1_URL"), Username: getEnv("MQTT_BROKER_1_USERNAME"), Password: getEnv("MQTT_BROKER_1_PASSWORD"), @@ -257,6 +263,7 @@ func main() { broker2 := ingest.New( ingest.Config{ BrokerName: "mqtt2", + LocalBorders: localBorders, URL: getEnv("MQTT_BROKER_2_URL"), Username: getEnv("MQTT_BROKER_2_USERNAME"), Password: getEnv("MQTT_BROKER_2_PASSWORD"), @@ -293,6 +300,9 @@ func main() { go scheduler.Start(ctx) // ── HTTP server ────────────────────────────────────────────────────────── + // Wrap after wiring cache invalidators and cleanup callbacks to the actual + // CachedReader. Only response projections receive the geographic annotation. + reader = api.WithLocalBorders(reader, localBorders) r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, resolved.MaxConnectsPerMinute, cfg.CORS, cfg.Server, cfg.Auth, resolved.RateLimit) srv := &http.Server{ diff --git a/config.yaml.example b/config.yaml.example index 9686dcdd..c32525b7 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -145,6 +145,9 @@ websocket: # Node staleness, deletion, and clock-drift thresholds. #nodes: +# mark_foreign: false # opt in to possiblyForeign on repeater nodes +# # uses the union of configured iatas.*.borderFile polygons; +# # requires at least one valid border; no packet filtering # stale_threshold: 24h # default: 24h # delete_after: 720h # default: 30 days, same default as packets.retention # clock_drift_threshold: 5m # default: 5m diff --git a/docs/docs.go b/docs/docs.go index fed853eb..7b096572 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -3094,6 +3094,10 @@ const docTemplate = `{ "description": "UUID of the associated observer row, if any", "type": "string" }, + "possiblyForeign": { + "description": "PossiblyForeign compares a repeater's stored position with this server's\nconfigured IATA border union. Omitted when disabled, unknown, or not repeater.", + "type": "boolean" + }, "publicKey": { "description": "hex-encoded Ed25519 public key", "type": "string" @@ -3224,6 +3228,10 @@ const docTemplate = `{ "description": "UUID of the associated observer row, if any", "type": "string" }, + "possiblyForeign": { + "description": "PossiblyForeign compares a repeater's stored position with this server's\nconfigured IATA border union. Omitted when disabled, unknown, or not repeater.", + "type": "boolean" + }, "publicKey": { "description": "hex-encoded Ed25519 public key", "type": "string" diff --git a/docs/swagger.json b/docs/swagger.json index b61d26a4..9803c083 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -3092,6 +3092,10 @@ "description": "UUID of the associated observer row, if any", "type": "string" }, + "possiblyForeign": { + "description": "PossiblyForeign compares a repeater's stored position with this server's\nconfigured IATA border union. Omitted when disabled, unknown, or not repeater.", + "type": "boolean" + }, "publicKey": { "description": "hex-encoded Ed25519 public key", "type": "string" @@ -3222,6 +3226,10 @@ "description": "UUID of the associated observer row, if any", "type": "string" }, + "possiblyForeign": { + "description": "PossiblyForeign compares a repeater's stored position with this server's\nconfigured IATA border union. Omitted when disabled, unknown, or not repeater.", + "type": "boolean" + }, "publicKey": { "description": "hex-encoded Ed25519 public key", "type": "string" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 26655f5d..f1d4293b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -318,6 +318,11 @@ definitions: observerId: description: UUID of the associated observer row, if any type: string + possiblyForeign: + description: |- + PossiblyForeign compares a repeater's stored position with this server's + configured IATA border union. Omitted when disabled, unknown, or not repeater. + type: boolean publicKey: description: hex-encoded Ed25519 public key type: string @@ -413,6 +418,11 @@ definitions: observerId: description: UUID of the associated observer row, if any type: string + possiblyForeign: + description: |- + PossiblyForeign compares a repeater's stored position with this server's + configured IATA border union. Omitted when disabled, unknown, or not repeater. + type: boolean publicKey: description: hex-encoded Ed25519 public key type: string diff --git a/internal/api/local_borders.go b/internal/api/local_borders.go new file mode 100644 index 00000000..278f7f0a --- /dev/null +++ b/internal/api/local_borders.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "context" + "slices" + + "github.com/MeshCore-Beacon/beacon-server/internal/borders" + "github.com/google/uuid" +) + +// WithLocalBorders annotates node responses after cache reads. Historical rows +// use the current startup configuration without extra SQL or cached-policy drift. +// Cached values are copied; the decorator never changes its inner reader's data. +func WithLocalBorders(reader Reader, local *borders.Local) Reader { + if local == nil { + return reader + } + return &localBorderReader{Reader: reader, local: local} +} + +type localBorderReader struct { + Reader + local *borders.Local +} + +func (r *localBorderReader) ListNodes(ctx context.Context, nodeType int16, iatas []string, paths, traces *bool, pubkey []byte, prefix, name, scope string, cursor int64, limit int32, neighbors bool) (Page[NodeSummary], error) { + page, err := r.Reader.ListNodes(ctx, nodeType, iatas, paths, traces, pubkey, prefix, name, scope, cursor, limit, neighbors) + if err != nil { + return page, err + } + page.Items = slices.Clone(page.Items) + for i := range page.Items { + node := &page.Items[i] + node.PossiblyForeign = r.local.PossiblyForeign(node.NodeType, node.Latitude, node.Longitude) + } + return page, nil +} + +func (r *localBorderReader) GetNode(ctx context.Context, id uuid.UUID) (*Node, error) { + node, err := r.Reader.GetNode(ctx, id) + if err != nil || node == nil { + return node, err + } + copy := *node + copy.PossiblyForeign = r.local.PossiblyForeign(node.NodeType, node.Latitude, node.Longitude) + return ©, nil +} diff --git a/internal/api/local_borders_test.go b/internal/api/local_borders_test.go new file mode 100644 index 00000000..033f37da --- /dev/null +++ b/internal/api/local_borders_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api_test + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/borders" + "github.com/MeshCore-Beacon/beacon-server/internal/cache" + "github.com/alicebob/miniredis/v2" + "github.com/google/uuid" + "github.com/paulmach/orb" +) + +type borderReader struct { + api.Reader + node *api.Node + page api.Page[api.NodeSummary] + getCalls, listCalls int + err error +} + +func (r *borderReader) GetNode(context.Context, uuid.UUID) (*api.Node, error) { + r.getCalls++ + return r.node, r.err +} +func (r *borderReader) ListNodes(_ context.Context, _ int16, _ []string, _, _ *bool, _ []byte, _, _, _ string, _ int64, _ int32, _ bool) (api.Page[api.NodeSummary], error) { + r.listCalls++ + return r.page, r.err +} + +func TestLocalBordersAfterCache(t *testing.T) { + ctx := context.Background() + local, err := borders.New([]orb.Polygon{{{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}}}}) + if err != nil { + t.Fatal(err) + } + other, err := borders.New([]orb.Polygon{{{{30, 30}, {40, 30}, {40, 40}, {30, 40}, {30, 30}}}}) + if err != nil { + t.Fatal(err) + } + node := &api.Node{NodeSummary: api.NodeSummary{ID: uuid.New(), NodeType: 2, Latitude: new(15.0), Longitude: new(15.0), IATAs: []api.NodeIATA{{IATA: "ZZZ", LastHeard: 1}}}} + base := &borderReader{node: node, page: api.Page[api.NodeSummary]{Items: []api.NodeSummary{node.NodeSummary}, HasMore: true, NextCursor: new(int64(123))}} + mr := miniredis.RunT(t) + client := cache.NewClient(mr.Addr(), "", 0) + t.Cleanup(func() { client.Close() }) + cached := cache.NewCachedReader(base, client, cache.CacheTTLs{Nodes: time.Hour}) + first := api.WithLocalBorders(cached, local) + second := api.WithLocalBorders(cached, other) + if api.WithLocalBorders(cached, nil) != cached { + t.Fatal("disabled decorator changed reader") + } + for _, tc := range []struct { + reader api.Reader + want bool + }{{first, false}, {second, true}} { + got, err := tc.reader.GetNode(ctx, node.ID) + if err != nil { + t.Fatal(err) + } + if got.PossiblyForeign == nil || *got.PossiblyForeign != tc.want || got.IATAs[0].IATA != "ZZZ" { + t.Fatalf("detail: %+v", got) + } + page, err := tc.reader.ListNodes(ctx, 2, []string{"ZZZ"}, nil, nil, nil, "", "", "", 0, 50, false) + if err != nil { + t.Fatal(err) + } + if !page.HasMore || *page.NextCursor != 123 || page.Items[0].PossiblyForeign == nil || *page.Items[0].PossiblyForeign != tc.want { + t.Fatalf("page: %+v", page) + } + data, _ := json.Marshal(got) + if !strings.Contains(string(data), `"possiblyForeign":`) { + t.Fatal("known false was omitted") + } + } + if base.getCalls != 1 || base.listCalls != 2 { + t.Fatalf("extra reads: get=%d list=%d", base.getCalls, base.listCalls) + } + if node.PossiblyForeign != nil || base.page.Items[0].PossiblyForeign != nil { + t.Fatal("shared reader data was mutated") + } + disabled, err := cached.GetNode(ctx, node.ID) + if err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(disabled) + if strings.Contains(string(data), "possiblyForeign") { + t.Fatal("classification leaked into cached data") + } + // Existing ingest invalidation still supplies fresh coordinates to the wrapper. + base.node.Longitude = new(35.0) + cached.(*cache.CachedReader).InvalidateNode(ctx, node.ID) + updated, err := first.GetNode(ctx, node.ID) + if err != nil || updated.PossiblyForeign == nil || !*updated.PossiblyForeign || base.getCalls != 2 { + t.Fatal("updated coordinates not classified") + } +} + +func TestLocalBordersPreserveReaderErrors(t *testing.T) { + local, err := borders.New([]orb.Polygon{{{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}}}}) + if err != nil { + t.Fatal(err) + } + want := errors.New("reader unavailable") + base := &borderReader{err: want} + reader := api.WithLocalBorders(base, local) + if _, err := reader.GetNode(context.Background(), uuid.New()); !errors.Is(err, want) { + t.Fatal(err) + } + if _, err := reader.ListNodes(context.Background(), 0, nil, nil, nil, nil, "", "", "", 0, 1, false); !errors.Is(err, want) { + t.Fatal(err) + } + base.err = nil + if node, err := reader.GetNode(context.Background(), uuid.New()); node != nil || err != nil { + t.Fatal("nil result changed") + } +} diff --git a/internal/api/nodes.go b/internal/api/nodes.go index a79280b7..9dce5579 100644 --- a/internal/api/nodes.go +++ b/internal/api/nodes.go @@ -53,6 +53,9 @@ type NodeSummary struct { // Applies to every node type, unlike ClockDriftSeconds/ClockOutOfSync on Node, which // are repeater/room-server only. Stale bool `json:"stale"` + // PossiblyForeign compares a repeater's stored position with this server's + // configured IATA border union. Omitted when disabled, unknown, or not repeater. + PossiblyForeign *bool `json:"possiblyForeign,omitempty"` } // Node is the full node representation including firmware capability flags, diff --git a/internal/borders/borders.go b/internal/borders/borders.go new file mode 100644 index 00000000..1ab15016 --- /dev/null +++ b/internal/borders/borders.go @@ -0,0 +1,90 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package borders classifies advertised repeater positions against a site's +// immutable, configured operating area. It does not change heard-in IATAs. +package borders + +import ( + "fmt" + "math" + + "github.com/meshcore-go/meshcore-go" + "github.com/paulmach/orb" + "github.com/paulmach/orb/planar" +) + +type polygon struct { + rings orb.Polygon + bound orb.Bound +} + +// Local owns the configured polygon union and precomputed bounds. +// A nil Local disables classification. Methods do not mutate shared state. +type Local struct{ polygons []polygon } + +// New copies the polygons so the classifier can be shared across readers and +// ingest workers. Antimeridian regions must use GeoJSON's split representation. +func New(polygons []orb.Polygon) (*Local, error) { + if len(polygons) == 0 { + return nil, fmt.Errorf("no local border polygons configured") + } + local := &Local{} + for _, poly := range polygons { + if len(poly) == 0 { + return nil, fmt.Errorf("border polygon has no rings") + } + for _, ring := range poly { + if !ring.Closed() { + return nil, fmt.Errorf("border ring must be closed with at least four points") + } + for i, p := range ring { + if !validPosition(p.Lat(), p.Lon()) { + return nil, fmt.Errorf("border coordinates must be finite longitude/latitude in range") + } + if i > 0 && math.Abs(p.Lon()-ring[i-1].Lon()) > 180 { + return nil, fmt.Errorf("split antimeridian-crossing borders into a MultiPolygon") + } + } + } + if planar.Area(poly) <= 0 { + return nil, fmt.Errorf("border polygon must have positive area") + } + copy := poly.Clone() + local.polygons = append(local.polygons, polygon{copy, copy.Bound()}) + } + return local, nil +} + +func validPosition(lat, lng float64) bool { + return !math.IsNaN(lat) && !math.IsNaN(lng) && lat >= -90 && lat <= 90 && lng >= -180 && lng <= 180 +} + +// PossiblyForeign is nil when disabled, the role is not repeater, or position +// is unknown/invalid (including the protocol's 0/0 location reset). +// All polygon boundaries, including hole edges, count as local; hole interiors +// remain outside. Location is self-reported, so this is an indication only. +func (local *Local) PossiblyForeign(role int16, lat, lng *float64) *bool { + if local == nil || role != int16(meshcore.AdvertTypeRepeater) || lat == nil || lng == nil || !validPosition(*lat, *lng) || *lat == 0 && *lng == 0 { + return nil + } + point := orb.Point{*lng, *lat} + foreign := true + for _, poly := range local.polygons { + if !poly.bound.Contains(point) || !planar.RingContains(poly.rings[0], point) { + continue + } + inside := true + for _, hole := range poly.rings[1:] { + if planar.RingContains(hole, point) && planar.DistanceFrom(orb.LineString(hole), point) != 0 { + inside = false + break + } + } + if inside { + foreign = false + break + } + } + return &foreign +} diff --git a/internal/borders/borders_test.go b/internal/borders/borders_test.go new file mode 100644 index 00000000..a90aaf94 --- /dev/null +++ b/internal/borders/borders_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package borders + +import ( + "math" + "testing" + + "github.com/paulmach/orb" +) + +func TestPossiblyForeign(t *testing.T) { + outer := orb.Ring{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}} + hole := orb.Ring{{12, 12}, {14, 12}, {14, 14}, {12, 14}, {12, 12}} + other := orb.Ring{{30, 10}, {40, 10}, {40, 20}, {30, 20}, {30, 10}} + local, err := New([]orb.Polygon{{outer, hole}, {other}}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + role int16 + lat, lng float64 + want *bool + }{ + {"inside", 2, 15, 15, new(false)}, + {"outside", 2, 25, 25, new(true)}, + {"second polygon", 2, 15, 35, new(false)}, + {"hole", 2, 13, 13, new(true)}, + {"exterior edge", 2, 15, 10, new(false)}, + {"hole edge", 2, 13, 12, new(false)}, + {"vertex", 2, 10, 10, new(false)}, + {"companion", 1, 25, 25, nil}, + {"room", 3, 25, 25, nil}, + {"sensor", 4, 25, 25, nil}, + {"zero reset", 2, 0, 0, nil}, + {"equator is valid", 2, 0, 25, new(true)}, + {"prime meridian is valid", 2, 25, 0, new(true)}, + {"invalid latitude", 2, 91, 25, nil}, + {"invalid longitude", 2, 25, -181, nil}, + {"NaN", 2, math.NaN(), 25, nil}, + {"infinite", 2, 25, math.Inf(1), nil}, + } { + t.Run(tc.name, func(t *testing.T) { + got := local.PossiblyForeign(tc.role, &tc.lat, &tc.lng) + if (got == nil) != (tc.want == nil) || got != nil && *got != *tc.want { + t.Fatalf("got %v, want %v", got, tc.want) + } + }) + } + if local.PossiblyForeign(2, nil, new(15.0)) != nil || local.PossiblyForeign(2, new(15.0), nil) != nil { + t.Fatal("missing coordinate classified") + } + var disabled *Local + if disabled.PossiblyForeign(2, new(25.0), new(25.0)) != nil { + t.Fatal("disabled classification") + } + // The shared startup classifier owns its geometry; callers cannot mutate it. + outer[0] = orb.Point{99, 99} + if *local.PossiblyForeign(2, new(15.0), new(15.0)) { + t.Fatal("input mutation changed classifier") + } +} + +func TestLocalBordersValidation(t *testing.T) { + for _, polys := range [][]orb.Polygon{ + nil, {{}}, {{{{0, 0}, {1, 0}, {1, 1}}}}, + {{{{0, 0}, {0, 0}, {0, 0}, {0, 0}}}}, + {{{{179, 10}, {-179, 10}, {-179, 20}, {179, 20}, {179, 10}}}}, + } { + if _, err := New(polys); err == nil { + t.Fatalf("accepted invalid/uncut geometry: %v", polys) + } + } + cut := []orb.Polygon{ + {{{179, 10}, {180, 10}, {180, 20}, {179, 20}, {179, 10}}}, + {{{-180, 10}, {-179, 10}, {-179, 20}, {-180, 20}, {-180, 10}}}, + } + local, err := New(cut) + if err != nil { + t.Fatal(err) + } + for _, lon := range []float64{179.5, -179.5, 180, -180} { + if *local.PossiblyForeign(2, new(15.0), &lon) { + t.Fatalf("dateline island excluded: %v", lon) + } + } + if !*local.PossiblyForeign(2, new(15.0), new(0.0)) { + t.Fatal("uncut global region inferred") + } +} + +func BenchmarkPossiblyForeign(b *testing.B) { + ring := make(orb.Ring, 10001) + for i := 0; i < 10000; i++ { + a := 2 * math.Pi * float64(i) / 10000 + ring[i] = orb.Point{20 + math.Cos(a), 50 + math.Sin(a)} + } + ring[10000] = ring[0] + local, err := New([]orb.Polygon{{ring}}) + if err != nil { + b.Fatal(err) + } + for _, tc := range []struct { + name string + lat, lng float64 + }{{"inside_10k_vertices", 50, 20}, {"outside_bounds_10k_vertices", 0, 1}} { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + local.PossiblyForeign(2, &tc.lat, &tc.lng) + } + }) + } +} diff --git a/internal/config/border.go b/internal/config/border.go index 0978a408..a297ee9c 100644 --- a/internal/config/border.go +++ b/internal/config/border.go @@ -30,6 +30,9 @@ func ValidateBorder(raw []byte) (json.RawMessage, error) { if feat.Type != "Feature" { return nil, fmt.Errorf(`top-level "type" must be "Feature", got %q`, feat.Type) } + if feat.Geometry == nil { + return nil, fmt.Errorf("Feature must have a Polygon or MultiPolygon geometry") + } switch geom := feat.Geometry.(type) { case orb.Polygon: if err := validateBorderPolygon(geom); err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index 9147b871..b5fc09a2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -259,6 +259,9 @@ type RoutesConfig struct { // NodesConfig controls node-derived signal thresholds. type NodesConfig struct { + // MarkForeign annotates repeaters outside the union of configured IATA border + // files. Disabled by default; enabling it requires at least one usable border. + MarkForeign bool `yaml:"mark_foreign"` // ClockDriftThreshold is the |device clock - server clock| magnitude, measured from a // repeater/room server's ADVERT timestamp, above which the node API reports // clockOutOfSync=true for that node. Defaults to 5m if not set. diff --git a/internal/config/foreign_borders.go b/internal/config/foreign_borders.go new file mode 100644 index 00000000..a9ca62e1 --- /dev/null +++ b/internal/config/foreign_borders.go @@ -0,0 +1,55 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "fmt" + "os" + "sort" + + "github.com/MeshCore-Beacon/beacon-server/internal/borders" + "github.com/paulmach/orb" + "github.com/paulmach/orb/geojson" +) + +// LoadLocalBorders builds one startup-only classifier from configured border +// files. IATA airport coordinates and auto-discovered IATAs are not boundaries. +func LoadLocalBorders(cfg *Config) (*borders.Local, error) { + if !cfg.Nodes.MarkForeign { + return nil, nil + } + var codes []string + for code, details := range cfg.IATAs { + if details.BorderFile != "" { + codes = append(codes, code) + } + } + sort.Strings(codes) + var polygons []orb.Polygon + for _, code := range codes { + raw, err := os.ReadFile(cfg.IATAs[code].BorderFile) + if err != nil { + return nil, fmt.Errorf("nodes.mark_foreign: reading border for %s: %w", code, err) + } + validated, err := ValidateBorder(raw) + if err != nil { + return nil, fmt.Errorf("nodes.mark_foreign: invalid border for %s: %w", code, err) + } + feature, err := geojson.UnmarshalFeature(validated) + if err != nil { + return nil, fmt.Errorf("nodes.mark_foreign: decoding border for %s: %w", code, err) + } + switch geometry := feature.Geometry.(type) { + case orb.Polygon: + polygons = append(polygons, geometry) + case orb.MultiPolygon: + polygons = append(polygons, geometry...) + } + } + local, err := borders.New(polygons) + if err != nil { + return nil, fmt.Errorf("nodes.mark_foreign: %w", err) + } + return local, nil +} diff --git a/internal/config/foreign_borders_test.go b/internal/config/foreign_borders_test.go new file mode 100644 index 00000000..d158f911 --- /dev/null +++ b/internal/config/foreign_borders_test.go @@ -0,0 +1,66 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestValidateBorderNoGeometry(t *testing.T) { + for _, raw := range []string{`{"type":"Feature","geometry":null}`, `{"type":"Feature"}`} { + if _, err := ValidateBorder([]byte(raw)); err == nil { + t.Fatal("missing geometry accepted") + } + } +} + +func TestLoadLocalBorders(t *testing.T) { + if got, err := LoadLocalBorders(&Config{}); err != nil || got != nil { + t.Fatalf("disabled: %v %v", got, err) + } + if _, err := LoadLocalBorders(&Config{Nodes: NodesConfig{MarkForeign: true}}); err == nil { + t.Fatal("enabled with no border") + } + dir := t.TempDir() + border := filepath.Join(dir, "region.json") + raw := `{"type":"Feature","geometry":{"type":"MultiPolygon","coordinates":[[[[10,10],[20,10],[20,20],[10,20],[10,10]]],[[[30,10],[40,10],[40,20],[30,20],[30,10]]]]}}` + if err := os.WriteFile(border, []byte(raw), 0600); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte("nodes:\n mark_foreign: true\niatas:\n AAA:\n borderFile: region.json\n BBB:\n name: Airport without an operating border\n"), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatal(err) + } + local, err := LoadLocalBorders(cfg) + if err != nil { + t.Fatal(err) + } + if *local.PossiblyForeign(2, new(15.0), new(15.0)) || *local.PossiblyForeign(2, new(15.0), new(35.0)) || !*local.PossiblyForeign(2, new(25.0), new(25.0)) { + t.Fatal("polygon union not used") + } + for _, bad := range []string{`not json`, `{"type":"Feature","geometry":null}`, `{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[179,10],[-179,10],[-179,20],[179,20],[179,10]]]}}`} { + if err := os.WriteFile(border, []byte(bad), 0600); err != nil { + t.Fatal(err) + } + if _, err := LoadLocalBorders(cfg); err == nil || !strings.Contains(err.Error(), "nodes.mark_foreign") { + t.Fatalf("bad boundary did not fail clearly: %v", err) + } + } + if err := os.Remove(border); err != nil { + t.Fatal(err) + } + if _, err := LoadLocalBorders(cfg); err == nil { + t.Fatal("missing file accepted") + } + if *local.PossiblyForeign(2, new(15.0), new(15.0)) { + t.Fatal("running classifier changed with its source file") + } +} diff --git a/internal/ingest/foreign_nodes_test.go b/internal/ingest/foreign_nodes_test.go new file mode 100644 index 00000000..20a3f78a --- /dev/null +++ b/internal/ingest/foreign_nodes_test.go @@ -0,0 +1,81 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/binary" + "encoding/json" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/borders" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/meshcore-go/meshcore-go" + "github.com/paulmach/orb" +) + +func TestForeignNodeUpdates(t *testing.T) { + local, err := borders.New([]orb.Polygon{{{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}}}}) + if err != nil { + t.Fatal(err) + } + w, _ := newTestWorker() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "foreign", hub.Scope{Events: []hub.EventType{hub.EventNodeUpdate, hub.EventObserverStatus}}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + waitForSummarySubscriber(t, ctx, w.hub, client) + for _, tc := range []struct { + name string + enabled, position bool + role byte + lat, lng int32 + present bool + want any + }{ + {"disabled", false, true, 2, 25000000, 25000000, false, nil}, + {"inside", true, true, 2, 15000000, 15000000, true, false}, + {"outside", true, true, 2, 25000000, 25000000, true, true}, + {"omitted retains old position", true, false, 2, 0, 0, false, nil}, + {"explicit reset clears flag", true, true, 2, 0, 0, true, nil}, + {"invalid position clears flag", true, true, 2, 91000000, 25000000, true, nil}, + {"role change clears flag", true, false, meshcore.AdvertTypeChat, 0, 0, true, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + w.cfg.LocalBorders = nil + if tc.enabled { + w.cfg.LocalBorders = local + } + data := []byte{tc.role} + if tc.position { + data[0] |= meshcore.AdvertLatLonMask + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lat)) + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lng)) + } + w.handlePayloadTypeSideEffects(ctx, buildAdvertPacketWithData(t, data, false), "AAA", []byte{1}, RadioSettings{}, nil, nil, nil, 0) + for { + select { + case event := <-client.Send: + if event.Type != hub.EventNodeUpdate { + continue + } + var payload map[string]any + if err := json.Unmarshal(event.Payload, &payload); err != nil { + t.Fatal(err) + } + got, present := payload["possiblyForeign"] + if present != tc.present || got != tc.want { + t.Fatalf("flag = %v (present=%v); want %v (present=%v)", got, present, tc.want, tc.present) + } + return + case <-ctx.Done(): + t.Fatal("node update not received") + } + } + }) + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index e4edf863..b322569b 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -50,6 +50,7 @@ import ( "github.com/google/uuid" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/borders" "github.com/MeshCore-Beacon/beacon-server/internal/hub" "github.com/MeshCore-Beacon/beacon-server/internal/keystore" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" @@ -57,6 +58,8 @@ import ( // Config holds the connection parameters for one broker. type Config struct { + // LocalBorders is the same immutable classifier used by node API reads. + LocalBorders *borders.Local // BrokerName is a short human-readable label ("mqtt1", "mqtt2") used in // log messages and stored in packet_observations.source_broker. BrokerName string diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 26d01849..5d5c92c3 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -61,6 +61,9 @@ type nodeUpdateEvent struct { IATAs []api.NodeIATA `json:"iatas"` DefaultScope *string `json:"defaultScope,omitempty"` Radio *string `json:"radio,omitempty"` + // Omission retains the previous classification (no advertised position). + // The outer pointer allows explicit null to clear an unknown/reset position. + PossiblyForeign **bool `json:"possiblyForeign,omitempty"` } // handlePayloadTypeSideEffects runs payload-type-specific processing after a @@ -173,6 +176,10 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc DefaultScope: defaultScope, Radio: radioStr, } + if w.cfg.LocalBorders != nil && (lat != nil || advert.Type() != meshcore.AdvertTypeRepeater) { + foreign := w.cfg.LocalBorders.PossiblyForeign(int16(advert.Type()), lat, lon) + evt.PossiblyForeign = &foreign + } w.broadcast(hub.EventNodeUpdate, iata, meshcore.PayloadTypeAdvert, "", evt) return } From 951b79bf6ef173b4e5afc549422b12d8e2df4073 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 19 Sep 2026 21:10:45 -0400 Subject: [PATCH 044/141] refactor(api): stabilize router settings and admin route wiring (#155) --- CONTRIBUTING.md | 18 ++++++++ cmd/beacon/main.go | 6 ++- internal/api/handlers/admin.go | 5 +- internal/api/router/admin_config_test.go | 2 +- internal/api/router/admin_routes_test.go | 54 ++++++++++++++++++++++ internal/api/router/auth_test.go | 2 +- internal/api/router/options.go | 25 ++++++++++ internal/api/router/rate_limit_test.go | 12 ++--- internal/api/router/router.go | 23 +++++---- internal/api/router/router_test.go | 2 +- internal/api/router/runtime_config_test.go | 2 +- internal/api/router/ws_attempt_test.go | 2 +- 12 files changed, 128 insertions(+), 25 deletions(-) create mode 100644 internal/api/router/admin_routes_test.go create mode 100644 internal/api/router/options.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7edf1eb0..7513a94d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,6 +100,24 @@ sqlc generate ## API changes +### Router configuration and optional admin endpoints + +Keep `router.New`'s signature stable. Pass startup settings by name in +`router.Options`; a new optional setting must not require unrelated callers and +tests to supply another positional argument. + +An optional admin feature owns its subrouter, dependencies and HTTP methods. +Register it once through `Options.AdminRoutes` in the startup wiring, using a +literal path such as `/accounts`. The parent admin router applies bearer +authentication to the entire subtree; the feature must not mount itself beside +that boundary. `/config` remains owned by the shared admin router. No global +registration or `init` side effects are needed. + +When concurrent PRs touch the same startup wiring, stack them in an explicit +merge order and link each feature-only comparison. Rebase the stack together +after prerequisite merges, regenerate Swagger rather than hand-merging it, and +compile the combined result even when GitHub reports it as mergeable. + Any new or modified REST endpoint must have swagger annotations and regenerated docs: diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index f2a9294c..b518ab1f 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -303,7 +303,11 @@ func main() { // Wrap after wiring cache invalidators and cleanup callbacks to the actual // CachedReader. Only response projections receive the geographic annotation. reader = api.WithLocalBorders(reader, localBorders) - r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, resolved.MaxConnsPerIP, resolved.MaxConnectsPerMinute, cfg.CORS, cfg.Server, cfg.Auth, resolved.RateLimit) + r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, router.Options{ + MaxConnsPerIP: resolved.MaxConnsPerIP, + MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, + CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, + }) srv := &http.Server{ Addr: addr, diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go index b775a515..cf4f0d6a 100644 --- a/internal/api/handlers/admin.go +++ b/internal/api/handlers/admin.go @@ -17,10 +17,13 @@ import ( // AdminRouter mounts operator endpoints. Its caller must wrap the // entire subrouter with BearerAuth, including unknown paths and methods. -func AdminRouter(runtime *mw.RuntimeConfig) http.Handler { +func AdminRouter(runtime *mw.RuntimeConfig, routes map[string]http.Handler) http.Handler { r := chi.NewRouter() r.Get("/config", getAdminConfig(runtime)) r.Put("/config", updateAdminConfig(runtime)) + for path, handler := range routes { + r.Mount(path, handler) + } return r } diff --git a/internal/api/router/admin_config_test.go b/internal/api/router/admin_config_test.go index 43ee7cfb..9acd524c 100644 --- a/internal/api/router/admin_config_test.go +++ b/internal/api/router/admin_config_test.go @@ -32,7 +32,7 @@ func TestAdminConfig(t *testing.T) { want = map[string]any{"allowed_origins": []any{"https://example.test"}, "allowed_methods": []any{"GET"}, "allowed_headers": []any{"Authorization", "X-Operator"}, "allow_credentials": true, "max_age": float64(600)} } - handler := New(nil, nil, []*ingest.Worker{{}, {}}, 5, 1000, cfg, config.ServerConfig{}, config.AuthConfig{APIKey: key}, config.ResolvedRateLimitConfig{}) + handler := New(nil, nil, []*ingest.Worker{{}, {}}, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, CORS: cfg, Auth: config.AuthConfig{APIKey: key}}) if custom { cfg.AllowedOrigins[0] = "https://changed.test" cfg.AllowedMethods[0] = "DELETE" diff --git a/internal/api/router/admin_routes_test.go b/internal/api/router/admin_routes_test.go new file mode 100644 index 00000000..c017236c --- /dev/null +++ b/internal/api/router/admin_routes_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestAdditionalAdminRoutesStayAuthenticated(t *testing.T) { + for _, key := range []string{"", "fixture-key"} { + calls := 0 + opts := Options{Auth: config.AuthConfig{APIKey: key}, AdminRoutes: map[string]http.Handler{ + "/feature": http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.WriteHeader(http.StatusNoContent) + }), + }} + handler := New(nil, nil, nil, opts) + // Registration happens at startup; later edits to the caller's map must + // not replace the registered handler or create a new unprotected route. + delete(opts.AdminRoutes, "/feature") + for _, path := range []string{"/api/v1/admin/feature", "/api/v1/admin/feature/nested"} { + for _, method := range []string{"GET", "POST", "DELETE"} { + for _, token := range []string{"", "wrong", "fixture-key"} { + request := httptest.NewRequest(method, path, nil) + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response := httptest.NewRecorder() + before := calls + handler.ServeHTTP(response, request) + want := http.StatusServiceUnavailable + if key != "" { + want = http.StatusUnauthorized + if token == key { + want = http.StatusNoContent + } + } + if response.Code != want || response.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("%s %s: status=%d want=%d", method, path, response.Code, want) + } + if (calls != before) != (want == http.StatusNoContent) { + t.Fatal("authentication boundary bypassed") + } + } + } + } + } +} diff --git a/internal/api/router/auth_test.go b/internal/api/router/auth_test.go index 66c9614e..d8796f79 100644 --- a/internal/api/router/auth_test.go +++ b/internal/api/router/auth_test.go @@ -13,7 +13,7 @@ import ( func TestAdminAuthBoundary(t *testing.T) { for _, key := range []string{"", "synthetic-test-key"} { - handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{APIKey: key}, config.ResolvedRateLimitConfig{}) + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, Auth: config.AuthConfig{APIKey: key}}) for _, path := range []string{"/api/v1/admin", "/api/v1/admin/", "/api/v1/admin/config", "/api/v1/admin//config"} { for _, method := range []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"} { for _, token := range []string{"", "wrong", "synthetic-test-key"} { diff --git a/internal/api/router/options.go b/internal/api/router/options.go new file mode 100644 index 00000000..ec9e1fbf --- /dev/null +++ b/internal/api/router/options.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +// Options names the startup settings used by the router. Adding an optional +// route does not change New's signature or require unrelated callers to change. +type Options struct { + MaxConnsPerIP int + MaxConnectsPerMinute int + CORS config.CORSConfig + Server config.ServerConfig + Auth config.AuthConfig + RateLimit config.ResolvedRateLimitConfig + // AdminRoutes mounts operator-only subrouters at literal paths such as + // "/accounts". Each feature owns its dependencies, methods and handlers. + // The router applies authentication to every path and method in this group. + AdminRoutes map[string]http.Handler +} diff --git a/internal/api/router/rate_limit_test.go b/internal/api/router/rate_limit_test.go index 95ab42f2..b0e1cd86 100644 --- a/internal/api/router/rate_limit_test.go +++ b/internal/api/router/rate_limit_test.go @@ -22,7 +22,7 @@ import ( ) func TestDefaultAPIRateLimit(t *testing.T) { - handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.Resolve(&config.Config{}).RateLimit) + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, RateLimit: config.Resolve(&config.Config{}).RateLimit}) for i := 0; i <= 300; i++ { request := httptest.NewRequest(http.MethodGet, "/api/v1/brokers", nil) request.RemoteAddr = "198.51.100.1:1234" @@ -56,7 +56,7 @@ func TestAPIRateLimitContractAndLogging(t *testing.T) { slog.SetDefault(slog.New(slog.NewJSONHandler(&logs, nil))) t.Cleanup(func() { slog.SetDefault(previous); log.SetOutput(writer); log.SetFlags(flags) }) proxy := config.ServerConfig{TrustedProxies: []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")}} - handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, proxy, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 2, Burst: 10}) + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, Server: proxy, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 2, Burst: 10}}) for _, path := range []string{"/api/v1/brokers", "/api/v1/packets?limit=0"} { response := rateRequest(handler, path, "127.0.0.1:1234", "198.51.100.25") if response.Code != http.StatusOK && response.Code != http.StatusBadRequest { @@ -111,7 +111,7 @@ func TestAPIRateLimitClientIdentity(t *testing.T) { if tc.trusted { proxy.TrustedProxies = []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")} } - handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, proxy, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1}) + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, Server: proxy, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1}}) if response := rateRequest(handler, "/api/v1/brokers", tc.firstPeer, tc.firstHeader); response.Code != http.StatusOK { t.Fatalf("first client: %d", response.Code) } @@ -128,7 +128,7 @@ func TestAPIRateLimitClientIdentity(t *testing.T) { func TestAPIRateLimitWindowsAndExclusions(t *testing.T) { synctest.Test(t, func(t *testing.T) { - handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 4, Burst: 2}) + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 4, Burst: 2}}) request := func() *httptest.ResponseRecorder { return rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", "") } @@ -164,7 +164,7 @@ func TestAPIRateLimitWindowsAndExclusions(t *testing.T) { t.Fatal("minute budget did not recover") } }) - handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: false, RequestsPerMinute: 1, Burst: 1}) + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, RateLimit: config.ResolvedRateLimitConfig{Enabled: false, RequestsPerMinute: 1, Burst: 1}}) for range 5 { if response := rateRequest(handler, "/api/v1/brokers", "198.51.100.1:1", ""); response.Code != http.StatusOK || response.Header().Get("Retry-After") != "" { t.Fatal("disabled limiter still applied") @@ -173,7 +173,7 @@ func TestAPIRateLimitWindowsAndExclusions(t *testing.T) { } func TestAPIRateLimitConcurrentRequests(t *testing.T) { - handler := New(nil, nil, nil, 5, 1000, config.CORSConfig{}, config.ServerConfig{}, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 5, Burst: 5}) + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 5, Burst: 5}}) var accepted, rejected atomic.Int32 var group sync.WaitGroup for range 20 { diff --git a/internal/api/router/router.go b/internal/api/router/router.go index d58e8b71..447d407e 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -15,7 +15,6 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" mw "github.com/MeshCore-Beacon/beacon-server/internal/api/middleware" - "github.com/MeshCore-Beacon/beacon-server/internal/config" "github.com/MeshCore-Beacon/beacon-server/internal/hub" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/MeshCore-Beacon/beacon-server/internal/ws" @@ -38,35 +37,35 @@ import ( // /stats → stats subrouter // // Admin endpoints require a configured bearer key. -func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP, maxConnectsPerMinute int, corsCfg config.CORSConfig, serverCfg config.ServerConfig, authCfg config.AuthConfig, rateLimitCfg config.ResolvedRateLimitConfig) http.Handler { +func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) http.Handler { r := chi.NewRouter() // ── CORS ───────────────────────────────────────────────────────────────── - allowedOrigins := corsCfg.AllowedOrigins + allowedOrigins := opts.CORS.AllowedOrigins if len(allowedOrigins) == 0 { allowedOrigins = []string{"*"} } - allowedMethods := corsCfg.AllowedMethods + allowedMethods := opts.CORS.AllowedMethods if len(allowedMethods) == 0 { allowedMethods = []string{"GET", "HEAD", "OPTIONS"} } - allowedHeaders := corsCfg.AllowedHeaders + allowedHeaders := opts.CORS.AllowedHeaders if len(allowedHeaders) == 0 { allowedHeaders = []string{"Accept", "Authorization", "Content-Type"} } - maxAge := corsCfg.MaxAge + maxAge := opts.CORS.MaxAge if maxAge == 0 { maxAge = 300 } // Capture only values used by this router. Do not retain Config, credentials // or caller-owned slices in the admin response. adminConfig := api.AdminConfig{ - Auth: api.AdminAuthConfig{Configured: authCfg.APIKey != ""}, + Auth: api.AdminAuthConfig{Configured: opts.Auth.APIKey != ""}, CORS: api.AdminCORSConfig{ AllowedOrigins: append([]string{}, allowedOrigins...), AllowedMethods: append([]string{}, allowedMethods...), AllowedHeaders: append([]string{}, allowedHeaders...), - AllowCredentials: corsCfg.AllowCredentials, + AllowCredentials: opts.CORS.AllowCredentials, MaxAge: maxAge, }, Ingest: api.AdminIngestConfig{BrokerCount: len(workers)}, @@ -76,7 +75,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP, // ── Global middleware ──────────────────────────────────────────────────── r.Use(middleware.RequestID) - r.Use(mw.TrustedProxyIP(serverCfg.TrustedProxies)) + r.Use(mw.TrustedProxyIP(opts.Server.TrustedProxies)) r.Use(mw.RequestLogger) r.Use(middleware.Recoverer) r.Use(middleware.CleanPath) @@ -92,11 +91,11 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP, )) // ── WebSocket ──────────────────────────────────────────────────────────── - r.Get("/ws", ws.Handler(h, reader, maxConnsPerIP, maxConnectsPerMinute)) + r.Get("/ws", ws.Handler(h, reader, opts.MaxConnsPerIP, opts.MaxConnectsPerMinute)) // ── Public REST API (v1) ───────────────────────────────────────────────── r.Route("/api/v1", func(r chi.Router) { - r.Use(mw.RateLimit(rateLimitCfg)) + r.Use(mw.RateLimit(opts.RateLimit)) // Public group — no authentication required. r.Group(func(r chi.Router) { r.Mount("/packets", handlers.PacketsRouter(reader)) @@ -114,7 +113,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, maxConnsPerIP, }) // Protect the entire subtree, including its root and unknown paths. - r.Mount("/admin", mw.BearerAuth(authCfg.APIKey, handlers.AdminRouter(runtimeConfig))) + r.Mount("/admin", mw.BearerAuth(opts.Auth.APIKey, handlers.AdminRouter(runtimeConfig, opts.AdminRoutes))) }) return r diff --git a/internal/api/router/router_test.go b/internal/api/router/router_test.go index 88927e29..bbb434dd 100644 --- a/internal/api/router/router_test.go +++ b/internal/api/router/router_test.go @@ -36,7 +36,7 @@ func TestWebSocketLimitUsesTrustedClientIP(t *testing.T) { func checkWebSocketLimit(t *testing.T, cfg config.ServerConfig, secondIP string, wantShed bool) { t.Helper() - server := httptest.NewServer(New(hub.New(), nil, nil, 1, 10, config.CORSConfig{}, cfg, config.AuthConfig{}, config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1})) + server := httptest.NewServer(New(hub.New(), nil, nil, Options{MaxConnsPerIP: 1, MaxConnectsPerMinute: 10, Server: cfg, RateLimit: config.ResolvedRateLimitConfig{Enabled: true, RequestsPerMinute: 1, Burst: 1}})) defer server.Close() // Exhaust this client's REST budget before checking its independent WS cap. for _, want := range []int{http.StatusOK, http.StatusTooManyRequests} { diff --git a/internal/api/router/runtime_config_test.go b/internal/api/router/runtime_config_test.go index 17dc67b9..6ac6073e 100644 --- a/internal/api/router/runtime_config_test.go +++ b/internal/api/router/runtime_config_test.go @@ -16,7 +16,7 @@ import ( func TestRuntimeConfigUpdate(t *testing.T) { newRouter := func() http.Handler { - return New(nil, nil, nil, 5, 1000, config.CORSConfig{AllowedOrigins: []string{"https://before.test"}}, config.ServerConfig{}, config.AuthConfig{APIKey: "test-key"}, config.ResolvedRateLimitConfig{}) + return New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1000, CORS: config.CORSConfig{AllowedOrigins: []string{"https://before.test"}}, Auth: config.AuthConfig{APIKey: "test-key"}}) } handler := newRouter() request := func(method, path, body, token, media, origin string) *httptest.ResponseRecorder { diff --git a/internal/api/router/ws_attempt_test.go b/internal/api/router/ws_attempt_test.go index b0f73982..853ecde6 100644 --- a/internal/api/router/ws_attempt_test.go +++ b/internal/api/router/ws_attempt_test.go @@ -29,7 +29,7 @@ func TestWebSocketAttemptClientIdentity(t *testing.T) { if tc.trusted { cfg.TrustedProxies = []netip.Prefix{netip.MustParsePrefix("127.0.0.1/32")} } - handler := New(nil, nil, nil, 5, 1, config.CORSConfig{}, cfg, config.AuthConfig{}, config.ResolvedRateLimitConfig{}) + handler := New(nil, nil, nil, Options{MaxConnsPerIP: 5, MaxConnectsPerMinute: 1, Server: cfg}) attempt := func(peer, header string) int { request := httptest.NewRequest(http.MethodGet, "/ws", nil) request.RemoteAddr = peer From 4d2642abecce49ab45615633e0c2fa7e958477a1 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 23 Sep 2026 20:10:35 -0400 Subject: [PATCH 045/141] feat(admin): add protected account lifecycle endpoints (#149) --- README.md | 18 +- cmd/beacon/main.go | 2 + config.yaml.example | 2 +- docs/docs.go | 358 +++++++++++++++++++++++++ docs/swagger.json | 358 +++++++++++++++++++++++++ docs/swagger.yaml | 236 ++++++++++++++++ internal/api/account_requests.go | 8 + internal/api/handlers/accounts.go | 173 ++++++++++++ internal/api/handlers/accounts_test.go | 147 ++++++++++ internal/api/router/accounts_test.go | 53 ++++ 10 files changed, 1352 insertions(+), 3 deletions(-) create mode 100644 internal/api/account_requests.go create mode 100644 internal/api/handlers/accounts.go create mode 100644 internal/api/handlers/accounts_test.go create mode 100644 internal/api/router/accounts_test.go diff --git a/README.md b/README.md index 2609e748..24af883c 100644 --- a/README.md +++ b/README.md @@ -133,8 +133,7 @@ broker workers, not connection status or a tunable processing-worker pool). The CORS lists are the options supplied to the middleware; its normal matching normalization still applies. The response excludes credential fields, broker addresses, channel material, database settings and -other configuration. Account operations are not implemented; unknown admin paths -return 404 and unsupported +other configuration. Unknown admin paths return 404 and unsupported methods on the config endpoint return 405 after authentication. Global CORS preflights remain public. Use a long, randomly generated key, keep it out of source control and logs, and send it only in the Authorization header, @@ -158,6 +157,21 @@ auth/credential fields and broker count cannot be changed here; there is no configurable `ingest.worker_count`. Cross-origin admin clients need PUT allowed in the saved CORS methods. CORS controls browser access, not authentication. +Operator accounts are available at `GET/POST /api/v1/admin/accounts` and +`GET/DELETE /api/v1/admin/accounts/{id}`. POST accepts a JSON `name` field in a +body up to 4 KiB; names are trimmed, case-sensitive and limited to 128 Unicode +characters without control characters. Active names are unique. DELETE soft +deactivates the record (204); missing IDs return 404 and an already inactive +record returns 409. A deactivated name may be reused by a new account. +Lists include active and inactive records, newest first, without pagination. +These are operator-defined records; no login, session or API token is created. +Cross-origin account clients need both `POST` and `DELETE` in the saved +`cors.allowed_methods`; the default `GET, HEAD, OPTIONS` is read-only. For an +admin UI that also updates configuration, use `[GET, HEAD, OPTIONS, POST, PUT, +DELETE]`, restrict `cors.allowed_origins` to that UI, and allow `Authorization` +and `Content-Type` headers. Otherwise browser preflight blocks these requests +even when the same bearer-authenticated request works with curl. + ### Environment variables (`.env`) | Variable | Default | Description | diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index b518ab1f..39f1c610 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -18,6 +18,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/db" _ "github.com/MeshCore-Beacon/beacon-server/docs" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/api/router" "github.com/MeshCore-Beacon/beacon-server/internal/background" "github.com/MeshCore-Beacon/beacon-server/internal/cache" @@ -307,6 +308,7 @@ func main() { MaxConnsPerIP: resolved.MaxConnsPerIP, MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, + AdminRoutes: map[string]http.Handler{"/accounts": handlers.AccountsRouter(store)}, }) srv := &http.Server{ diff --git a/config.yaml.example b/config.yaml.example index c32525b7..8f955b36 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -166,7 +166,7 @@ websocket: #cors: # allowed_origins: # - "https://app.example.com" -# allowed_methods: [GET, HEAD, OPTIONS, POST] +# allowed_methods: [GET, HEAD, OPTIONS, POST, PUT, DELETE] # admin UI writes # allowed_headers: [Accept, Authorization, Content-Type] # allow_credentials: false # max_age: 300 diff --git a/docs/docs.go b/docs/docs.go index 7b096572..5973f96a 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -22,6 +22,322 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { + "/admin/accounts": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Includes active and inactive accounts, newest first. These records are not login users.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "List operator accounts", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "post": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Trims the name; names are case-sensitive and unique among active accounts. No credential or login is created.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Create an operator account", + "parameters": [ + { + "description": "Account name (1-128 Unicode characters, no control characters); body at most 4 KiB", + "name": "account", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/accounts/{id}": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns active or inactive account records.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Get an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "delete": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Soft deactivation preserves the record. Its name may be reused by a new account.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Deactivate an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "Deactivated" + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/config": { "get": { "security": [ @@ -2640,6 +2956,37 @@ const docTemplate = `{ } }, "definitions": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Account": { + "type": "object", + "properties": { + "active": { + "type": "boolean" + }, + "created_at": { + "type": "string" + }, + "deactivated_at": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig": { "type": "object", "properties": { @@ -2893,6 +3240,17 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 9803c083..52f700e1 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -20,6 +20,322 @@ "host": "localhost:8080", "basePath": "/api/v1", "paths": { + "/admin/accounts": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Includes active and inactive accounts, newest first. These records are not login users.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "List operator accounts", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "post": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Trims the name; names are case-sensitive and unique among active accounts. No credential or login is created.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Create an operator account", + "parameters": [ + { + "description": "Account name (1-128 Unicode characters, no control characters); body at most 4 KiB", + "name": "account", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "413": { + "description": "Request Entity Too Large", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "415": { + "description": "Unsupported Media Type", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, + "/admin/accounts/{id}": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Returns active or inactive account records.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Get an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, + "delete": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Soft deactivation preserves the record. Its name may be reused by a new account.", + "produces": [ + "application/json" + ], + "tags": [ + "Admin" + ], + "summary": "Deactivate an operator account", + "parameters": [ + { + "type": "string", + "format": "uuid", + "description": "Account UUID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "Deactivated" + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "404": { + "description": "Not Found", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/config": { "get": { "security": [ @@ -2638,6 +2954,37 @@ } }, "definitions": { + "github_com_MeshCore-Beacon_beacon-server_internal_api.Account": { + "type": "object", + "properties": { + "active": { + "type": "boolean" + }, + "created_at": { + "type": "string" + }, + "deactivated_at": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account" + } + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig": { "type": "object", "properties": { @@ -2891,6 +3238,17 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f1d4293b..6ef3079a 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,5 +1,25 @@ basePath: /api/v1 definitions: + github_com_MeshCore-Beacon_beacon-server_internal_api.Account: + properties: + active: + type: boolean + created_at: + type: string + deactivated_at: + type: string + id: + type: string + name: + type: string + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList: + properties: + items: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account' + type: array + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.AdminAuthConfig: properties: configured: @@ -177,6 +197,13 @@ definitions: nodeTypeName: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest: + properties: + name: + type: string + required: + - name + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.CrossIATAHop: properties: fromIata: @@ -1289,6 +1316,215 @@ info: title: MeshCore Beacon API version: 1.6.0 paths: + /admin/accounts: + get: + description: Includes active and inactive accounts, newest first. These records + are not login users. + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.AccountList' + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: List operator accounts + tags: + - Admin + post: + consumes: + - application/json + description: Trims the name; names are case-sensitive and unique among active + accounts. No credential or login is created. + parameters: + - description: Account name (1-128 Unicode characters, no control characters); + body at most 4 KiB + in: body + name: account + required: true + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.CreateAccountRequest' + produces: + - application/json + responses: + "201": + description: Created + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account' + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "409": + description: Conflict + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "413": + description: Request Entity Too Large + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "415": + description: Unsupported Media Type + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Create an operator account + tags: + - Admin + /admin/accounts/{id}: + delete: + description: Soft deactivation preserves the record. Its name may be reused + by a new account. + parameters: + - description: Account UUID + format: uuid + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "204": + description: Deactivated + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "404": + description: Not Found + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "409": + description: Conflict + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Deactivate an operator account + tags: + - Admin + get: + description: Returns active or inactive account records. + parameters: + - description: Account UUID + format: uuid + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.Account' + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "404": + description: Not Found + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Get an operator account + tags: + - Admin /admin/config: get: description: Returns current CORS options, auth configuration status and configured diff --git a/internal/api/account_requests.go b/internal/api/account_requests.go new file mode 100644 index 00000000..f040f980 --- /dev/null +++ b/internal/api/account_requests.go @@ -0,0 +1,8 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +type CreateAccountRequest struct { + Name string `json:"name" validate:"required"` +} diff --git a/internal/api/handlers/accounts.go b/internal/api/handlers/accounts.go new file mode 100644 index 00000000..92c1fc20 --- /dev/null +++ b/internal/api/handlers/accounts.go @@ -0,0 +1,173 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "encoding/json" + "errors" + "io" + "mime" + "net/http" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" + "github.com/google/uuid" +) + +// AccountsRouter must be mounted inside the authenticated admin subtree. +func AccountsRouter(store api.AccountStore) http.Handler { + if store == nil { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { respondError(w, 503, "account storage is unavailable") }) + } + r := chi.NewRouter() + r.Get("/", listAccounts(store)) + r.Post("/", createAccount(store)) + r.Get("/{id}", getAccount(store)) + r.Delete("/{id}", deactivateAccount(store)) + return r +} + +// listAccounts godoc +// @Summary List operator accounts +// @Description Includes active and inactive accounts, newest first. These records are not login users. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Success 200 {object} api.AccountList +// @Failure 401,503,500 {object} map[string]APIError +// @Router /admin/accounts [get] +func listAccounts(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + items, err := store.ListAccounts(r.Context()) + if err != nil { + accountError(w, err) + return + } + if items == nil { + items = []api.Account{} + } + respond(w, 200, api.AccountList{Items: items}) + } +} + +// createAccount godoc +// @Summary Create an operator account +// @Description Trims the name; names are case-sensitive and unique among active accounts. No credential or login is created. +// @Tags Admin +// @Accept json +// @Produce json +// @Security AdminKey +// @Param account body api.CreateAccountRequest true "Account name (1-128 Unicode characters, no control characters); body at most 4 KiB" +// @Success 201 {object} api.Account +// @Failure 400,401,409,413,415,503,500 {object} map[string]APIError +// @Router /admin/accounts [post] +func createAccount(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/json" { + respondError(w, 415, "Content-Type must be application/json") + return + } + r.Body = http.MaxBytesReader(w, r.Body, 4096) + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + var input api.CreateAccountRequest + err = decoder.Decode(&input) + if err == nil { + var extra any + err = decoder.Decode(&extra) + if errors.Is(err, io.EOF) { + err = nil + } else if err == nil { + err = errors.New("multiple JSON values") + } + } + if err != nil { + var sizeError *http.MaxBytesError + if errors.As(err, &sizeError) { + respondError(w, 413, "request body exceeds 4 KiB") + } else { + respondError(w, 400, "invalid account JSON") + } + return + } + name, err := api.NormalizeAccountName(input.Name) + if err != nil { + accountError(w, err) + return + } + account, err := store.CreateAccount(r.Context(), name) + if err != nil { + accountError(w, err) + return + } + respond(w, 201, account) + } +} + +// getAccount godoc +// @Summary Get an operator account +// @Description Returns active or inactive account records. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Param id path string true "Account UUID" format(uuid) +// @Success 200 {object} api.Account +// @Failure 400,401,404,503,500 {object} map[string]APIError +// @Router /admin/accounts/{id} [get] +func getAccount(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id, err := uuid.Parse(chi.URLParam(r, "id")) + if err != nil { + respondError(w, 400, "invalid account ID") + return + } + account, err := store.GetAccount(r.Context(), id) + if err != nil { + accountError(w, err) + return + } + respond(w, 200, account) + } +} + +// deactivateAccount godoc +// @Summary Deactivate an operator account +// @Description Soft deactivation preserves the record. Its name may be reused by a new account. +// @Tags Admin +// @Produce json +// @Security AdminKey +// @Param id path string true "Account UUID" format(uuid) +// @Success 204 "Deactivated" +// @Failure 400,401,404,409,503,500 {object} map[string]APIError +// @Router /admin/accounts/{id} [delete] +func deactivateAccount(store api.AccountStore) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id, err := uuid.Parse(chi.URLParam(r, "id")) + if err != nil { + respondError(w, 400, "invalid account ID") + return + } + if err := store.DeactivateAccount(r.Context(), id); err != nil { + accountError(w, err) + return + } + w.WriteHeader(204) + } +} + +func accountError(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, api.ErrAccountNameInvalid): + respondError(w, 400, api.ErrAccountNameInvalid.Error()) + case errors.Is(err, api.ErrAccountNameConflict): + respondError(w, 409, api.ErrAccountNameConflict.Error()) + case errors.Is(err, api.ErrAccountNotFound): + respondError(w, 404, api.ErrAccountNotFound.Error()) + case errors.Is(err, api.ErrAccountInactive): + respondError(w, 409, api.ErrAccountInactive.Error()) + default: + respondError(w, 500, "internal server error") + } +} diff --git a/internal/api/handlers/accounts_test.go b/internal/api/handlers/accounts_test.go new file mode 100644 index 00000000..d6b96833 --- /dev/null +++ b/internal/api/handlers/accounts_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + mw "github.com/MeshCore-Beacon/beacon-server/internal/api/middleware" + "github.com/google/uuid" +) + +type accountStub struct { + account api.Account + items []api.Account + err error + calls int + name string + id uuid.UUID +} + +func (s *accountStub) CreateAccount(_ context.Context, name string) (api.Account, error) { + s.calls++ + s.name = name + return s.account, s.err +} +func (s *accountStub) ListAccounts(context.Context) ([]api.Account, error) { + s.calls++ + return s.items, s.err +} +func (s *accountStub) GetAccount(_ context.Context, id uuid.UUID) (api.Account, error) { + s.calls++ + s.id = id + return s.account, s.err +} +func (s *accountStub) DeactivateAccount(_ context.Context, id uuid.UUID) error { + s.calls++ + s.id = id + return s.err +} + +func accountRequest(handler http.Handler, method, path, body, contentType, token string) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + return w +} + +func TestAccountInputAndErrors(t *testing.T) { + id := uuid.New() + for _, tc := range []struct { + name, method, path, body, media string + err error + status, calls int + }{ + {"create", "POST", "/", `{"name":" Alice "}`, "application/json", nil, 201, 1}, + {"media parameters", "POST", "/", `{"name":"Alice"}`, "application/json; charset=utf-8", nil, 201, 1}, + {"empty", "POST", "/", `{}`, "application/json", nil, 400, 0}, + {"null", "POST", "/", `null`, "application/json", nil, 400, 0}, + {"wrong field", "POST", "/", `{"name":"Alice","admin":true}`, "application/json", nil, 400, 0}, + {"trailing JSON", "POST", "/", `{"name":"Alice"} {}`, "application/json", nil, 400, 0}, + {"malformed", "POST", "/", `{"name":`, "application/json", nil, 400, 0}, + {"wrong name type", "POST", "/", `{"name":3}`, "application/json", nil, 400, 0}, + {"control", "POST", "/", `{"name":"x\u0000y"}`, "application/json", nil, 400, 0}, + {"oversized", "POST", "/", `{"name":"` + strings.Repeat("a", 5000) + `"}`, "application/json", nil, 413, 0}, + {"oversized trailing space", "POST", "/", `{"name":"Alice"}` + strings.Repeat(" ", 5000), "application/json", nil, 413, 0}, + {"missing media", "POST", "/", `{"name":"Alice"}`, "", nil, 415, 0}, + {"wrong media", "POST", "/", `{"name":"Alice"}`, "text/plain", nil, 415, 0}, + {"conflict", "POST", "/", `{"name":"Alice"}`, "application/json", fmt.Errorf("private-error: %w", api.ErrAccountNameConflict), 409, 1}, + {"get", "GET", "/" + id.String(), "", "", nil, 200, 1}, + {"bad ID", "GET", "/invalid", "", "", nil, 400, 0}, + {"bad delete ID", "DELETE", "/invalid", "", "", nil, 400, 0}, + {"not found", "GET", "/" + id.String(), "", "", api.ErrAccountNotFound, 404, 1}, + {"deactivate", "DELETE", "/" + id.String(), "", "", nil, 204, 1}, + {"inactive", "DELETE", "/" + id.String(), "", "", api.ErrAccountInactive, 409, 1}, + {"delete absent", "DELETE", "/" + id.String(), "", "", api.ErrAccountNotFound, 404, 1}, + {"list database failure", "GET", "/", "", "", errors.New("private-error"), 500, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + store := &accountStub{account: api.Account{ID: id, Name: "Alice", CreatedAt: time.Unix(1, 0).UTC(), Active: true}, err: tc.err} + handler := mw.BearerAuth("test-key", AccountsRouter(store)) + w := accountRequest(handler, tc.method, tc.path, tc.body, tc.media, "test-key") + if w.Code != tc.status || store.calls != tc.calls { + t.Fatalf("status=%d calls=%d", w.Code, store.calls) + } + if w.Header().Get("Cache-Control") != "no-store" || strings.Contains(w.Body.String(), "private-error") { + t.Fatal("cache policy or error disclosure") + } + if tc.status == 204 { + if w.Body.Len() != 0 { + t.Fatal("204 body") + } + return + } + if w.Header().Get("Content-Type") != "application/json" || !json.Valid(w.Body.Bytes()) { + t.Fatal("invalid JSON response") + } + if tc.status == 201 && store.name != "Alice" { + t.Fatal("untrimmed name reached storage") + } + if (tc.method == "DELETE" || tc.method == "GET") && tc.calls > 0 && tc.path != "/" && store.id != id { + t.Fatal("wrong account ID") + } + }) + } +} + +func TestAccountListAndAuthorization(t *testing.T) { + for _, items := range [][]api.Account{nil, {{ID: uuid.New(), Name: "inactive", Active: false}}} { + store := &accountStub{items: items} + handler := mw.BearerAuth("test-key", AccountsRouter(store)) + w := accountRequest(handler, "GET", "/", "", "", "test-key") + var body api.AccountList + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &body) != nil || body.Items == nil || len(body.Items) != len(items) { + t.Fatal("account list shape") + } + } + for _, key := range []string{"", "test-key"} { + store := &accountStub{} + handler := mw.BearerAuth(key, AccountsRouter(store)) + for _, method := range []string{"GET", "POST", "DELETE"} { + w := accountRequest(handler, method, "/", "{}", "application/json", "") + want := 401 + if key == "" { + want = 503 + } + if w.Code != want || store.calls != 0 { + t.Fatal("unauthorized store access") + } + } + } +} diff --git a/internal/api/router/accounts_test.go b/internal/api/router/accounts_test.go new file mode 100644 index 00000000..7cd7fd10 --- /dev/null +++ b/internal/api/router/accounts_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/google/uuid" +) + +type routedAccountStore struct { + api.AccountStore + calls int +} + +func (s *routedAccountStore) CreateAccount(_ context.Context, name string) (api.Account, error) { + s.calls++ + return api.Account{ID: uuid.New(), Name: name, Active: true}, nil +} +func TestAccountRouterUsesProtectedStore(t *testing.T) { + store := &routedAccountStore{} + r := New(nil, nil, nil, Options{Auth: config.AuthConfig{APIKey: "test-key"}, AdminRoutes: map[string]http.Handler{"/accounts": handlers.AccountsRouter(store)}}) + for _, token := range []string{"", "wrong", "test-key"} { + req := httptest.NewRequest("POST", "/api/v1/admin/accounts", strings.NewReader(`{"name":"test"}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + want := 401 + if token == "test-key" { + want = 201 + } + if w.Code != want { + t.Fatalf("route status=%d want=%d", w.Code, want) + } + } + if store.calls != 1 { + t.Fatal("auth boundary was bypassed") + } + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest("POST", "/api/v1/accounts", nil)) + if w.Code != 404 || store.calls != 1 { + t.Fatal("accounts mounted publicly") + } +} From a25e2675c8b29ec62ea93f96507d17f3cdef9972 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 23 Sep 2026 20:13:40 -0400 Subject: [PATCH 046/141] feat(backup): add protected database and config download (#154) --- .build/Dockerfile | 3 +- .github/workflows/ci.yml | 11 ++ README.md | 4 +- cmd/beacon/backup.go | 45 +++++ cmd/beacon/backup_test.go | 58 ++++++ cmd/beacon/main.go | 9 +- config.yaml.example | 6 + docs/backup-export.md | 56 ++++++ docs/docs.go | 94 ++++++++++ docs/swagger.json | 94 ++++++++++ docs/swagger.yaml | 63 +++++++ internal/api/handlers/backup.go | 107 +++++++++++ internal/api/handlers/backup_test.go | 106 +++++++++++ .../api/router/backup_integration_test.go | 172 ++++++++++++++++++ internal/api/router/backup_test.go | 38 ++++ internal/backup/client.go | 40 ++++ internal/backup/client_test.go | 31 ++++ internal/backup/connection.go | 97 ++++++++++ internal/backup/connection_test.go | 48 +++++ internal/backup/export.go | 21 ++- internal/config/config.go | 6 + 21 files changed, 1103 insertions(+), 6 deletions(-) create mode 100644 cmd/beacon/backup.go create mode 100644 cmd/beacon/backup_test.go create mode 100644 internal/api/handlers/backup.go create mode 100644 internal/api/handlers/backup_test.go create mode 100644 internal/api/router/backup_integration_test.go create mode 100644 internal/api/router/backup_test.go create mode 100644 internal/backup/client.go create mode 100644 internal/backup/client_test.go create mode 100644 internal/backup/connection.go create mode 100644 internal/backup/connection_test.go diff --git a/.build/Dockerfile b/.build/Dockerfile index 19e41ba7..7703603a 100644 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -9,7 +9,8 @@ COPY . . RUN go build -o beacon ./cmd/beacon FROM alpine:3.19 -RUN apk add --no-cache ca-certificates tzdata +ARG POSTGRES_CLIENT_MAJOR=16 +RUN apk add --no-cache ca-certificates tzdata postgresql${POSTGRES_CLIENT_MAJOR}-client WORKDIR /app COPY --from=builder /app/beacon . CMD ["./beacon"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0972374f..49b0f24e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,10 +47,16 @@ jobs: - name: Test run: go test ./... + - name: Verify backup startup against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./cmd/beacon -run '^TestBackupConfigurationPostgres$' -count=1 -v + - name: Verify backup command against PostgreSQL 16 run: | CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup.test" ./internal/backup + CGO_ENABLED=0 go test -c -o "$RUNNER_TEMP/backup-http.test" ./internal/api/router docker run --rm --network host \ -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable \ @@ -58,6 +64,11 @@ jobs: -v "$RUNNER_TEMP/beacon-backup:/beacon-backup:ro" \ -v "$RUNNER_TEMP/backup.test:/backup.test:ro" \ postgres:16-alpine /backup.test -test.run '^TestExportPostgres$' -test.v + docker run --rm --network host \ + -e PGHOST=127.0.0.1 -e PGUSER=postgres -e PGDATABASE=postgres \ + -e PGPASSWORD=backup-ci-only -e PGSSLMODE=disable -e BEACON_BACKUP_TEST_POSTGRES=1 \ + -v "$RUNNER_TEMP/backup-http.test:/backup-http.test:ro" \ + postgres:16-alpine /backup-http.test -test.run '^TestBackupDownloadPostgres$' -test.v - name: Install govulncheck run: go install golang.org/x/vuln/cmd/govulncheck@latest diff --git a/README.md b/README.md index 24af883c..becb8bc4 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ in PostgreSQL, and streams live events to WebSocket clients. For deployment instructions including the frontend app, see the deployment docs. For a bounded private database and saved-config bundle, see -[backup export](docs/backup-export.md). This is a standalone export tool; the -backup web interface and import workflow are separate follow-ups. +[backup export](docs/backup-export.md). A standalone export tool is available; the +protected download API is opt-in; browser login and import remain separate follow-ups. --- diff --git a/cmd/beacon/backup.go b/cmd/beacon/backup.go new file mode 100644 index 00000000..ee3bc148 --- /dev/null +++ b/cmd/beacon/backup.go @@ -0,0 +1,45 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/jackc/pgx/v5/pgxpool" +) + +// A failed optional export prerequisite must not stop ingest or the public API. +// Empty options leave the authenticated backup endpoint unavailable (503). +func configureBackup(ctx context.Context, cfg *config.Config, pool *pgxpool.Pool, dsn, configPath string) backup.Options { + if !cfg.Backup.Enabled { + return backup.Options{} + } + if cfg.Auth.APIKey == "" { + slog.Error("backup unavailable: configure an admin bearer key", "component", "backup") + return backup.Options{} + } + service, err := backup.ConnectionService(dsn) + if err != nil { + slog.Error("backup unavailable: POSTGRES_DSN must be a supported single-host PostgreSQL URL; keyword DSNs remain valid for Beacon", "component", "backup") + return backup.Options{} + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + var serverVersion int + if err := pool.QueryRow(ctx, "SELECT current_setting('server_version_num')::integer").Scan(&serverVersion); err != nil { + slog.Error("backup unavailable: could not determine PostgreSQL server version", "component", "backup") + return backup.Options{} + } + if err := backup.CheckClient(ctx, serverVersion); err != nil { + // CheckClient never returns process output or connection settings. + slog.Error("backup unavailable", "component", "backup", "error", err) + return backup.Options{} + } + return backup.Options{ConfigPath: configPath, MaxBytes: backup.DefaultMaxBytes, + Timeout: backup.DefaultTimeout, Version: version, ConnectionService: service} +} diff --git a/cmd/beacon/backup_test.go b/cmd/beacon/backup_test.go new file mode 100644 index 00000000..3a49a44a --- /dev/null +++ b/cmd/beacon/backup_test.go @@ -0,0 +1,58 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestBackupPreconditionsDegradeWithoutDatabaseOrExit(t *testing.T) { + cfg := &config.Config{} + if got := configureBackup(context.Background(), cfg, nil, "", "saved.yaml"); got.ConnectionService != "" { + t.Fatal("disabled backup configured") + } + cfg.Backup.Enabled = true + if got := configureBackup(context.Background(), cfg, nil, "", "saved.yaml"); got.ConnectionService != "" { + t.Fatal("missing admin key configured") + } + cfg.Auth.APIKey = "test-only-key" + if got := configureBackup(context.Background(), cfg, nil, "host=localhost user=test dbname=test", "saved.yaml"); got.ConnectionService != "" { + t.Fatal("keyword connection configured for backup") + } +} + +func TestBackupConfigurationPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" || runtime.GOOS == "windows" { + t.Skip("requires BEACON_TEST_POSTGRES_DSN and a Unix test client") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal("test database configuration unavailable") + } + defer pool.Close() + bin := t.TempDir() + t.Setenv("PATH", bin) + cfg := &config.Config{} + cfg.Backup.Enabled, cfg.Auth.APIKey = true, "test-only-key" + for _, major := range []string{"999", "15"} { + if err := os.WriteFile(filepath.Join(bin, "pg_dump"), []byte("#!/bin/sh\necho 'pg_dump (PostgreSQL) "+major+".0'\n"), 0700); err != nil { + t.Fatal(err) + } + got := configureBackup(ctx, cfg, pool, dsn, "saved.yaml") + if (got.ConnectionService != "") != (major == "999") { + t.Fatalf("unexpected backup availability for test client major %s", major) + } + } +} diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 39f1c610..7c54b7c1 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -125,13 +125,15 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - pool, err := pgxpool.New(ctx, getEnv("POSTGRES_DSN")) + dsn := getEnv("POSTGRES_DSN") + pool, err := pgxpool.New(ctx, dsn) if err != nil { // Parse errors can embed the complete DSN, including its password. slog.Error("invalid PostgreSQL connection configuration; check POSTGRES_DSN", "component", "startup") os.Exit(1) } defer pool.Close() + backupOpts := configureBackup(ctx, cfg, pool, dsn, configPath) if err := db.RunMigrations(ctx, pool); err != nil { slog.Error("migrations failed", "component", "startup", "error", err) @@ -308,7 +310,10 @@ func main() { MaxConnsPerIP: resolved.MaxConnsPerIP, MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, - AdminRoutes: map[string]http.Handler{"/accounts": handlers.AccountsRouter(store)}, + AdminRoutes: map[string]http.Handler{ + "/accounts": handlers.AccountsRouter(store), + "/backup": handlers.BackupRouter(backupOpts), + }, }) srv := &http.Server{ diff --git a/config.yaml.example b/config.yaml.example index 8f955b36..dd7bfac4 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -112,6 +112,12 @@ telemetry: retention: 672h # 4 weeks resolution: 1h # store one snapshot per observer per hour +# Optional authenticated database/config downloads. Disabled unless explicitly enabled. +# Requires auth.api_key, a supported PostgreSQL URL and a compatible pg_dump client. +# A failed prerequisite leaves this feature unavailable; ingest and other APIs continue. +#backup: +# enabled: false + packets: retention: 720h # 30 days diff --git a/docs/backup-export.md b/docs/backup-export.md index 09afb501..160561f6 100644 --- a/docs/backup-export.md +++ b/docs/backup-export.md @@ -31,6 +31,62 @@ without publishing an output. Normal failure, timeout and handled interruption remove temporary files; a power loss or SIGKILL can leave a private `.beacon-backup-*` staging directory for the operator to inspect. +## Protected download API + +Set `backup.enabled: true` in saved YAML and configure the existing admin bearer +key to enable `GET /api/v1/admin/backup`. Supply the key only in the `Authorization: +Bearer ...` header over HTTPS. No query parameters or request body are accepted. +This is an operator API, not account login or a browser admin panel. Keep it off +on public previews that do not need private exports. + +The server requires `pg_dump` on its own PATH when this option is enabled. The +Docker image includes PostgreSQL 16's client by default. Its Alpine 3.19 base does +not provide PostgreSQL 17/18 packages: changing `POSTGRES_CLIENT_MAJOR` alone is +not an upgrade path. For a newer database, supply a runtime with a compatible +client. Startup compares `pg_dump --version` with `server_version_num` and leaves +backup unavailable if the client is missing, unrecognized or older than the server. +Match the source database major version. A client on the host or in a different container is not +sufficient. No Docker socket or host-command bridge is used. + +The download uses the startup `POSTGRES_DSN`, not the standalone export's ambient +`PGDATABASE`. This first adapter supports a single-host `postgres://` or +`postgresql://` URL with an explicit user and database. Passwords, IPv6 and URL +escaping are supported. It copies TLS settings, `passfile`, `connect_timeout`, +`application_name`, `target_session_attrs` and `options`; known `pool_*` controls +are omitted from libpq. Supported PG environment defaults are captured at startup +and URL values override them. Prefer explicit `sslmode=verify-full` and a trusted +`sslrootcert` for remote databases; password-file/TLS paths must be readable in +the same runtime. Their contents are not included in the archive. + +Keyword DSNs, service indirection, multiple hosts, duplicate/unknown query options, +newer protocol negotiation settings and service-file values with line breaks or +edge whitespace are rejected rather than silently changing targets or TLS policy. +Ambient `PGSERVICE`, `PGSERVICEFILE`, `PGSSLNEGOTIATION`, `PGMINPROTOCOLVERSION`, +`PGMAXPROTOCOLVERSION` and `PGTZ` are also unsupported for this adapter. The +standalone command retains native PG* support for those deployments. Errors do +not echo the DSN. Any failed backup prerequisite logs its reason and leaves only +this feature unavailable; Beacon still starts ingest, WebSocket and other APIs. +Fix the prerequisite and restart to enable downloads. Native connection settings live only in a 0600 service file in +private staging; process arguments contain no connection string or credential, +and no request changes Beacon's environment. + +Only one export or transfer is allowed at a time; another request receives 409. +The existing ten-minute/1 GiB SQL/1 MiB YAML limits apply, and disconnecting +cancels an in-progress export. The completed transfer has a ten-minute write +deadline. Temporary files are removed after completion/failure/cancellation; +abrupt process termination can leave private staging. Responses use `no-store` +and a fixed attachment filename. A failed export returns an error without any +partial archive. Authentication missing/invalid is 401, unconfigured access is +503, invalid input is 400, export failure is 500, timeout is 504 and exceeding the +configured export size limit is 507. Other export failures retain sanitized +operator diagnostics in the server log; raw pg_dump stderr remains discarded. + +Provision a private writable temporary directory (`TMPDIR` on Unix) with space +for roughly twice the SQL limit plus overhead. A small read-only-container tmpfs +is insufficient for a large backup. Exports include saved YAML verbatim and may +include keys/message data; the exclusions below still apply. This API does not +implement archive upload, import/overwrite, schedules or remote storage. + ## Format 1 The tar contains exactly three regular files with fixed names: diff --git a/docs/docs.go b/docs/docs.go index 5973f96a..5988c94b 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -338,6 +338,100 @@ const docTemplate = `{ } } }, + "/admin/backup": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint.", + "produces": [ + "application/gzip" + ], + "tags": [ + "Admin" + ], + "summary": "Download a private database and saved-config backup", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + }, + "headers": { + "Content-Disposition": { + "type": "string", + "description": "attachment; filename=beacon-backup.tar.gz" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "504": { + "description": "Gateway Timeout", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "507": { + "description": "Insufficient Storage", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/config": { "get": { "security": [ diff --git a/docs/swagger.json b/docs/swagger.json index 52f700e1..b9a03899 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -336,6 +336,100 @@ } } }, + "/admin/backup": { + "get": { + "security": [ + { + "AdminKey": [] + } + ], + "description": "Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint.", + "produces": [ + "application/gzip" + ], + "tags": [ + "Admin" + ], + "summary": "Download a private database and saved-config backup", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + }, + "headers": { + "Content-Disposition": { + "type": "string", + "description": "attachment; filename=beacon-backup.tar.gz" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "409": { + "description": "Conflict", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "504": { + "description": "Gateway Timeout", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + }, + "507": { + "description": "Insufficient Storage", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + } + }, "/admin/config": { "get": { "security": [ diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6ef3079a..971bc30f 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1525,6 +1525,69 @@ paths: summary: Get an operator account tags: - Admin + /admin/backup: + get: + description: Opt-in operator export. Returns only a completed bundle; one export/transfer + at a time. Contains secrets. Login sessions, external deployment files and + import are outside this endpoint. + produces: + - application/gzip + responses: + "200": + description: OK + headers: + Content-Disposition: + description: attachment; filename=beacon-backup.tar.gz + type: string + schema: + type: file + "400": + description: Bad Request + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "401": + description: Unauthorized + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "409": + description: Conflict + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "500": + description: Internal Server Error + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "503": + description: Service Unavailable + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "504": + description: Gateway Timeout + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + "507": + description: Insufficient Storage + schema: + additionalProperties: + $ref: '#/definitions/internal_api_handlers.APIError' + type: object + security: + - AdminKey: [] + summary: Download a private database and saved-config backup + tags: + - Admin /admin/config: get: description: Returns current CORS options, auth configuration status and configured diff --git a/internal/api/handlers/backup.go b/internal/api/handlers/backup.go new file mode 100644 index 00000000..67f9641d --- /dev/null +++ b/internal/api/handlers/backup.go @@ -0,0 +1,107 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strconv" + "sync/atomic" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/go-chi/chi/v5" +) + +// BackupRouter returns an operator-only subrouter with one bounded export or +// transfer at a time. Its caller must apply the admin authentication middleware. +// The options are startup-owned; the request selects no targets or paths. +func BackupRouter(opts backup.Options) http.Handler { + r := chi.NewRouter() + r.Get("/", backupDownload(opts, backup.Export)) + return r +} + +// backupDownload godoc +// +// @Summary Download a private database and saved-config backup +// @Description Opt-in operator export. Returns only a completed bundle; one export/transfer at a time. Contains secrets. Login sessions, external deployment files and import are outside this endpoint. +// @Tags Admin +// @Produce application/gzip +// @Security AdminKey +// @Success 200 {file} binary +// @Header 200 {string} Content-Disposition "attachment; filename=beacon-backup.tar.gz" +// @Failure 400 {object} map[string]APIError +// @Failure 401 {object} map[string]APIError +// @Failure 409 {object} map[string]APIError +// @Failure 500 {object} map[string]APIError +// @Failure 503 {object} map[string]APIError +// @Failure 504 {object} map[string]APIError +// @Failure 507 {object} map[string]APIError +// @Router /admin/backup [get] +func backupDownload(opts backup.Options, export func(context.Context, backup.Options) error) http.HandlerFunc { + var busy atomic.Bool + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + if opts.ConnectionService == "" { + respondError(w, 503, "backup download is not configured") + return + } + if r.URL.RawQuery != "" || r.ContentLength != 0 || len(r.TransferEncoding) != 0 { + respondError(w, 400, "backup download accepts no query parameters or body") + return + } + if !busy.CompareAndSwap(false, true) { + respondError(w, 409, "a backup download is already in progress") + return + } + defer busy.Store(false) + dir, err := os.MkdirTemp("", "beacon-download-") + if err != nil { + respondError(w, 500, "cannot stage backup") + return + } + defer os.RemoveAll(dir) + requestOpts := opts + requestOpts.OutputPath = filepath.Join(dir, "backup.tar.gz") + if err := export(r.Context(), requestOpts); err != nil { + if errors.Is(err, backup.ErrTooLarge) { + respondError(w, http.StatusInsufficientStorage, "backup exceeds the configured export size limit") + } else if errors.Is(err, context.DeadlineExceeded) { + respondError(w, 504, "backup export timed out") + } else if r.Context().Err() == nil { + // Export discards pg_dump stderr and never returns connection settings. + slog.Error("backup export failed", "component", "backup", "error", err) + respondError(w, 500, "backup export failed; check private client configuration and capacity") + } + return + } + file, err := os.Open(requestOpts.OutputPath) + if err != nil { + respondError(w, 500, "cannot read completed backup") + return + } + defer file.Close() + info, err := file.Stat() + if err != nil { + respondError(w, 500, "cannot inspect completed backup") + return + } + // Keep slow downloads from retaining the only slot and private files + // indefinitely. The native server supports ResponseController deadlines. + controller := http.NewResponseController(w) + _ = controller.SetWriteDeadline(time.Now().Add(backup.DefaultTimeout)) + defer controller.SetWriteDeadline(time.Time{}) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Content-Disposition", `attachment; filename="beacon-backup.tar.gz"`) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Content-Length", strconv.FormatInt(info.Size(), 10)) + _, _ = io.Copy(w, file) + } +} diff --git a/internal/api/handlers/backup_test.go b/internal/api/handlers/backup_test.go new file mode 100644 index 00000000..64c77247 --- /dev/null +++ b/internal/api/handlers/backup_test.go @@ -0,0 +1,106 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" +) + +func TestBackupDownload(t *testing.T) { + opts := backup.Options{ConnectionService: "synthetic-private-service", MaxBytes: 1024, Timeout: time.Second, ConfigPath: "saved.yaml"} + var output string + export := func(ctx context.Context, got backup.Options) error { + if got.ConfigPath != opts.ConfigPath || got.ConnectionService != opts.ConnectionService || got.MaxBytes != opts.MaxBytes || got.Timeout != opts.Timeout { + t.Fatal("startup options changed") + } + output = got.OutputPath + return os.WriteFile(output, []byte("completed fixture"), 0600) + } + handler := backupDownload(opts, export) + for _, query := range []string{"?database=other", "?path=/private", "?command=anything"} { + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup"+query, nil)) + if w.Code != 400 || output != "" { + t.Fatal("caller-selected arguments accepted") + } + } + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", strings.NewReader("body"))) + if w.Code != 400 || output != "" { + t.Fatal("request body accepted") + } + w = httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", nil)) + if w.Code != 200 || w.Body.String() != "completed fixture" || w.Header().Get("Content-Type") != "application/gzip" || + w.Header().Get("Cache-Control") != "no-store" || w.Header().Get("Content-Length") != "17" || w.Header().Get("Content-Disposition") != `attachment; filename="beacon-backup.tar.gz"` { + t.Fatalf("unexpected download: %d %v", w.Code, w.Header()) + } + if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { + t.Fatal("download staging retained") + } + for _, failure := range []error{errors.New("PRIVATE_CANARY"), context.DeadlineExceeded, backup.ErrTooLarge} { + handler = backupDownload(opts, func(_ context.Context, got backup.Options) error { + output = got.OutputPath + return failure + }) + w = httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", nil)) + want := 500 + if errors.Is(failure, context.DeadlineExceeded) { + want = 504 + } + if errors.Is(failure, backup.ErrTooLarge) { + want = 507 + } + if w.Code != want || strings.Contains(w.Body.String(), "PRIVATE_CANARY") || w.Header().Get("Content-Disposition") != "" { + t.Fatal("failure leaked diagnostics or published a download") + } + if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { + t.Fatal("failed staging retained") + } + } + w = httptest.NewRecorder() + BackupRouter(backup.Options{}).ServeHTTP(w, httptest.NewRequest("GET", "/", nil)) + if w.Code != 503 { + t.Fatal("unconfigured download enabled") + } +} + +func TestBackupDownloadConcurrencyAndCancellation(t *testing.T) { + started, finished := make(chan struct{}), make(chan struct{}) + var output string + handler := backupDownload(backup.Options{ConnectionService: "fixture"}, func(ctx context.Context, opts backup.Options) error { + output = opts.OutputPath + close(started) + <-ctx.Done() + return ctx.Err() + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + defer close(finished) + handler(httptest.NewRecorder(), httptest.NewRequest("GET", "/backup", nil).WithContext(ctx)) + }() + <-started + w := httptest.NewRecorder() + handler(w, httptest.NewRequest("GET", "/backup", nil)) + if w.Code != http.StatusConflict { + t.Fatal("concurrent export started") + } + cancel() + <-finished + if _, err := os.Stat(filepath.Dir(output)); !os.IsNotExist(err) { + t.Fatal("cancelled staging retained") + } +} diff --git a/internal/api/router/backup_integration_test.go b/internal/api/router/backup_integration_test.go new file mode 100644 index 00000000..33965a4b --- /dev/null +++ b/internal/api/router/backup_integration_test.go @@ -0,0 +1,172 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// This opt-in test only creates/restores/drops its own randomly named databases. +func TestBackupDownloadPostgres(t *testing.T) { + if os.Getenv("BEACON_BACKUP_TEST_POSTGRES") != "1" { + t.Skip("requires pg_dump/psql and an explicitly selected private PostgreSQL test server") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + admin, err := pgx.Connect(ctx, "") + if err != nil { + t.Fatal("cannot connect to private PostgreSQL") + } + defer admin.Close(context.Background()) + sourceName, targetName := "beacon_download_"+strings.ToLower(rand.Text()), "beacon_download_"+strings.ToLower(rand.Text()) + for _, name := range []string{sourceName, targetName} { + ident := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil { + t.Fatal("cannot create private fixture") + } + defer func() { + if _, err := admin.Exec(ctx, "DROP DATABASE "+ident+" WITH (FORCE)"); err != nil { + t.Error("cannot remove private fixture") + } + }() + } + cfg, err := pgxpool.ParseConfig("") + if err != nil { + t.Fatal("invalid fixture configuration") + } + cfg.ConnConfig.Database = sourceName + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal("cannot open source fixture") + } + defer pool.Close() + if err := db.RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, "CREATE TABLE download_fixture (id bigint GENERATED ALWAYS AS IDENTITY, message text); INSERT INTO download_fixture(message) VALUES ('café 雪')"); err != nil { + t.Fatal(err) + } + dsn := (&url.URL{Scheme: "postgres", Host: net.JoinHostPort(cfg.ConnConfig.Host, strconv.Itoa(int(cfg.ConnConfig.Port))), + User: url.UserPassword(cfg.ConnConfig.User, cfg.ConnConfig.Password), Path: "/" + sourceName, + RawQuery: "sslmode=disable&pool_max_conns=2"}).String() + service, err := backup.ConnectionService(dsn) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + configPath := filepath.Join(dir, "config.yaml") + saved := []byte("# verbatim saved file\nbackup:\n enabled: true\n") + if err := os.WriteFile(configPath, saved, 0600); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", dir) + server := httptest.NewServer(New(nil, nil, nil, Options{ + Auth: config.AuthConfig{APIKey: "fixture-key"}, + AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ + ConnectionService: service, ConfigPath: configPath, MaxBytes: 16 << 20, Timeout: 30 * time.Second, Version: "http-fixture", + })}, + })) + defer server.Close() + request, _ := http.NewRequestWithContext(ctx, "GET", server.URL+"/api/v1/admin/backup", nil) + request.Header.Set("Authorization", "Bearer fixture-key") + response, err := server.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != 200 || response.Header.Get("Cache-Control") != "no-store" { + t.Fatalf("download status=%d", response.StatusCode) + } + gz, err := gzip.NewReader(response.Body) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tarReader, members := tar.NewReader(gz), map[string][]byte{} + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil || header.Typeflag != tar.TypeReg || header.Mode != 0600 { + t.Fatal("invalid member") + } + members[header.Name], err = io.ReadAll(tarReader) + if err != nil { + t.Fatal(err) + } + } + if _, err := io.Copy(io.Discard, gz); err != nil { + t.Fatal(err) + } + var manifest backup.Manifest + if json.Unmarshal(members["manifest.json"], &manifest) != nil || len(members) != 3 || manifest.FormatVersion != 1 || !bytes.Equal(members["config.yaml"], saved) { + t.Fatal("invalid manifest/config") + } + for _, file := range manifest.Files { + sum := sha256.Sum256(members[file.Name]) + if file.Size != int64(len(members[file.Name])) || file.SHA256 != hex.EncodeToString(sum[:]) { + t.Fatal("checksum mismatch") + } + } + if bytes.Contains(members["database.sql"], []byte("beacon_backup]")) { + t.Fatal("connection settings leaked into bundle") + } + restore := exec.CommandContext(ctx, "psql", "-X", "--set=ON_ERROR_STOP=on", "--single-transaction") + restore.Env = append(os.Environ(), "PGDATABASE="+targetName) + restore.Stdin = bytes.NewReader(members["database.sql"]) + if err := restore.Run(); err != nil { + t.Fatal("cannot restore downloaded SQL into empty fixture") + } + targetCfg := admin.Config().Copy() + targetCfg.Database = targetName + target, err := pgx.ConnectConfig(ctx, targetCfg) + if err != nil { + t.Fatal("cannot connect to restored fixture") + } + defer target.Close(context.Background()) + const fingerprint = `SELECT md5(string_agg(table_name||column_name||data_type, ',' ORDER BY table_name,ordinal_position)) FROM information_schema.columns WHERE table_schema='public'` + var before, after, message string + if pool.QueryRow(ctx, fingerprint).Scan(&before) != nil || target.QueryRow(ctx, fingerprint).Scan(&after) != nil || before != after { + t.Fatal("schema differs after restore") + } + if target.QueryRow(ctx, "SELECT message FROM download_fixture").Scan(&message) != nil || message != "café 雪" { + t.Fatal("wrong database or changed data") + } + var nextID int + if target.QueryRow(ctx, "INSERT INTO download_fixture(message) VALUES ('next') RETURNING id").Scan(&nextID) != nil || nextID != 2 { + t.Fatal("identity sequence changed") + } + paths, _ := filepath.Glob(filepath.Join(dir, "beacon-download-*")) + if len(paths) != 0 { + t.Fatal("private HTTP staging retained") + } + t.Log("HTTP bundle restored current Beacon schema and Unicode data; native client ignored ambient PGDATABASE") +} diff --git a/internal/api/router/backup_test.go b/internal/api/router/backup_test.go new file mode 100644 index 00000000..76bdf6e7 --- /dev/null +++ b/internal/api/router/backup_test.go @@ -0,0 +1,38 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package router + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/backup" + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +func TestBackupAuthenticationBeforeExport(t *testing.T) { + for _, key := range []string{"", "fixture-key"} { + handler := New(nil, nil, nil, Options{Auth: config.AuthConfig{APIKey: key}, AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ConnectionService: "fixture"})}}) + for _, token := range []string{"", "wrong", "fixture-key"} { + r := httptest.NewRequest("GET", "/api/v1/admin/backup?database=other", nil) + if token != "" { + r.Header.Set("Authorization", "Bearer "+token) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + want := 503 + if key != "" { + want = 401 + if token == key { + want = 400 + } + } + if w.Code != want || w.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("status=%d want=%d", w.Code, want) + } + } + } +} diff --git a/internal/backup/client.go b/internal/backup/client.go new file mode 100644 index 00000000..55e6f975 --- /dev/null +++ b/internal/backup/client.go @@ -0,0 +1,40 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "context" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" +) + +var dumpVersion = regexp.MustCompile(`^pg_dump \(PostgreSQL\) ([0-9]+)(?:\.[0-9]+)+(?:[ \r\n]|$)`) + +// CheckClient checks the client inside this runtime before enabling downloads. +// pg_dump cannot export a server newer than its own major version. +func CheckClient(ctx context.Context, serverVersion int) error { + output, err := exec.CommandContext(ctx, "pg_dump", "--version").Output() + if err != nil { + return errors.New("cannot run pg_dump --version; install the PostgreSQL client in this runtime") + } + return checkClientVersion(output, serverVersion) +} + +func checkClientVersion(output []byte, serverVersion int) error { + match := dumpVersion.FindSubmatch(output) + if match == nil || serverVersion < 100000 { + return errors.New("cannot verify pg_dump and PostgreSQL server major versions") + } + client, err := strconv.Atoi(string(match[1])) + if err != nil { + return errors.New("cannot verify pg_dump major version") + } + if client < serverVersion/10000 { + return fmt.Errorf("pg_dump major %d is older than PostgreSQL server major %d; install a compatible client and restart Beacon", client, serverVersion/10000) + } + return nil +} diff --git a/internal/backup/client_test.go b/internal/backup/client_test.go new file mode 100644 index 00000000..605921dd --- /dev/null +++ b/internal/backup/client_test.go @@ -0,0 +1,31 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "strings" + "testing" +) + +func TestCheckClientVersion(t *testing.T) { + for _, tc := range []struct { + output string + server int + valid bool + }{ + {"pg_dump (PostgreSQL) 16.10\n", 160010, true}, + {"pg_dump (PostgreSQL) 17.6 (Debian 17.6)\n", 160010, true}, + {"pg_dump (PostgreSQL) 16.10\n", 170006, false}, + {"pg_dump (PostgreSQL) 16.10\n", 180000, false}, + {"pg_dump (PostgreSQL) 18.0\n", 180000, true}, + {"PRIVATE_CANARY", 160010, false}, + {"pg_dump (PostgreSQL) 16.bad", 160010, false}, + {"pg_dump (PostgreSQL) 16.10\n", 0, false}, + } { + err := checkClientVersion([]byte(tc.output), tc.server) + if (err == nil) != tc.valid || (err != nil && strings.Contains(err.Error(), "PRIVATE_CANARY")) { + t.Fatalf("server %d: %v", tc.server, err) + } + } +} diff --git a/internal/backup/connection.go b/internal/backup/connection.go new file mode 100644 index 00000000..59fc2fbc --- /dev/null +++ b/internal/backup/connection.go @@ -0,0 +1,97 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "errors" + "net/url" + "os" + "sort" + "strconv" + "strings" + "unicode" +) + +// ConnectionService captures a single-host PostgreSQL URL for pg_dump. The +// result contains secrets and must only be written to private staging, never +// logged or passed as a process argument. Unsupported settings fail closed. +func ConnectionService(dsn string) (string, error) { + return connectionService(dsn, os.Environ()) +} + +func connectionService(dsn string, environ []string) (string, error) { + invalid := errors.New("backup requires a single-host PostgreSQL URL with explicit user and database and supported libpq settings") + u, err := url.Parse(dsn) + if err != nil || (u.Scheme != "postgres" && u.Scheme != "postgresql") || u.User == nil || u.User.Username() == "" || + u.Hostname() == "" || strings.Contains(u.Hostname(), ",") || strings.TrimLeft(u.Path, "/") == "" || u.Fragment != "" { + return "", invalid + } + settings := map[string]string{"host": u.Hostname(), "port": "5432", "user": u.User.Username(), + "dbname": strings.TrimLeft(u.Path, "/"), "sslmode": "prefer", "gssencmode": "disable"} + // Match the pgx startup environment for the supported native client settings. + // Target fields come exclusively from the URL. Service indirection and newer + // protocol options need separate compatibility work, not a guessed translation. + envKeys := map[string]string{"PGPASSWORD": "password", "PGPASSFILE": "passfile", "PGAPPNAME": "application_name", + "PGCONNECT_TIMEOUT": "connect_timeout", "PGSSLMODE": "sslmode", "PGSSLKEY": "sslkey", "PGSSLCERT": "sslcert", + "PGSSLROOTCERT": "sslrootcert", "PGSSLPASSWORD": "sslpassword", "PGSSLSNI": "sslsni", + "PGTARGETSESSIONATTRS": "target_session_attrs", "PGOPTIONS": "options"} + for _, entry := range environ { + key, value, _ := strings.Cut(entry, "=") + if value == "" { + continue + } + switch key { + case "PGSERVICE", "PGSERVICEFILE", "PGSSLNEGOTIATION", "PGMINPROTOCOLVERSION", "PGMAXPROTOCOLVERSION", "PGTZ": + return "", invalid + } + if name := envKeys[key]; name != "" { + settings[name] = value + } + if key == "PGPORT" && u.Port() == "" { + settings["port"] = value + } + } + if port := u.Port(); port != "" { + settings["port"] = port + } + if port, err := strconv.Atoi(settings["port"]); err != nil || port < 1 || port > 65535 { + return "", invalid + } + if password, ok := u.User.Password(); ok { + settings["password"] = password + } + query, err := url.ParseQuery(u.RawQuery) + if err != nil { + return "", invalid + } + for key, values := range query { + if len(values) != 1 || values[0] == "" { + return "", invalid + } + switch key { + case "pool_max_conns", "pool_min_conns", "pool_min_idle_conns", "pool_max_conn_lifetime", "pool_max_conn_idle_time", "pool_health_check_period", "pool_max_conn_lifetime_jitter": + continue // pgx-only pool controls; main has already validated them. + case "sslmode", "sslcert", "sslkey", "sslrootcert", "sslpassword", "sslsni", "passfile", "connect_timeout", "application_name", "target_session_attrs", "options": + settings[key] = values[0] + default: + return "", invalid + } + } + var result strings.Builder + result.WriteString("[beacon_backup]\n") + keys := make([]string, 0, len(settings)) + for key, value := range settings { + // libpq service files have line-based, unquoted values. Reject anything + // their parser would trim or interpret as another entry. + if strings.TrimSpace(value) != value || strings.IndexFunc(value, unicode.IsControl) >= 0 || len(value) > 1000 { + return "", invalid + } + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + result.WriteString(key + "=" + settings[key] + "\n") + } + return result.String(), nil +} diff --git a/internal/backup/connection_test.go b/internal/backup/connection_test.go new file mode 100644 index 00000000..e3c0b118 --- /dev/null +++ b/internal/backup/connection_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "strings" + "testing" +) + +func TestConnectionService(t *testing.T) { + service, err := connectionService("postgresql://user:p%23ass%3Dword@[::1]:5544/db%20name?sslmode=verify-full&sslrootcert=%2Fprivate%2Fca.pem&passfile=%2Fprivate%2Fpass&pool_max_conns=2", []string{ + "PGHOST=wrong", "PGDATABASE=wrong", "PGUSER=wrong", "PGPASSWORD=wrong", "PGSSLMODE=disable", "PGPASSFILE=/wrong", "PGSSLKEY=/private/key.pem"}) + if err != nil { + t.Fatal(err) + } + for _, line := range []string{"[beacon_backup]", "host=::1", "port=5544", "user=user", "password=p#ass=word", "dbname=db name", + "sslmode=verify-full", "sslrootcert=/private/ca.pem", "passfile=/private/pass", "sslkey=/private/key.pem"} { + if !strings.Contains(service, line+"\n") { + t.Fatalf("missing expected setting %q", line) + } + } + if strings.Contains(service, "wrong") || strings.Contains(service, "pool_") { + t.Fatal("unrelated target or pool settings leaked into libpq") + } + service, err = connectionService("postgres://user@host/db", []string{"PGPASSWORD=from-environment", "PGPORT=5555", "PGSSLMODE=require"}) + if err != nil || !strings.Contains(service, "password=from-environment\n") || !strings.Contains(service, "port=5555\n") || !strings.Contains(service, "sslmode=require\n") { + t.Fatal("supported environment defaults were not captured") + } +} + +func TestConnectionServiceRejectsAmbiguousSettings(t *testing.T) { + for _, dsn := range []string{ + "host=host user=user dbname=db", "postgres://host/db", "postgres://user@host/", "postgres://user@a,b/db", + "postgres://user@host/db?service=other", "postgres://user@host/db?host=other", "postgres://user@host/db?sslmode=require&sslmode=disable", + "postgres://user@host/db?unknown=x", "postgres://user:secret%0Ahost%3Dother@host/db", "postgres://user:%20secret@host/db", + "postgres://user@host:99999/db", "postgres://user@host/db#fragment", "postgres://user@host/db?sslnegotiation=direct", + } { + if _, err := connectionService(dsn, nil); err == nil || strings.Contains(err.Error(), "secret") { + t.Fatal("unsupported connection must fail without echoing input") + } + } + for _, env := range []string{"PGSERVICE=other", "PGSERVICEFILE=/private", "PGSSLNEGOTIATION=direct", "PGMINPROTOCOLVERSION=3.0", "PGTZ=UTC", "PGPORT=5432,5433"} { + if _, err := connectionService("postgres://user@host/db", []string{env}); err == nil { + t.Fatal("unsupported ambient connection settings accepted") + } + } +} diff --git a/internal/backup/export.go b/internal/backup/export.go index 63abf68c..e7b1a56d 100644 --- a/internal/backup/export.go +++ b/internal/backup/export.go @@ -17,6 +17,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" ) @@ -36,6 +37,9 @@ type Options struct { MaxBytes int64 // Maximum uncompressed database dump size. Timeout time.Duration Version string + // ConnectionService overrides PG* using a private service file. Construct it + // with ConnectionService at startup; empty retains the standalone PG* mode. + ConnectionService string } // Manifest describes format 1; hashes cover the uncompressed archive members. @@ -72,7 +76,7 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex if opts.ConfigPath == "" || opts.OutputPath == "" || opts.MaxBytes <= 0 || opts.MaxBytes > 1<<40 || opts.Timeout <= 0 { return errors.New("config, output, positive timeout and max-bytes (at most 1 TiB) are required") } - if os.Getenv("PGDATABASE") == "" { + if opts.ConnectionService == "" && os.Getenv("PGDATABASE") == "" { return errors.New("PGDATABASE must explicitly name the database; POSTGRES_DSN is not used") } ctx, cancel := context.WithTimeout(ctx, opts.Timeout) @@ -104,6 +108,21 @@ func export(ctx context.Context, opts Options, command func(context.Context) *ex hash := sha256.New() output := &limitedWriter{ctx: ctx, cancel: cancel, dst: io.MultiWriter(dump, hash), remaining: opts.MaxBytes} cmd := command(ctx) + if opts.ConnectionService != "" { + servicePath := filepath.Join(temp, "pg_service.conf") + if err := os.WriteFile(servicePath, []byte(opts.ConnectionService), 0600); err != nil { + return errors.New("cannot prepare private backup connection") + } + // Do not let unrelated PG* values redirect the client, and never mutate + // the Beacon process environment while another request is running. + cmd.Env = nil + for _, entry := range os.Environ() { + if !strings.HasPrefix(strings.ToUpper(entry), "PG") { + cmd.Env = append(cmd.Env, entry) + } + } + cmd.Env = append(cmd.Env, "PGSERVICEFILE="+servicePath, "PGSERVICE=beacon_backup") + } cmd.Stdout, cmd.Stderr = output, io.Discard cmd.WaitDelay = time.Second err = cmd.Run() diff --git a/internal/config/config.go b/internal/config/config.go index b5fc09a2..61786364 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -18,6 +18,7 @@ import ( // Config is the top-level structure of the Beacon config file. type Config struct { Auth AuthConfig `yaml:"auth"` + Backup BackupConfig `yaml:"backup"` Log LogConfig `yaml:"log"` Server ServerConfig `yaml:"server"` IATAs map[string]IATAConfig `yaml:"iatas"` @@ -38,6 +39,11 @@ type Config struct { Observers ObserversConfig `yaml:"observers"` } +// BackupConfig enables protected downloads. Disabled by default. +type BackupConfig struct { + Enabled bool `yaml:"enabled"` +} + // AuthConfig holds the operator key for the protected admin subtree. // The key is excluded from JSON; it must not be exposed by configuration APIs. type AuthConfig struct { From 6107578c25489a5df78313027e3557fc61c21afe Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 23 Sep 2026 20:16:22 -0400 Subject: [PATCH 047/141] feat(stats): add bounded reception signal analytics (#157) --- .github/workflows/ci.yml | 5 + db/migrations/035_mv_signal_stats.sql | 33 +++++ db/queries/signal.sql | 31 +++++ db/signal.go | 72 ++++++++++ db/signal_integration_test.go | 153 ++++++++++++++++++++++ db/signal_test.go | 21 +++ db/sqlc/mock/querier.go | 29 ++++ db/sqlc/models.go | 13 ++ db/sqlc/querier.go | 4 + db/sqlc/signal.sql.go | 99 ++++++++++++++ db/stats_migration_integration_test.go | 33 +++++ docs/docs.go | 152 +++++++++++++++++++++ docs/swagger.json | 152 +++++++++++++++++++++ docs/swagger.yaml | 108 +++++++++++++++ internal/api/handlers/regions.go | 19 +++ internal/api/handlers/signal.go | 68 ++++++++++ internal/api/handlers/signal_test.go | 113 ++++++++++++++++ internal/api/handlers/stats.go | 1 + internal/api/handlers/stub_reader_test.go | 4 + internal/api/reader.go | 3 + internal/api/signal.go | 42 ++++++ internal/background/tasks.go | 2 + internal/background/tasks_test.go | 9 +- internal/cache/cache_test.go | 4 + internal/cache/reader.go | 1 + internal/cache/signal.go | 25 ++++ internal/cache/signal_test.go | 48 +++++++ 27 files changed, 1242 insertions(+), 2 deletions(-) create mode 100644 db/migrations/035_mv_signal_stats.sql create mode 100644 db/queries/signal.sql create mode 100644 db/signal.go create mode 100644 db/signal_integration_test.go create mode 100644 db/signal_test.go create mode 100644 db/sqlc/signal.sql.go create mode 100644 db/stats_migration_integration_test.go create mode 100644 internal/api/handlers/signal.go create mode 100644 internal/api/handlers/signal_test.go create mode 100644 internal/api/signal.go create mode 100644 internal/cache/signal.go create mode 100644 internal/cache/signal_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49b0f24e..9b6b62a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,11 @@ jobs: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable run: go test ./cmd/beacon -run '^TestBackupConfigurationPostgres$' -count=1 -v + - name: Verify signal aggregates against PostgreSQL 16 + env: + BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable + run: go test ./db -run '^TestSignalPostgres$' -count=1 -v + - name: Verify backup command against PostgreSQL 16 run: | CGO_ENABLED=0 go build -o "$RUNNER_TEMP/beacon-backup" ./cmd/beacon-backup diff --git a/db/migrations/035_mv_signal_stats.sql b/db/migrations/035_mv_signal_stats.sql new file mode 100644 index 00000000..14b3dd3f --- /dev/null +++ b/db/migrations/035_mv_signal_stats.sql @@ -0,0 +1,33 @@ +-- Reception-IATA rollups include legacy NULL payload types and exclude non-finite +-- SNR. mv_observer_activity_hourly has a different grain/filter, so reusing its +-- means would disagree with these histograms and the reception coverage counts. +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_signal_stats_hourly AS +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_signal_stats_hourly + ON mv_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin); +CREATE INDEX IF NOT EXISTS idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly (hour); diff --git a/db/queries/signal.sql b/db/queries/signal.sql new file mode 100644 index 00000000..24bb800c --- /dev/null +++ b/db/queries/signal.sql @@ -0,0 +1,31 @@ +-- name: GetSignalStats :many +-- Read compact hourly snapshots, never observations on an HTTP request. +-- Weight averages by sample counts instead of averaging regional/hourly means. +WITH readings AS ( + SELECT * FROM mv_signal_stats_hourly + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz + AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT * FROM mv_signal_stats_hourly + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz + AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) +) +SELECT (3 + 4 * grouping(hour))::integer AS kind, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + -1::integer AS snr_bin, -1::integer AS rssi_bin, + COALESCE(sum(receptions), 0)::bigint AS receptions, + COALESCE(sum(snr_samples), 0)::bigint AS snr_samples, + COALESCE(sum(snr_sum) / NULLIF(sum(snr_samples), 0), 0)::double precision AS snr_average, + COALESCE(sum(rssi_samples), 0)::bigint AS rssi_samples, + COALESCE(sum(rssi_sum) / NULLIF(sum(rssi_samples), 0), 0)::double precision AS rssi_average +FROM readings WHERE kind = 3 +GROUP BY GROUPING SETS ((), (hour)) +UNION ALL +SELECT (kind + 4)::integer, 'epoch'::timestamptz, snr_bin, rssi_bin, + sum(receptions)::bigint, 0::bigint, 0::double precision, 0::bigint, 0::double precision +FROM readings WHERE kind IN (1, 2) +GROUP BY kind, snr_bin, rssi_bin +ORDER BY kind, hour, snr_bin, rssi_bin; + +-- name: RefreshSignalStats :exec +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_signal_stats_hourly; diff --git a/db/signal.go b/db/signal.go new file mode 100644 index 00000000..9a7b1a66 --- /dev/null +++ b/db/signal.go @@ -0,0 +1,72 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + rows, err := s.q.GetSignalStats(ctx, sqlc.GetSignalStatsParams{ + Since: pgtype.Timestamptz{Time: since, Valid: true}, Until: pgtype.Timestamptz{Time: until, Valid: true}, Iatas: iatas, + }) + if err != nil { + return nil, err + } + stats := &api.SignalStats{Since: since.UnixMilli(), Until: until.UnixMilli(), + SNR: api.SignalMetric{Histogram: signalBins(-30, 5, 12)}, RSSI: api.SignalMetric{Histogram: signalBins(-140, 10, 14)}, Hourly: []api.SignalHour{}} + for _, row := range rows { + // GROUPING bits distinguish real null buckets from dimensions not in the set. + switch row.Kind { + case 7: + stats.Receptions = row.Receptions + stats.SNR.Samples, stats.SNR.Average = row.SnrSamples, signalAverage(row.SnrAverage, row.SnrSamples) + stats.RSSI.Samples, stats.RSSI.Average = row.RssiSamples, signalAverage(row.RssiAverage, row.RssiSamples) + case 3: + stats.Hourly = append(stats.Hourly, api.SignalHour{Hour: row.Hour.Time.UnixMilli(), Receptions: row.Receptions, + SNRSamples: row.SnrSamples, SNRAverage: signalAverage(row.SnrAverage, row.SnrSamples), + RSSISamples: row.RssiSamples, RSSIAverage: signalAverage(row.RssiAverage, row.RssiSamples)}) + case 5: + if row.SnrBin >= 0 && int(row.SnrBin) < len(stats.SNR.Histogram) { + stats.SNR.Histogram[row.SnrBin].Count = row.Receptions + } + case 6: + if row.RssiBin >= 0 && int(row.RssiBin) < len(stats.RSSI.Histogram) { + stats.RSSI.Histogram[row.RssiBin].Count = row.Receptions + } + } + } + return stats, nil +} + +func (s *Store) RefreshSignalStats(ctx context.Context) error { + return s.q.RefreshSignalStats(ctx) +} + +func signalAverage(value float64, count int64) *float64 { + if count == 0 { + return nil + } + return &value +} + +func signalBins(lower, width float64, count int) []api.SignalBin { + bins := make([]api.SignalBin, count+2) + for i := range bins { + if i > 0 { + v := lower + float64(i-1)*width + bins[i].Lower = &v + } + if i <= count { + v := lower + float64(i)*width + bins[i].Upper = &v + } + } + return bins +} diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go new file mode 100644 index 00000000..7eef6bbf --- /dev/null +++ b/db/signal_integration_test.go @@ -0,0 +1,153 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http/httptest" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/jackc/pgx/v5" +) + +// Real view DDL and fixture rows are isolated in a rolled-back private schema. +func TestSignalPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + isolateStatsSchema(t, ctx, tx) + _, err = tx.Exec(ctx, ` +SET LOCAL TIME ZONE 'America/Vancouver'; +CREATE TABLE packet_observations (heard_at timestamptz NOT NULL, iata char(3) NOT NULL, snr real, rssi smallint); +INSERT INTO packet_observations +SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FROM (VALUES + ('00:30:00','YVR','0',-100), ('00:45:00','YVR','0',0), ('00:59:00','YVR',NULL,NULL), + ('01:00:00','YVR','-30',-140), ('01:10:00','YVR','-30.25',-141), ('01:20:00','YVR','30',0), + ('01:30:00','YVR','5',NULL), ('01:40:00','YVR',NULL,-90), ('01:50:00','YVR','NaN',-80), + ('02:00:00','YVR','Infinity',-70), ('02:10:00','YVR','-Infinity',-60), + ('02:20:00','YYJ','10',-50), ('03:00:00','YYZ','-5',-110), + ('04:00:00','YVR','20',-40), ('04:30:00','YVR','20',-40), + ('03:10:00','YVR','0',NULL), ('03:20:00','YVR',NULL,0) +) v(at,iata,snr,rssi);`) + if err != nil { + t.Fatal(err) + } + since := time.Now().UTC().Truncate(24 * time.Hour).Add(-48 * time.Hour) + if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { + t.Fatal(err) + } + applyStatsMigration(t, ctx, tx, "035_mv_signal_stats.sql") + applyStatsMigration(t, ctx, tx, "035_mv_signal_stats.sql") // interrupted journal retry + store := &Store{q: sqlc.New(tx)} + until := since.Add(4 * time.Hour) + for _, tc := range []struct { + name string + iatas []string + receptions, snr, rssi int64 + }{ + {"global", nil, 15, 7, 10}, {"empty filter", []string{}, 15, 7, 10}, {"YVR", []string{"YVR"}, 13, 5, 8}, + {"duplicate/overlapping IATAs", []string{"YVR", "YYJ", "YVR"}, 14, 6, 9}, + {"unknown IATA", []string{"ZZZ"}, 0, 0, 0}, {"empty region", []string{""}, 0, 0, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := store.GetSignalStats(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + if got.Receptions != tc.receptions || got.SNR.Samples != tc.snr || got.RSSI.Samples != tc.rssi { + t.Fatalf("counts: %+v", got) + } + for _, metric := range []api.SignalMetric{got.SNR, got.RSSI} { + var total int64 + for _, bin := range metric.Histogram { + total += bin.Count + } + if total != metric.Samples || (metric.Average == nil) != (metric.Samples == 0) { + t.Fatalf("histogram/average: %+v", metric) + } + } + var receptions, snr, rssi int64 + for i, hour := range got.Hourly { + if hour.Hour%3600000 != 0 || hour.Hour < since.Truncate(time.Hour).UnixMilli() || hour.Hour >= until.UnixMilli() || (i > 0 && hour.Hour <= got.Hourly[i-1].Hour) { + t.Fatalf("UTC order/window: %+v", got.Hourly) + } + receptions += hour.Receptions + snr += hour.SNRSamples + rssi += hour.RSSISamples + } + if receptions != got.Receptions || snr != got.SNR.Samples || rssi != got.RSSI.Samples { + t.Fatal("hourly totals differ") + } + if _, err := json.Marshal(got); err != nil { + t.Fatal(err) + } + if tc.name == "YVR" { + if math.Abs(*got.SNR.Average-(-5.05)) > 1e-9 || *got.RSSI.Average != -85.125 { + t.Fatalf("unweighted/missing readings: %+v", got) + } + if got.SNR.Histogram[0].Count != 1 || got.SNR.Histogram[1].Count != 1 || got.SNR.Histogram[7].Count != 1 || got.SNR.Histogram[8].Count != 1 || got.SNR.Histogram[13].Count != 1 { + t.Fatalf("SNR boundaries/zero: %+v", got.SNR.Histogram) + } + if got.RSSI.Histogram[0].Count != 1 || got.RSSI.Histogram[1].Count != 1 || got.RSSI.Histogram[15].Count != 1 { + t.Fatalf("RSSI boundaries: %+v", got.RSSI.Histogram) + } + last := got.Hourly[len(got.Hourly)-1] + if last.Receptions != 2 || last.SNRAverage != nil || last.RSSIAverage != nil { + t.Fatalf("missing-only hour: %+v", last) + } + } + }) + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err = tx.Exec(ctx, "SET LOCAL plan_cache_mode="+mode); err != nil { + t.Fatal(err) + } + got, err := store.GetSignalStats(ctx, since, until, []string{"YVR"}) + if err != nil || got.Receptions != 13 || got.SNR.Samples != 5 || got.RSSI.Samples != 8 { + t.Fatalf("%s: %+v %v", mode, got, err) + } + } + w := httptest.NewRecorder() + request := httptest.NewRequest("GET", fmt.Sprintf("/signal?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx) + handlers.StatsRouter(store).ServeHTTP(w, request) + var response api.SignalStats + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 13 || response.SNR.Samples != 5 { + t.Fatalf("PostgreSQL HTTP response: status=%d body=%s error=%v", w.Code, w.Body.String(), err) + } + if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { + t.Fatal(err) + } + stale, err := store.GetSignalStats(ctx, since, until, nil) + if err != nil || stale.Receptions != 15 { + t.Fatalf("snapshot lost: %+v %v", stale, err) + } + if err := store.RefreshSignalStats(ctx); err != nil { + t.Fatal(err) + } + got, err := store.GetSignalStats(ctx, since, until, nil) + if err != nil || got.Receptions != 0 || got.Hourly == nil || len(got.Hourly) != 0 || got.SNR.Average != nil { + t.Fatalf("empty: %+v %v", got, err) + } +} diff --git a/db/signal_test.go b/db/signal_test.go new file mode 100644 index 00000000..35b24f69 --- /dev/null +++ b/db/signal_test.go @@ -0,0 +1,21 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import "testing" + +func TestSignalBinsAndMissingAverage(t *testing.T) { + bins := signalBins(-30, 5, 12) + if len(bins) != 14 || bins[0].Lower != nil || *bins[0].Upper != -30 || *bins[13].Lower != 30 || bins[13].Upper != nil { + t.Fatalf("bounds: %+v", bins) + } + for i := 1; i < 13; i++ { + if *bins[i].Upper-*bins[i].Lower != 5 || *bins[i].Lower != *bins[i-1].Upper { + t.Fatal("non-contiguous buckets") + } + } + if signalAverage(0, 0) != nil || *signalAverage(0, 1) != 0 { + t.Fatal("zero must be distinct from no samples") + } +} diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index bb46595c..add69431 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -697,6 +697,21 @@ func (mr *MockQuerierMockRecorder) GetScopesByIATAs(ctx, dollar_1 any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopesByIATAs", reflect.TypeOf((*MockQuerier)(nil).GetScopesByIATAs), ctx, dollar_1) } +// GetSignalStats mocks base method. +func (m *MockQuerier) GetSignalStats(ctx context.Context, arg db.GetSignalStatsParams) ([]db.GetSignalStatsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSignalStats", ctx, arg) + ret0, _ := ret[0].([]db.GetSignalStatsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSignalStats indicates an expected call of GetSignalStats. +func (mr *MockQuerierMockRecorder) GetSignalStats(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSignalStats", reflect.TypeOf((*MockQuerier)(nil).GetSignalStats), ctx, arg) +} + // GetStatsClockDrift mocks base method. func (m *MockQuerier) GetStatsClockDrift(ctx context.Context, arg db.GetStatsClockDriftParams) ([]db.GetStatsClockDriftRow, error) { m.ctrl.T.Helper() @@ -1275,6 +1290,20 @@ func (mr *MockQuerierMockRecorder) RefreshRadioPresets(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshRadioPresets", reflect.TypeOf((*MockQuerier)(nil).RefreshRadioPresets), ctx) } +// RefreshSignalStats mocks base method. +func (m *MockQuerier) RefreshSignalStats(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RefreshSignalStats", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// RefreshSignalStats indicates an expected call of RefreshSignalStats. +func (mr *MockQuerierMockRecorder) RefreshSignalStats(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshSignalStats", reflect.TypeOf((*MockQuerier)(nil).RefreshSignalStats), ctx) +} + // RefreshTopAdvertisers mocks base method. func (m *MockQuerier) RefreshTopAdvertisers(ctx context.Context) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 80ab12bb..074994d4 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -112,6 +112,19 @@ type MvRadioPreset struct { Count int64 `json:"count"` } +type MvSignalStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrSum float64 `json:"snr_sum"` + RssiSamples int64 `json:"rssi_samples"` + RssiSum float64 `json:"rssi_sum"` +} + type MvTopAdvertisersByIatum struct { Iata string `json:"iata"` NodeID uuid.UUID `json:"node_id"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 7496e554..4256c4e1 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -90,6 +90,9 @@ type Querier interface { // whose observations have expired; the filtered aggregates are empty in that case. GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) + // Read compact hourly snapshots, never observations on an HTTP request. + // Weight averages by sample counts instead of averaging regional/hourly means. + GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds // the given threshold in magnitude, worst first. Not time-windowed -- reflects each node's // latest measured drift, not an aggregate over a period. @@ -214,6 +217,7 @@ type Querier interface { RefreshObserverActivity(ctx context.Context) error RefreshPayloadBreakdown(ctx context.Context) error RefreshRadioPresets(ctx context.Context) error + RefreshSignalStats(ctx context.Context) error RefreshTopAdvertisers(ctx context.Context) error RefreshTopNodes(ctx context.Context) error RefreshTopObservers(ctx context.Context) error diff --git a/db/sqlc/signal.sql.go b/db/sqlc/signal.sql.go new file mode 100644 index 00000000..8a490490 --- /dev/null +++ b/db/sqlc/signal.sql.go @@ -0,0 +1,99 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: signal.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getSignalStats = `-- name: GetSignalStats :many +WITH readings AS ( + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM mv_signal_stats_hourly + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz + AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM mv_signal_stats_hourly + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz + AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) +) +SELECT (3 + 4 * grouping(hour))::integer AS kind, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + -1::integer AS snr_bin, -1::integer AS rssi_bin, + COALESCE(sum(receptions), 0)::bigint AS receptions, + COALESCE(sum(snr_samples), 0)::bigint AS snr_samples, + COALESCE(sum(snr_sum) / NULLIF(sum(snr_samples), 0), 0)::double precision AS snr_average, + COALESCE(sum(rssi_samples), 0)::bigint AS rssi_samples, + COALESCE(sum(rssi_sum) / NULLIF(sum(rssi_samples), 0), 0)::double precision AS rssi_average +FROM readings WHERE kind = 3 +GROUP BY GROUPING SETS ((), (hour)) +UNION ALL +SELECT (kind + 4)::integer, 'epoch'::timestamptz, snr_bin, rssi_bin, + sum(receptions)::bigint, 0::bigint, 0::double precision, 0::bigint, 0::double precision +FROM readings WHERE kind IN (1, 2) +GROUP BY kind, snr_bin, rssi_bin +ORDER BY kind, hour, snr_bin, rssi_bin +` + +type GetSignalStatsParams struct { + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetSignalStatsRow struct { + Kind int32 `json:"kind"` + Hour pgtype.Timestamptz `json:"hour"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrAverage float64 `json:"snr_average"` + RssiSamples int64 `json:"rssi_samples"` + RssiAverage float64 `json:"rssi_average"` +} + +// Read compact hourly snapshots, never observations on an HTTP request. +// Weight averages by sample counts instead of averaging regional/hourly means. +func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) { + rows, err := q.db.Query(ctx, getSignalStats, arg.Since, arg.Until, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetSignalStatsRow{} + for rows.Next() { + var i GetSignalStatsRow + if err := rows.Scan( + &i.Kind, + &i.Hour, + &i.SnrBin, + &i.RssiBin, + &i.Receptions, + &i.SnrSamples, + &i.SnrAverage, + &i.RssiSamples, + &i.RssiAverage, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const refreshSignalStats = `-- name: RefreshSignalStats :exec +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_signal_stats_hourly +` + +func (q *Queries) RefreshSignalStats(ctx context.Context) error { + _, err := q.db.Exec(ctx, refreshSignalStats) + return err +} diff --git a/db/stats_migration_integration_test.go b/db/stats_migration_integration_test.go new file mode 100644 index 00000000..cfe3f94f --- /dev/null +++ b/db/stats_migration_integration_test.go @@ -0,0 +1,33 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// Views cannot reference temporary tables. Keep fixtures and real migration DDL +// in a unique schema inside the caller's rolled-back transaction instead. +func isolateStatsSchema(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + schema := pgx.Identifier{"stats_test_" + uuid.NewString()}.Sanitize() + if _, err := tx.Exec(ctx, "CREATE SCHEMA "+schema+"; SET LOCAL search_path TO "+schema); err != nil { + t.Fatal(err) + } +} + +func applyStatsMigration(t *testing.T, ctx context.Context, tx pgx.Tx, name string) { + t.Helper() + migration, err := migrationFiles.ReadFile("migrations/" + name) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, string(migration)); err != nil { + t.Fatal(err) + } +} diff --git a/docs/docs.go b/docs/docs.go index 5988c94b..95a95b60 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2674,6 +2674,78 @@ const docTemplate = `{ } } }, + "/stats/signal": { + "get": { + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Reception signal distributions and hourly trends", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/top-advertisers": { "get": { "produces": [ @@ -4626,6 +4698,86 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "lower": { + "type": "number" + }, + "upper": { + "type": "number" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "rssiAverage": { + "type": "number" + }, + "rssiSamples": { + "type": "integer" + }, + "snrAverage": { + "type": "number" + }, + "snrSamples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric": { + "type": "object", + "properties": { + "average": { + "type": "number" + }, + "histogram": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin" + } + }, + "samples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats": { + "type": "object", + "properties": { + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour" + } + }, + "receptions": { + "type": "integer" + }, + "rssi": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "since": { + "type": "integer" + }, + "snr": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index b9a03899..2460fe3d 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2672,6 +2672,78 @@ } } }, + "/stats/signal": { + "get": { + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Reception signal distributions and hourly trends", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/top-advertisers": { "get": { "produces": [ @@ -4624,6 +4696,86 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "lower": { + "type": "number" + }, + "upper": { + "type": "number" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "rssiAverage": { + "type": "number" + }, + "rssiSamples": { + "type": "integer" + }, + "snrAverage": { + "type": "number" + }, + "snrSamples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric": { + "type": "object", + "properties": { + "average": { + "type": "number" + }, + "histogram": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin" + } + }, + "samples": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats": { + "type": "object", + "properties": { + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour" + } + }, + "receptions": { + "type": "integer" + }, + "rssi": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "since": { + "type": "integer" + }, + "snr": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 971bc30f..be2d5b2b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1105,6 +1105,58 @@ definitions: description: distinct packets matched to this scope type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin: + properties: + count: + type: integer + lower: + type: number + upper: + type: number + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour: + properties: + hour: + type: integer + receptions: + type: integer + rssiAverage: + type: number + rssiSamples: + type: integer + snrAverage: + type: number + snrSamples: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric: + properties: + average: + type: number + histogram: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalBin' + type: array + samples: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats: + properties: + hourly: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalHour' + type: array + receptions: + type: integer + rssi: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric' + since: + type: integer + snr: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalMetric' + until: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.StatsOverview: properties: activeIatas: @@ -3101,6 +3153,62 @@ paths: summary: Scope statistics tags: - Stats + /stats/signal: + get: + description: Aggregates stored observer receptions in [since, until), at most + 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero + unavailable sentinel are excluded per metric; actual zero SNR with nonzero + RSSI remains valid. Averages are null without samples. Histogram bounds are + lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries + round down to UTC hours and the response reports that effective window. Reads + use hourly materialized snapshots refreshed by background.view_refresh; the + current partial hour is excluded and absent hours are omitted. These last-hop + readings do not measure end-to-end quality or packet loss. + parameters: + - description: Inclusive start, epoch milliseconds (0 through 253402300799999) + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; at most 30 days after since + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.SignalStats' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Reception signal distributions and hourly trends + tags: + - Stats /stats/top-advertisers: get: parameters: diff --git a/internal/api/handlers/regions.go b/internal/api/handlers/regions.go index b9bfe5c5..0b0f4008 100644 --- a/internal/api/handlers/regions.go +++ b/internal/api/handlers/regions.go @@ -9,6 +9,7 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" @@ -74,6 +75,24 @@ func getRegion(reader api.Reader) http.HandlerFunc { } } +// parseStatsWindow rounds both endpoints down to UTC hours. Polls within an +// hour share a cache key; the response reports the effective window. A valid +// sub-hour request can contain no complete buckets and return an empty result. +func parseStatsWindow(r *http.Request) (time.Time, time.Time, error) { + q := r.URL.Query() + for _, key := range []string{"since", "until"} { + if len(q[key]) != 1 || q.Get(key) == "" { + return time.Time{}, time.Time{}, fmt.Errorf("%s must be supplied exactly once", key) + } + } + since, errSince := strconv.ParseInt(q.Get("since"), 10, 64) + until, errUntil := strconv.ParseInt(q.Get("until"), 10, 64) + if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 || until-since > int64((30*24*time.Hour)/time.Millisecond) { + return time.Time{}, time.Time{}, fmt.Errorf("since and until must be epoch milliseconds between 0 and 253402300799999, with a positive window of at most 30 days") + } + return time.UnixMilli(since).UTC().Truncate(time.Hour), time.UnixMilli(until).UTC().Truncate(time.Hour), nil +} + // parseIATAs splits a comma-separated iatas query param and uppercases each value. func parseIATAs(r *http.Request) []string { raw := r.URL.Query().Get("iatas") diff --git a/internal/api/handlers/signal.go b/internal/api/handlers/signal.go new file mode 100644 index 00000000..b1029a98 --- /dev/null +++ b/internal/api/handlers/signal.go @@ -0,0 +1,68 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// getSignalStats godoc +// +// @Summary Reception signal distributions and hourly trends +// @Description Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss. +// @Tags Stats +// @Produce json +// @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" +// @Param until query int true "Exclusive end, epoch milliseconds; at most 30 days after since" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.SignalStats +// @Failure 400 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/signal [get] +func getSignalStats(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + since, until, err := parseStatsWindow(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondError(w, http.StatusBadRequest, "region not found") + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} + } + } + stats, err := reader.GetSignalStats(ctx, since, until, iatas) + switch { + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "signal query timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("signal stats failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, stats) + } + } +} diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go new file mode 100644 index 00000000..405db1ca --- /dev/null +++ b/internal/api/handlers/signal_test.go @@ -0,0 +1,113 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type signalReader struct { + stubReader + read func(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) +} + +func (s signalReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + return s.read(ctx, since, until, iatas) +} + +func TestSignalRejectsInvalidWindow(t *testing.T) { + for _, query := range []string{"", "since=0", "until=1", "since=x&until=1", "since=-1&until=1", "since=0&until=0", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { + w := httptest.NewRecorder() + StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) + if w.Code != 400 { + t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) + } + } +} + +func TestSignalFiltersAndErrors(t *testing.T) { + for _, tc := range []struct { + name, query string + iatas []string + err error + status int + }{ + {"global", "", nil, nil, 200}, + {"IATAs", "&iatas=yvr,yyj", []string{"YVR", "YYJ"}, nil, 200}, + {"region", "®ion=west", []string{"YVR"}, nil, 200}, + {"empty region", "®ion=empty", []string{""}, nil, 200}, + {"missing region", "®ion=missing", nil, nil, 400}, + {"timeout", "", nil, context.DeadlineExceeded, 503}, + {"database error", "", nil, errors.New("private detail"), 500}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := signalReader{stubReader: stubReader{getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + switch slug { + case "empty": + return &api.Region{}, nil + case "missing": + return nil, errors.New("not found") + default: + return &api.Region{IATAs: []string{"YVR"}}, nil + } + }}, read: func(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { + t.Fatal("missing bounded query deadline") + } + if since.UnixMilli() != 0 || until.UnixMilli() != 2592000000 || !reflect.DeepEqual(iatas, tc.iatas) { + t.Fatalf("unexpected arguments: %v %v %v", since, until, iatas) + } + return &api.SignalStats{Until: until.UnixMilli(), Receptions: 5, Hourly: []api.SignalHour{}}, tc.err + }} + w := httptest.NewRecorder() + StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) + if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + if tc.status == 200 { + var got api.SignalStats + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || got.Receptions != 5 || got.SNR.Average != nil { + t.Fatalf("invalid response: %+v %v", got, err) + } + } + }) + } +} + +func TestSignalCancellationReachesReader(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + called := false + reader := signalReader{read: func(ctx context.Context, _ time.Time, _ time.Time, _ []string) (*api.SignalStats, error) { + called = true + return nil, ctx.Err() + }} + w := httptest.NewRecorder() + StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) + if !called || w.Body.Len() != 0 { + t.Fatalf("cancellation: called=%v body=%s", called, w.Body.String()) + } +} + +func TestStatsWindowSnapsPollingTimes(t *testing.T) { + for _, query := range []string{"since=123&until=86400123", "since=3599999&until=89999999"} { + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?"+query, nil)) + if err != nil || since.UnixMilli() != 0 || until.UnixMilli() != 86400000 { + t.Fatalf("%v %v %v", since, until, err) + } + } + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?since=1&until=2", nil)) + if err != nil || !since.Equal(until) { + t.Fatal("sub-hour normalization") + } +} diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index f65bdc04..56b7b0a8 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -30,6 +30,7 @@ func StatsRouter(reader api.Reader) http.Handler { r := chi.NewRouter() r.Get("/overview", getStatsOverview(reader)) r.Get("/observations", getStatsObservations(reader)) + r.Get("/signal", getSignalStats(reader)) r.Get("/observer-comparison", getObserverComparison(reader)) r.Get("/payload-breakdown", getStatsPayloadBreakdown(reader)) r.Get("/top-nodes", getStatsTopNodes(reader)) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 559200a0..644d6c03 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -66,6 +66,10 @@ type stubReader struct { getNodesByIDs func(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]*api.ResolvedNode, error) } +func (s stubReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { + return nil, nil +} + func (s stubReader) GetObserverComparison(ctx context.Context, a, b uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { if s.getObserverComparison != nil { return s.getObserverComparison(ctx, a, b, since, until, iatas) diff --git a/internal/api/reader.go b/internal/api/reader.go index 1b90b75c..90593549 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -161,6 +161,9 @@ type Reader interface { // since defines the start of the window; pass zero time for default (last 7 days). GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]ObservationPoint, error) + // GetSignalStats aggregates retained reception readings in [since, until). + GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*SignalStats, error) + // GetStatsPayloadBreakdown returns observation counts grouped by payload type. // Pass nil for iatas to return stats across all IATAs. // since defines the start of the window; pass zero time for default (last 24h). diff --git a/internal/api/signal.go b/internal/api/signal.go new file mode 100644 index 00000000..1514dce4 --- /dev/null +++ b/internal/api/signal.go @@ -0,0 +1,42 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +// SignalStats describes a materialized reception snapshot in [Since, Until). +// Since and Until are the effective UTC-hour boundaries, rounded down from input. +// A reception is a stored observation, not a unique packet or a lost-packet estimate. +type SignalStats struct { + Since int64 `json:"since"` + Until int64 `json:"until"` + Receptions int64 `json:"receptions"` + SNR SignalMetric `json:"snr"` + RSSI SignalMetric `json:"rssi"` + Hourly []SignalHour `json:"hourly"` +} + +// SignalMetric counts available finite samples; Average is null without samples. +type SignalMetric struct { + Samples int64 `json:"samples"` + Average *float64 `json:"average"` + Histogram []SignalBin `json:"histogram"` +} + +// SignalBin includes Lower and excludes Upper. Null denotes an unbounded end. +// SNR uses 5 dB bins from -30 to 30; RSSI uses 10 dBm bins from -140 to 0, +// both with underflow and overflow bins. These are display bins, not quality ratings. +type SignalBin struct { + Lower *float64 `json:"lower"` + Upper *float64 `json:"upper"` + Count int64 `json:"count"` +} + +// SignalHour is a complete UTC bucket in the effective window. Missing hours are omitted. +type SignalHour struct { + Hour int64 `json:"hour"` + Receptions int64 `json:"receptions"` + SNRSamples int64 `json:"snrSamples"` + SNRAverage *float64 `json:"snrAverage"` + RSSISamples int64 `json:"rssiSamples"` + RSSIAverage *float64 `json:"rssiAverage"` +} diff --git a/internal/background/tasks.go b/internal/background/tasks.go index b29760ad..0290df8a 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -22,6 +22,7 @@ type viewRefresher interface { RefreshTopAdvertisers(context.Context) error RefreshRadioPresets(context.Context) error RefreshObserverActivity(context.Context) error + RefreshSignalStats(context.Context) error } // ViewRefreshTask returns a Task that refreshes all materialized views. @@ -43,6 +44,7 @@ func ViewRefreshTask(store viewRefresher, interval time.Duration) Task { {"top advertisers", store.RefreshTopAdvertisers}, {"radio presets", store.RefreshRadioPresets}, {"observer activity", store.RefreshObserverActivity}, + {"signal stats", store.RefreshSignalStats}, } { if err := view.refresh(ctx); err != nil { errs = append(errs, fmt.Errorf("%s: %w", view.name, err)) diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go index af1dae22..2c4526dd 100644 --- a/internal/background/tasks_test.go +++ b/internal/background/tasks_test.go @@ -119,6 +119,10 @@ func (s *refreshStub) RefreshObserverActivity(ctx context.Context) error { return s.refresh(ctx, "observer activity") } +func (s *refreshStub) RefreshSignalStats(ctx context.Context) error { + return s.refresh(ctx, "signal stats") +} + func TestViewRefreshTask(t *testing.T) { first, second := errors.New("first failure"), errors.New("second failure") for _, tc := range []struct { @@ -127,13 +131,14 @@ func TestViewRefreshTask(t *testing.T) { }{ {"success", nil}, {"partial failure", map[string]error{"hourly stats": first, "top talkers": second}}, + {"signal failure", map[string]error{"signal stats": first}}, } { t.Run(tc.name, func(t *testing.T) { store := &refreshStub{errs: tc.errs} task := ViewRefreshTask(store, time.Minute) err := task.Run(context.Background()) - if len(store.calls) != 8 { - t.Fatalf("refreshed %d views, want 8", len(store.calls)) + if len(store.calls) != 9 { + t.Fatalf("refreshed %d views, want 9", len(store.calls)) } if len(tc.errs) == 0 && err != nil { t.Fatal(err) diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index e39ccc03..4d7ec1b6 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -31,6 +31,10 @@ type stubReader struct { calls int } +func (s *stubReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { + return nil, s.err +} + func (s *stubReader) ListIATAs(_ context.Context) ([]api.IATA, error) { s.calls++ return s.iatas, s.err diff --git a/internal/cache/reader.go b/internal/cache/reader.go index aa4f402c..5d846c6b 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -29,6 +29,7 @@ const ( keyScopeByNamePrefix = "beacon:scope:name:" keyStatsOverviewPrefix = "beacon:stats:overview:" keyStatsObservationsPrefix = "beacon:stats:observations:" + keySignalStatsPrefix = "beacon:stats:signal:" keyStatsBreakdownPrefix = "beacon:stats:breakdown:" keyStatsTopNodesPrefix = "beacon:stats:top-nodes:" keyStatsTopObsPrefix = "beacon:stats:top-observers:" diff --git a/internal/cache/signal.go b/internal/cache/signal.go new file mode 100644 index 00000000..d1dfaa51 --- /dev/null +++ b/internal/cache/signal.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func (cr *CachedReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + sorted := append([]string{}, iatas...) + slices.Sort(sorted) + sorted = slices.Compact(sorted) + segment, _ := json.Marshal(sorted) // [] and [""] must remain distinct (global vs empty region). + key := fmt.Sprintf("%s%s:%d:%d", keySignalStatsPrefix, segment, since.UnixMilli(), until.UnixMilli()) + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.SignalStats, error) { + return cr.inner.GetSignalStats(ctx, since, until, iatas) + }) +} diff --git a/internal/cache/signal_test.go b/internal/cache/signal_test.go new file mode 100644 index 00000000..89b739b6 --- /dev/null +++ b/internal/cache/signal_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type signalStatsReader struct { + api.Reader + calls int64 +} + +func (r *signalStatsReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { + r.calls++ + return &api.SignalStats{Receptions: r.calls}, nil +} + +func TestSignalCacheSeparatesWindowAndRegion(t *testing.T) { + c, mr := newTestClient(t) + inner := &signalStatsReader{} + r := NewCachedReader(inner, c, CacheTTLs{Stats: time.Minute}) + for _, tc := range []struct { + iatas []string + since, until, want int64 + }{ + {nil, 0, 10, 1}, {[]string{}, 0, 10, 1}, {[]string{""}, 0, 10, 2}, {[]string{"YVR"}, 0, 10, 3}, + {[]string{"YVR", "YYJ"}, 0, 10, 4}, {[]string{"YYJ", "YVR", "YYJ"}, 0, 10, 4}, + {nil, 1, 10, 5}, {nil, 0, 11, 6}, {nil, 0, 10, 1}, + } { + before := slices.Clone(tc.iatas) + got, err := r.GetSignalStats(context.Background(), time.UnixMilli(tc.since), time.UnixMilli(tc.until), tc.iatas) + if err != nil || got.Receptions != tc.want || !slices.Equal(before, tc.iatas) { + t.Fatalf("%+v: %+v %v", tc, got, err) + } + } + mr.FastForward(time.Minute) + got, err := r.GetSignalStats(context.Background(), time.UnixMilli(0), time.UnixMilli(10), nil) + if err != nil || got.Receptions != 7 { + t.Fatalf("expiry: %+v %v", got, err) + } +} From 8a3fa7e6bf9ef16e45218a2fc165fb8a7ea40a1b Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 23 Sep 2026 20:18:41 -0400 Subject: [PATCH 048/141] feat(stats): add bounded path and hash-width analytics (#159) --- .github/workflows/ci.yml | 4 +- db/migrations/036_mv_path_stats.sql | 31 ++++ db/paths.go | 76 ++++++++++ db/paths_integration_test.go | 163 +++++++++++++++++++++ db/paths_test.go | 33 +++++ db/queries/paths.sql | 21 +++ db/sqlc/mock/querier.go | 29 ++++ db/sqlc/models.go | 9 ++ db/sqlc/paths.sql.go | 83 +++++++++++ db/sqlc/querier.go | 3 + docs/docs.go | 167 ++++++++++++++++++++++ docs/swagger.json | 167 ++++++++++++++++++++++ docs/swagger.yaml | 119 +++++++++++++++ internal/api/handlers/paths.go | 68 +++++++++ internal/api/handlers/paths_test.go | 93 ++++++++++++ internal/api/handlers/stats.go | 1 + internal/api/handlers/stub_reader_test.go | 4 + internal/api/paths.go | 46 ++++++ internal/api/reader.go | 3 + internal/background/tasks.go | 2 + internal/background/tasks_test.go | 9 +- internal/cache/cache_test.go | 4 + internal/cache/paths.go | 25 ++++ internal/cache/paths_test.go | 48 +++++++ internal/cache/reader.go | 1 + 25 files changed, 1205 insertions(+), 4 deletions(-) create mode 100644 db/migrations/036_mv_path_stats.sql create mode 100644 db/paths.go create mode 100644 db/paths_integration_test.go create mode 100644 db/paths_test.go create mode 100644 db/queries/paths.sql create mode 100644 db/sqlc/paths.sql.go create mode 100644 internal/api/handlers/paths.go create mode 100644 internal/api/handlers/paths_test.go create mode 100644 internal/api/paths.go create mode 100644 internal/cache/paths.go create mode 100644 internal/cache/paths_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b6b62a9..43b9d5c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,10 +52,10 @@ jobs: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable run: go test ./cmd/beacon -run '^TestBackupConfigurationPostgres$' -count=1 -v - - name: Verify signal aggregates against PostgreSQL 16 + - name: Verify signal and path aggregates against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^TestSignalPostgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/db/migrations/036_mv_path_stats.sql b/db/migrations/036_mv_path_stats.sql new file mode 100644 index 00000000..7612255c --- /dev/null +++ b/db/migrations/036_mv_path_stats.sql @@ -0,0 +1,31 @@ +-- Classify once per background refresh, preserving the decoder's header rules. +-- No observation rows or deduplication constraints are changed. +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_path_stats_hourly AS +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; +CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_path_stats_hourly + ON mv_path_stats_hourly (iata, hour, category, hash_bytes, entries); +CREATE INDEX IF NOT EXISTS idx_mv_path_stats_hourly_hour ON mv_path_stats_hourly (hour); diff --git a/db/paths.go b/db/paths.go new file mode 100644 index 00000000..9df530d6 --- /dev/null +++ b/db/paths.go @@ -0,0 +1,76 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + rows, err := s.q.GetPathStats(ctx, sqlc.GetPathStatsParams{Since: pgtype.Timestamptz{Time: since, Valid: true}, Until: pgtype.Timestamptz{Time: until, Valid: true}, Iatas: iatas}) + if err != nil { + return nil, err + } + stats := &api.PathStats{Since: since.UnixMilli(), Until: until.UnixMilli(), HashWidths: []api.PathHashWidth{{Bytes: 1}, {Bytes: 2}, {Bytes: 3}}, PathLengths: []api.PathLengthBin{}, Hourly: []api.PathHour{}} + var lengths [64]int64 + for _, row := range rows { + if row.Category == 0 && (row.HashBytes < 1 || int(row.HashBytes) > len(stats.HashWidths) || row.Entries < 0 || int(row.Entries) >= len(lengths)) { + return nil, fmt.Errorf("invalid path statistics bucket") + } + if row.IsHourly { + hour := row.Hour.Time.UnixMilli() + if len(stats.Hourly) == 0 || stats.Hourly[len(stats.Hourly)-1].Hour != hour { + stats.Hourly = append(stats.Hourly, api.PathHour{Hour: hour}) + } + point := &stats.Hourly[len(stats.Hourly)-1] + point.Receptions += row.Receptions + switch row.Category { + case 0: + switch row.HashBytes { + case 1: + point.OneByte += row.Receptions + case 2: + point.TwoByte += row.Receptions + case 3: + point.ThreeByte += row.Receptions + } + case 1: + point.Empty += row.Receptions + case 2: + point.Trace += row.Receptions + case 3: + point.Unclassified += row.Receptions + } + continue + } + stats.Receptions += row.Receptions + switch row.Category { + case 0: + stats.Hashed += row.Receptions + stats.HashWidths[row.HashBytes-1].Receptions += row.Receptions + lengths[row.Entries] += row.Receptions + case 1: + stats.Empty += row.Receptions + lengths[0] += row.Receptions + case 2: + stats.Trace += row.Receptions + case 3: + stats.Unclassified += row.Receptions + } + } + for entries, count := range lengths { + if count > 0 { + stats.PathLengths = append(stats.PathLengths, api.PathLengthBin{Entries: int32(entries), Receptions: count}) + } + } + return stats, nil +} + +func (s *Store) RefreshPathStats(ctx context.Context) error { return s.q.RefreshPathStats(ctx) } diff --git a/db/paths_integration_test.go b/db/paths_integration_test.go new file mode 100644 index 00000000..e10e7935 --- /dev/null +++ b/db/paths_integration_test.go @@ -0,0 +1,163 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "fmt" + "net/http/httptest" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/jackc/pgx/v5" + meshcore "github.com/meshcore-go/meshcore-go" +) + +func TestPathsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + isolateStatsSchema(t, ctx, tx) + _, err = tx.Exec(ctx, ` +SET LOCAL TIME ZONE 'America/Vancouver'; +CREATE TABLE packet_observations(heard_at timestamptz NOT NULL,iata char(3) NOT NULL,payload_type smallint,path_length_byte smallint NOT NULL,hash_size smallint NOT NULL,hop_count smallint NOT NULL,path_bytes bytea); +INSERT INTO packet_observations +SELECT CASE id WHEN 1 THEN '2026-01-01 00:30+00'::timestamptz WHEN 17 THEN '2026-01-01 04:00+00'::timestamptz WHEN 18 THEN '2026-01-01 04:30+00'::timestamptz WHEN 20 THEN '2026-01-01 03:00+00'::timestamptz + ELSE '2026-01-01 00:30+00'::timestamptz+(id/10)*interval '2 hour'+(id%10)*interval '1 minute' END, + iata,payload,raw,width,hops,CASE WHEN size IS NULL THEN NULL ELSE decode(repeat('ab',size),'hex') END +FROM (VALUES + (1,'YVR',4,2,1,2,2),(2,'YVR',4,67,2,3,6),(3,'YYJ',5,130,3,2,6), + (4,'YVR',4,64,2,0,NULL),(5,'YVR',9,3,1,3,3),(6,'YYJ',NULL,2,1,2,2), + (7,'YVR',4,3,1,3,2),(8,'YVR',4,194,4,2,8),(9,'YVR',4,96,2,32,64), + (10,'YVR',4,97,2,33,66),(11,'YVR',4,64,1,0,0),(12,'YVR',4,0,1,1,1), + (13,'YYZ',5,0,1,0,0),(14,'YVR',16,1,1,1,1),(15,'YVR',4,63,1,63,63), + (16,'YVR',4,149,3,21,63),(17,'YVR',4,1,1,1,1),(18,'YVR',4,1,1,1,1), + (19,'YYJ',9,3,1,3,3),(20,'YVR',4,64,2,0,0) +) v(id,iata,payload,raw,width,hops,size);`) + if err != nil { + t.Fatal(err) + } + since := time.Now().UTC().Truncate(24 * time.Hour).Add(-48 * time.Hour) + if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { + t.Fatal(err) + } + applyStatsMigration(t, ctx, tx, "036_mv_path_stats.sql") + applyStatsMigration(t, ctx, tx, "036_mv_path_stats.sql") // interrupted journal retry + store := &Store{q: sqlc.New(tx)} + until := since.Add(4 * time.Hour) + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err := tx.Exec(ctx, "SET LOCAL plan_cache_mode="+mode); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + iatas []string + counts [5]int64 + }{ + {"global", nil, [5]int64{18, 6, 3, 2, 7}}, {"empty filter", []string{}, [5]int64{18, 6, 3, 2, 7}}, + {"YVR", []string{"YVR"}, [5]int64{14, 5, 2, 1, 6}}, {"overlap", []string{"YVR", "YYJ", "YVR"}, [5]int64{17, 6, 2, 2, 7}}, + {"unknown", []string{"ZZZ"}, [5]int64{}}, {"empty region", []string{""}, [5]int64{}}, + } { + t.Run(mode+"/"+tc.name, func(t *testing.T) { + got, err := store.GetPathStats(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + if [5]int64{got.Receptions, got.Hashed, got.Empty, got.Trace, got.Unclassified} != tc.counts { + t.Fatalf("counts: %+v", got) + } + var widths, lengths, hours int64 + for _, b := range got.HashWidths { + widths += b.Receptions + } + for _, b := range got.PathLengths { + lengths += b.Receptions + } + for i, h := range got.Hourly { + hours += h.Receptions + if h.Receptions != h.OneByte+h.TwoByte+h.ThreeByte+h.Empty+h.Trace+h.Unclassified || h.Hour%3600000 != 0 || (i > 0 && h.Hour <= got.Hourly[i-1].Hour) { + t.Fatalf("hour: %+v", h) + } + } + if widths != got.Hashed || lengths != got.Hashed+got.Empty || hours != got.Receptions { + t.Fatal("partitions do not reconcile") + } + if tc.name == "global" && (got.HashWidths[0].Receptions != 2 || got.HashWidths[1].Receptions != 2 || got.HashWidths[2].Receptions != 2 || got.PathLengths[0].Entries != 0 || got.PathLengths[0].Receptions != 3 || len(got.Hourly) != 3) { + t.Fatalf("empty/trace widths or missing hour: %+v", got) + } + }) + } + } + w := httptest.NewRecorder() + handlers.StatsRouter(store).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) + var response api.PathStats + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 14 || response.Hashed != 5 { + t.Fatalf("HTTP %d: %s (%v)", w.Code, w.Body.String(), err) + } + if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `INSERT INTO packet_observations SELECT $1::timestamptz+interval '1 hour','YVR',4,i,(i>>6)+1,i&63,decode(repeat('ab',((i>>6)+1)*(i&63)),'hex') FROM generate_series(0,255)i;`, since); err != nil { + t.Fatal(err) + } + stale, err := store.GetPathStats(ctx, since, until, nil) + if err != nil || stale.Receptions != 18 { + t.Fatalf("snapshot lost: %+v %v", stale, err) + } + if err := store.RefreshPathStats(ctx); err != nil { + t.Fatal(err) + } + got, err := store.GetPathStats(ctx, since, until, nil) + if err != nil { + t.Fatal(err) + } + var hashed, empty, unknown int64 + var widths [3]int64 + for raw := 0; raw < 256; raw++ { + if !meshcore.IsValidPathLen(uint8(raw)) { + unknown++ + } else if raw&63 == 0 { + empty++ + } else { + hashed++ + widths[raw>>6]++ + } + } + if got.Hashed != hashed || got.Empty != empty || got.Unclassified != unknown || got.Receptions != 256 { + t.Fatalf("decoder agreement: %+v", got) + } + for i, count := range widths { + if got.HashWidths[i].Receptions != count { + t.Fatal("decoder width disagreement") + } + } + if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { + t.Fatal(err) + } + if err := store.RefreshPathStats(ctx); err != nil { + t.Fatal(err) + } + got, err = store.GetPathStats(ctx, since, until, nil) + if err != nil || got.Receptions != 0 || got.Hourly == nil || got.PathLengths == nil || len(got.HashWidths) != 3 { + t.Fatalf("empty: %+v %v", got, err) + } +} diff --git a/db/paths_test.go b/db/paths_test.go new file mode 100644 index 00000000..beb2f582 --- /dev/null +++ b/db/paths_test.go @@ -0,0 +1,33 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" +) + +type pathBucketQuerier struct { + sqlc.Querier + row sqlc.GetPathStatsRow +} + +func (q pathBucketQuerier) GetPathStats(context.Context, sqlc.GetPathStatsParams) ([]sqlc.GetPathStatsRow, error) { + return []sqlc.GetPathStatsRow{q.row}, nil +} + +func TestPathStatsRejectsInvalidDatabaseBucket(t *testing.T) { + for _, row := range []sqlc.GetPathStatsRow{ + {HashBytes: 0}, {HashBytes: 4}, {HashBytes: 1, Entries: -1}, {HashBytes: 1, Entries: 64}, + {HashBytes: 4, IsHourly: true}, + } { + store := &Store{q: pathBucketQuerier{row: row}} + if _, err := store.GetPathStats(context.Background(), time.Time{}, time.Time{}, nil); err == nil { + t.Fatalf("invalid bucket accepted: %+v", row) + } + } +} diff --git a/db/queries/paths.sql b/db/queries/paths.sql new file mode 100644 index 00000000..93703be0 --- /dev/null +++ b/db/queries/paths.sql @@ -0,0 +1,21 @@ +-- name: GetPathStats :many +-- All HTTP aggregates read the compact classification snapshot. +WITH readings AS ( + SELECT * FROM mv_path_stats_hourly + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz + AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT * FROM mv_path_stats_hourly + WHERE hour >= @since::timestamptz AND hour < @until::timestamptz + AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) +) +SELECT (grouping(hour) = 0)::boolean AS is_hourly, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + category, hash_bytes, COALESCE(entries, 0)::integer AS entries, + sum(receptions)::bigint AS receptions +FROM readings +GROUP BY GROUPING SETS ((category, hash_bytes, entries), (hour, category, hash_bytes)) +ORDER BY is_hourly, hour, category, hash_bytes, entries; + +-- name: RefreshPathStats :exec +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_path_stats_hourly; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index add69431..d941dc96 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -577,6 +577,21 @@ func (mr *MockQuerierMockRecorder) GetPacketsByTraceTag(ctx, decode any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPacketsByTraceTag", reflect.TypeOf((*MockQuerier)(nil).GetPacketsByTraceTag), ctx, decode) } +// GetPathStats mocks base method. +func (m *MockQuerier) GetPathStats(ctx context.Context, arg db.GetPathStatsParams) ([]db.GetPathStatsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPathStats", ctx, arg) + ret0, _ := ret[0].([]db.GetPathStatsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPathStats indicates an expected call of GetPathStats. +func (mr *MockQuerierMockRecorder) GetPathStats(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPathStats", reflect.TypeOf((*MockQuerier)(nil).GetPathStats), ctx, arg) +} + // GetRadioPresets mocks base method. func (m *MockQuerier) GetRadioPresets(ctx context.Context, arg db.GetRadioPresetsParams) ([]db.MvRadioPreset, error) { m.ctrl.T.Helper() @@ -1262,6 +1277,20 @@ func (mr *MockQuerierMockRecorder) RefreshObserverActivity(ctx any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshObserverActivity", reflect.TypeOf((*MockQuerier)(nil).RefreshObserverActivity), ctx) } +// RefreshPathStats mocks base method. +func (m *MockQuerier) RefreshPathStats(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RefreshPathStats", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// RefreshPathStats indicates an expected call of RefreshPathStats. +func (mr *MockQuerierMockRecorder) RefreshPathStats(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshPathStats", reflect.TypeOf((*MockQuerier)(nil).RefreshPathStats), ctx) +} + // RefreshPayloadBreakdown mocks base method. func (m *MockQuerier) RefreshPayloadBreakdown(ctx context.Context) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 074994d4..126b37a4 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -98,6 +98,15 @@ type MvObserverActivityHourly struct { RssiN int64 `json:"rssi_n"` } +type MvPathStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + type MvPayloadBreakdownByIatum struct { Iata string `json:"iata"` PayloadType *int16 `json:"payload_type"` diff --git a/db/sqlc/paths.sql.go b/db/sqlc/paths.sql.go new file mode 100644 index 00000000..05bf5b1c --- /dev/null +++ b/db/sqlc/paths.sql.go @@ -0,0 +1,83 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: paths.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getPathStats = `-- name: GetPathStats :many +WITH readings AS ( + SELECT iata, hour, category, hash_bytes, entries, receptions FROM mv_path_stats_hourly + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz + AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT iata, hour, category, hash_bytes, entries, receptions FROM mv_path_stats_hourly + WHERE hour >= $1::timestamptz AND hour < $2::timestamptz + AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) +) +SELECT (grouping(hour) = 0)::boolean AS is_hourly, + COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, + category, hash_bytes, COALESCE(entries, 0)::integer AS entries, + sum(receptions)::bigint AS receptions +FROM readings +GROUP BY GROUPING SETS ((category, hash_bytes, entries), (hour, category, hash_bytes)) +ORDER BY is_hourly, hour, category, hash_bytes, entries +` + +type GetPathStatsParams struct { + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetPathStatsRow struct { + IsHourly bool `json:"is_hourly"` + Hour pgtype.Timestamptz `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +// All HTTP aggregates read the compact classification snapshot. +func (q *Queries) GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]GetPathStatsRow, error) { + rows, err := q.db.Query(ctx, getPathStats, arg.Since, arg.Until, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetPathStatsRow{} + for rows.Next() { + var i GetPathStatsRow + if err := rows.Scan( + &i.IsHourly, + &i.Hour, + &i.Category, + &i.HashBytes, + &i.Entries, + &i.Receptions, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const refreshPathStats = `-- name: RefreshPathStats :exec +REFRESH MATERIALIZED VIEW CONCURRENTLY mv_path_stats_hourly +` + +func (q *Queries) RefreshPathStats(ctx context.Context) error { + _, err := q.db.Exec(ctx, refreshPathStats) + return err +} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 4256c4e1..aa4bd874 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -79,6 +79,8 @@ type Querier interface { // Return distinct observation IATAs in first-heard order for path resolution, // without fetching full observations separately for every trace packet. GetPacketsByTraceTag(ctx context.Context, decode string) ([]GetPacketsByTraceTagRow, error) + // All HTTP aggregates read the compact classification snapshot. + GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]GetPathStatsRow, error) GetRadioPresets(ctx context.Context, arg GetRadioPresetsParams) ([]MvRadioPreset, error) GetRegion(ctx context.Context, id int32) (GetRegionRow, error) GetRegionBySlug(ctx context.Context, slug string) (GetRegionBySlugRow, error) @@ -215,6 +217,7 @@ type Querier interface { ReconfirmRoutes(ctx context.Context, limit int32) error RefreshHourlyStats(ctx context.Context) error RefreshObserverActivity(ctx context.Context) error + RefreshPathStats(ctx context.Context) error RefreshPayloadBreakdown(ctx context.Context) error RefreshRadioPresets(ctx context.Context) error RefreshSignalStats(ctx context.Context) error diff --git a/docs/docs.go b/docs/docs.go index 95a95b60..43ed843a 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2505,6 +2505,78 @@ const docTemplate = `{ } } }, + "/stats/paths": { + "get": { + "description": "Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads use materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and missing hours are omitted.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Received path-entry and hash-width distributions", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/payload-breakdown": { "get": { "produces": [ @@ -4492,6 +4564,101 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth": { + "type": "object", + "properties": { + "bytes": { + "type": "integer" + }, + "receptions": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour": { + "type": "object", + "properties": { + "empty": { + "type": "integer" + }, + "hour": { + "type": "integer" + }, + "oneByte": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "threeByte": { + "type": "integer" + }, + "trace": { + "type": "integer" + }, + "twoByte": { + "type": "integer" + }, + "unclassified": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin": { + "type": "object", + "properties": { + "entries": { + "type": "integer" + }, + "receptions": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats": { + "type": "object", + "properties": { + "empty": { + "type": "integer" + }, + "hashWidths": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth" + } + }, + "hashed": { + "type": "integer" + }, + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour" + } + }, + "pathLengths": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin" + } + }, + "receptions": { + "type": "integer" + }, + "since": { + "type": "integer" + }, + "trace": { + "type": "integer" + }, + "unclassified": { + "type": "integer" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 2460fe3d..8a83f4c3 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2503,6 +2503,78 @@ } } }, + "/stats/paths": { + "get": { + "description": "Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads use materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and missing hours are omitted.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Received path-entry and hash-width distributions", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds (0 through 253402300799999)", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; at most 30 days after since", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/payload-breakdown": { "get": { "produces": [ @@ -4490,6 +4562,101 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth": { + "type": "object", + "properties": { + "bytes": { + "type": "integer" + }, + "receptions": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour": { + "type": "object", + "properties": { + "empty": { + "type": "integer" + }, + "hour": { + "type": "integer" + }, + "oneByte": { + "type": "integer" + }, + "receptions": { + "type": "integer" + }, + "threeByte": { + "type": "integer" + }, + "trace": { + "type": "integer" + }, + "twoByte": { + "type": "integer" + }, + "unclassified": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin": { + "type": "object", + "properties": { + "entries": { + "type": "integer" + }, + "receptions": { + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats": { + "type": "object", + "properties": { + "empty": { + "type": "integer" + }, + "hashWidths": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth" + } + }, + "hashed": { + "type": "integer" + }, + "hourly": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour" + } + }, + "pathLengths": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin" + } + }, + "receptions": { + "type": "integer" + }, + "since": { + "type": "integer" + }, + "trace": { + "type": "integer" + }, + "unclassified": { + "type": "integer" + }, + "until": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index be2d5b2b..f03b978b 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -966,6 +966,68 @@ definitions: nextCursor: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth: + properties: + bytes: + type: integer + receptions: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour: + properties: + empty: + type: integer + hour: + type: integer + oneByte: + type: integer + receptions: + type: integer + threeByte: + type: integer + trace: + type: integer + twoByte: + type: integer + unclassified: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin: + properties: + entries: + type: integer + receptions: + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats: + properties: + empty: + type: integer + hashWidths: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHashWidth' + type: array + hashed: + type: integer + hourly: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathHour' + type: array + pathLengths: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathLengthBin' + type: array + receptions: + type: integer + since: + type: integer + trace: + type: integer + unclassified: + type: integer + until: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.PayloadBreakdownItem: properties: count: @@ -3039,6 +3101,63 @@ paths: summary: Network overview stats (last 24h) tags: - Stats + /stats/paths: + get: + description: Counts stored receptions in [since, until), at most 30 days. Hash + widths count only validated nonempty ordinary header paths (1/2/3 bytes). + Empty paths do not vote for a width. TRACE header paths contain signal readings + and are separate. Missing payload type or inconsistent/unsupported path metadata + is unclassified. The four categories partition total receptions. Flood paths + accumulate entries; direct paths contain remaining entries, so counts do not + measure distance or a complete traversed route. Both boundaries round down + to UTC hours and the response reports that effective window. Reads use materialized + snapshots refreshed by background.view_refresh; the current partial hour is + excluded and missing hours are omitted. + parameters: + - description: Inclusive start, epoch milliseconds (0 through 253402300799999) + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; at most 30 days after since + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.PathStats' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Received path-entry and hash-width distributions + tags: + - Stats /stats/payload-breakdown: get: parameters: diff --git a/internal/api/handlers/paths.go b/internal/api/handlers/paths.go new file mode 100644 index 00000000..58f5acb4 --- /dev/null +++ b/internal/api/handlers/paths.go @@ -0,0 +1,68 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// getPathStats godoc +// +// @Summary Received path-entry and hash-width distributions +// @Description Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads use materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and missing hours are omitted. +// @Tags Stats +// @Produce json +// @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" +// @Param until query int true "Exclusive end, epoch milliseconds; at most 30 days after since" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.PathStats +// @Failure 400 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/paths [get] +func getPathStats(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + since, until, err := parseStatsWindow(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondError(w, http.StatusBadRequest, "region not found") + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} + } + } + stats, err := reader.GetPathStats(ctx, since, until, iatas) + switch { + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "path query timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("path stats failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, stats) + } + } +} diff --git a/internal/api/handlers/paths_test.go b/internal/api/handlers/paths_test.go new file mode 100644 index 00000000..64473463 --- /dev/null +++ b/internal/api/handlers/paths_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type pathReader struct { + stubReader + read func(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) +} + +func (s pathReader) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + return s.read(ctx, since, until, iatas) +} + +func TestPathsRejectInvalidWindow(t *testing.T) { + for _, query := range []string{"", "since=0", "since=x&until=1", "since=-1&until=1", "since=1&until=1", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { + w := httptest.NewRecorder() + StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) + if w.Code != 400 { + t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) + } + } +} + +func TestPathsFiltersErrorsAndCancellation(t *testing.T) { + for _, tc := range []struct { + name, query string + iatas []string + err error + status int + }{ + {"global", "", nil, nil, 200}, {"IATAs", "&iatas=yvr,yyj", []string{"YVR", "YYJ"}, nil, 200}, + {"region", "®ion=west", []string{"YVR"}, nil, 200}, {"empty region", "®ion=empty", []string{""}, nil, 200}, + {"missing region", "®ion=missing", nil, nil, 400}, {"timeout", "", nil, context.DeadlineExceeded, 503}, + {"database error", "", nil, errors.New("private detail"), 500}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := pathReader{stubReader: stubReader{getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + if slug == "missing" { + return nil, errors.New("not found") + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }}, read: func(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + if d, ok := ctx.Deadline(); !ok || time.Until(d) > 15*time.Second { + t.Fatal("query has no bounded deadline") + } + if since.UnixMilli() != 0 || until.UnixMilli() != 2592000000 || !reflect.DeepEqual(iatas, tc.iatas) { + t.Fatalf("unexpected arguments: %v %v %v", since, until, iatas) + } + return &api.PathStats{Until: until.UnixMilli(), Receptions: 4, Hashed: 1, Empty: 1, Trace: 1, Unclassified: 1}, tc.err + }} + w := httptest.NewRecorder() + StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) + if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + if tc.status == 200 { + var got api.PathStats + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || got.Receptions != 4 || got.Trace != 1 { + t.Fatalf("response: %+v %v", got, err) + } + } + }) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + called := false + r := pathReader{read: func(ctx context.Context, _ time.Time, _ time.Time, _ []string) (*api.PathStats, error) { + called = true + return nil, ctx.Err() + }} + w := httptest.NewRecorder() + StatsRouter(r).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) + if !called || w.Body.Len() != 0 { + t.Fatal("cancellation did not reach reader") + } +} diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index 56b7b0a8..1309efda 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -31,6 +31,7 @@ func StatsRouter(reader api.Reader) http.Handler { r.Get("/overview", getStatsOverview(reader)) r.Get("/observations", getStatsObservations(reader)) r.Get("/signal", getSignalStats(reader)) + r.Get("/paths", getPathStats(reader)) r.Get("/observer-comparison", getObserverComparison(reader)) r.Get("/payload-breakdown", getStatsPayloadBreakdown(reader)) r.Get("/top-nodes", getStatsTopNodes(reader)) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 644d6c03..42200c1a 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -70,6 +70,10 @@ func (s stubReader) GetSignalStats(context.Context, time.Time, time.Time, []stri return nil, nil } +func (s stubReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { + return nil, nil +} + func (s stubReader) GetObserverComparison(ctx context.Context, a, b uuid.UUID, since, until time.Time, iatas []string) (*api.ObserverComparison, error) { if s.getObserverComparison != nil { return s.getObserverComparison(ctx, a, b, since, until, iatas) diff --git a/internal/api/paths.go b/internal/api/paths.go new file mode 100644 index 00000000..86c3a00d --- /dev/null +++ b/internal/api/paths.go @@ -0,0 +1,46 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +// PathStats counts stored receptions in [Since, Until), not unique packets or nodes. +// Hashed + Empty + Trace + Unclassified partitions Receptions. Empty paths never +// vote for a hash width. TRACE header paths contain signal readings, not hashes. +// Since and Until are the effective UTC-hour boundaries of the materialized snapshot. +type PathStats struct { + Since int64 `json:"since"` + Until int64 `json:"until"` + Receptions int64 `json:"receptions"` + Hashed int64 `json:"hashed"` + Empty int64 `json:"empty"` + Trace int64 `json:"trace"` + Unclassified int64 `json:"unclassified"` + HashWidths []PathHashWidth `json:"hashWidths"` + PathLengths []PathLengthBin `json:"pathLengths"` + Hourly []PathHour `json:"hourly"` +} + +type PathHashWidth struct { + Bytes int32 `json:"bytes"` + Receptions int64 `json:"receptions"` +} + +// PathLengthBin counts entries in validated ordinary header paths, including empty. +// Flood paths accumulate entries; direct paths contain remaining route entries. +// These counts cannot establish distance or a complete end-to-end hop count. +type PathLengthBin struct { + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +// PathHour is a UTC hour clipped to the requested window. Missing hours are omitted. +type PathHour struct { + Hour int64 `json:"hour"` + Receptions int64 `json:"receptions"` + OneByte int64 `json:"oneByte"` + TwoByte int64 `json:"twoByte"` + ThreeByte int64 `json:"threeByte"` + Empty int64 `json:"empty"` + Trace int64 `json:"trace"` + Unclassified int64 `json:"unclassified"` +} diff --git a/internal/api/reader.go b/internal/api/reader.go index 90593549..8aa3f1af 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -164,6 +164,9 @@ type Reader interface { // GetSignalStats aggregates retained reception readings in [since, until). GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*SignalStats, error) + // GetPathStats counts received path entries and validated ordinary hash widths. + GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*PathStats, error) + // GetStatsPayloadBreakdown returns observation counts grouped by payload type. // Pass nil for iatas to return stats across all IATAs. // since defines the start of the window; pass zero time for default (last 24h). diff --git a/internal/background/tasks.go b/internal/background/tasks.go index 0290df8a..64c4297d 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -23,6 +23,7 @@ type viewRefresher interface { RefreshRadioPresets(context.Context) error RefreshObserverActivity(context.Context) error RefreshSignalStats(context.Context) error + RefreshPathStats(context.Context) error } // ViewRefreshTask returns a Task that refreshes all materialized views. @@ -45,6 +46,7 @@ func ViewRefreshTask(store viewRefresher, interval time.Duration) Task { {"radio presets", store.RefreshRadioPresets}, {"observer activity", store.RefreshObserverActivity}, {"signal stats", store.RefreshSignalStats}, + {"path stats", store.RefreshPathStats}, } { if err := view.refresh(ctx); err != nil { errs = append(errs, fmt.Errorf("%s: %w", view.name, err)) diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go index 2c4526dd..483f8586 100644 --- a/internal/background/tasks_test.go +++ b/internal/background/tasks_test.go @@ -123,6 +123,10 @@ func (s *refreshStub) RefreshSignalStats(ctx context.Context) error { return s.refresh(ctx, "signal stats") } +func (s *refreshStub) RefreshPathStats(ctx context.Context) error { + return s.refresh(ctx, "path stats") +} + func TestViewRefreshTask(t *testing.T) { first, second := errors.New("first failure"), errors.New("second failure") for _, tc := range []struct { @@ -132,13 +136,14 @@ func TestViewRefreshTask(t *testing.T) { {"success", nil}, {"partial failure", map[string]error{"hourly stats": first, "top talkers": second}}, {"signal failure", map[string]error{"signal stats": first}}, + {"path failure", map[string]error{"path stats": second}}, } { t.Run(tc.name, func(t *testing.T) { store := &refreshStub{errs: tc.errs} task := ViewRefreshTask(store, time.Minute) err := task.Run(context.Background()) - if len(store.calls) != 9 { - t.Fatalf("refreshed %d views, want 9", len(store.calls)) + if len(store.calls) != 10 { + t.Fatalf("refreshed %d views, want 10", len(store.calls)) } if len(tc.errs) == 0 && err != nil { t.Fatal(err) diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 4d7ec1b6..e58f0544 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -35,6 +35,10 @@ func (s *stubReader) GetSignalStats(context.Context, time.Time, time.Time, []str return nil, s.err } +func (s *stubReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { + return nil, s.err +} + func (s *stubReader) ListIATAs(_ context.Context) ([]api.IATA, error) { s.calls++ return s.iatas, s.err diff --git a/internal/cache/paths.go b/internal/cache/paths.go new file mode 100644 index 00000000..f5951cb3 --- /dev/null +++ b/internal/cache/paths.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func (cr *CachedReader) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + sorted := append([]string{}, iatas...) + slices.Sort(sorted) + sorted = slices.Compact(sorted) + segment, _ := json.Marshal(sorted) // [] and [""] must remain distinct (global vs empty region). + key := fmt.Sprintf("%s%s:%d:%d", keyPathStatsPrefix, segment, since.UnixMilli(), until.UnixMilli()) + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.PathStats, error) { + return cr.inner.GetPathStats(ctx, since, until, iatas) + }) +} diff --git a/internal/cache/paths_test.go b/internal/cache/paths_test.go new file mode 100644 index 00000000..fdffe97e --- /dev/null +++ b/internal/cache/paths_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type pathStatsReader struct { + api.Reader + calls int64 +} + +func (r *pathStatsReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { + r.calls++ + return &api.PathStats{Receptions: r.calls}, nil +} + +func TestPathCacheSeparatesWindowAndRegion(t *testing.T) { + c, mr := newTestClient(t) + inner := &pathStatsReader{} + r := NewCachedReader(inner, c, CacheTTLs{Stats: time.Minute}) + for _, tc := range []struct { + iatas []string + since, until, want int64 + }{ + {nil, 0, 10, 1}, {[]string{}, 0, 10, 1}, {[]string{""}, 0, 10, 2}, {[]string{"YVR"}, 0, 10, 3}, + {[]string{"YVR", "YYJ"}, 0, 10, 4}, {[]string{"YYJ", "YVR", "YYJ"}, 0, 10, 4}, + {nil, 1, 10, 5}, {nil, 0, 11, 6}, {nil, 0, 10, 1}, + } { + before := slices.Clone(tc.iatas) + got, err := r.GetPathStats(context.Background(), time.UnixMilli(tc.since), time.UnixMilli(tc.until), tc.iatas) + if err != nil || got.Receptions != tc.want || !slices.Equal(before, tc.iatas) { + t.Fatalf("%+v: %+v %v", tc, got, err) + } + } + mr.FastForward(time.Minute) + got, err := r.GetPathStats(context.Background(), time.UnixMilli(0), time.UnixMilli(10), nil) + if err != nil || got.Receptions != 7 { + t.Fatalf("expiry: %+v %v", got, err) + } +} diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 5d846c6b..65551da8 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -30,6 +30,7 @@ const ( keyStatsOverviewPrefix = "beacon:stats:overview:" keyStatsObservationsPrefix = "beacon:stats:observations:" keySignalStatsPrefix = "beacon:stats:signal:" + keyPathStatsPrefix = "beacon:stats:paths:" keyStatsBreakdownPrefix = "beacon:stats:breakdown:" keyStatsTopNodesPrefix = "beacon:stats:top-nodes:" keyStatsTopObsPrefix = "beacon:stats:top-observers:" From 20691dc5a3e1eecbb59ebcf0ac0971877a16c26d Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 23 Sep 2026 20:18:45 -0400 Subject: [PATCH 049/141] feat(backup): verify native archives offline without extraction (#160) --- cmd/beacon-backup/main.go | 81 +++++-- cmd/beacon-backup/main_test.go | 82 +++++++ docs/backup-export.md | 61 +++-- internal/backup/export_integration_test.go | 15 +- internal/backup/verify.go | 205 ++++++++++++++++ internal/backup/verify_test.go | 257 +++++++++++++++++++++ 6 files changed, 672 insertions(+), 29 deletions(-) create mode 100644 cmd/beacon-backup/main_test.go create mode 100644 internal/backup/verify.go create mode 100644 internal/backup/verify_test.go diff --git a/cmd/beacon-backup/main.go b/cmd/beacon-backup/main.go index 7e593e37..6484382a 100644 --- a/cmd/beacon-backup/main.go +++ b/cmd/beacon-backup/main.go @@ -1,13 +1,15 @@ // Copyright 2026 Beacon Contributors // SPDX-License-Identifier: AGPL-3.0-or-later -// beacon-backup creates a private database and saved-config export. +// beacon-backup creates or verifies a private database and saved-config export. package main import ( "context" + "errors" "flag" "fmt" + "io" "os" "os/signal" "syscall" @@ -18,22 +20,73 @@ import ( var version = "dev" func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + code := run(ctx, os.Args[1:], os.Stdout, os.Stderr) + stop() + os.Exit(code) +} + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) int { var opts backup.Options - flag.StringVar(&opts.ConfigPath, "config", "config.yaml", "saved YAML file to include verbatim (may contain secrets)") - flag.StringVar(&opts.OutputPath, "output", "", "new private .tar.gz destination (required; never overwritten)") - flag.Int64Var(&opts.MaxBytes, "max-bytes", backup.DefaultMaxBytes, "maximum uncompressed database dump size") - flag.DurationVar(&opts.Timeout, "timeout", backup.DefaultTimeout, "maximum export duration") - flag.Parse() - if flag.NArg() != 0 { - fmt.Fprintln(os.Stderr, "unexpected positional arguments; connection settings use PG* environment variables") - os.Exit(2) + var verifyPath string + flags := flag.NewFlagSet("beacon-backup", flag.ContinueOnError) + flags.SetOutput(stderr) + flags.StringVar(&opts.ConfigPath, "config", "config.yaml", "saved YAML file to export verbatim (may contain secrets)") + flags.StringVar(&opts.OutputPath, "output", "", "new private .tar.gz export destination (never overwritten)") + flags.StringVar(&verifyPath, "verify", "", "verify an existing .tar.gz without extraction or database access") + flags.Int64Var(&opts.MaxBytes, "max-bytes", backup.DefaultMaxBytes, "maximum uncompressed database dump size") + flags.DurationVar(&opts.Timeout, "timeout", backup.DefaultTimeout, "maximum export or verification duration") + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if flags.NArg() != 0 || opts.Timeout <= 0 { + fmt.Fprintln(stderr, "positive timeout and no positional arguments required") + return 2 + } + verifyMode, exportFlag := false, false + flags.Visit(func(f *flag.Flag) { + verifyMode = verifyMode || f.Name == "verify" + exportFlag = exportFlag || f.Name == "config" || f.Name == "output" + }) + if verifyMode { + if verifyPath == "" || exportFlag { + fmt.Fprintln(stderr, "verify requires an archive path and cannot be combined with config or output") + return 2 + } + // Reject directories/devices/FIFOs before opening, and check the opened + // descriptor too. A symlink to a regular, operator-selected file is fine. + info, err := os.Stat(verifyPath) + if err != nil || !info.Mode().IsRegular() { + fmt.Fprintln(stderr, "verification requires a readable regular archive file") + return 1 + } + file, err := os.Open(verifyPath) + if err != nil { + fmt.Fprintln(stderr, "cannot open backup archive") + return 1 + } + defer file.Close() + if info, err = file.Stat(); err != nil || !info.Mode().IsRegular() { + fmt.Fprintln(stderr, "verification requires a regular archive file") + return 1 + } + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) + defer cancel() + if _, err := backup.Verify(ctx, file, opts.MaxBytes); err != nil { + fmt.Fprintln(stderr, "verification failed:", err) + return 1 + } + fmt.Fprintln(stdout, "Archive structure and checksums verified. No files extracted or SQL executed.") + return 0 } opts.Version = version - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() if err := backup.Export(ctx, opts); err != nil { - fmt.Fprintln(os.Stderr, "backup failed:", err) - os.Exit(1) + fmt.Fprintln(stderr, "backup failed:", err) + return 1 } - fmt.Println("Backup complete. Store this bundle privately; it may contain keys and message data.") + fmt.Fprintln(stdout, "Backup complete. Store this bundle privately; it may contain keys and message data.") + return 0 } diff --git a/cmd/beacon-backup/main_test.go b/cmd/beacon-backup/main_test.go new file mode 100644 index 00000000..e55caf6a --- /dev/null +++ b/cmd/beacon-backup/main_test.go @@ -0,0 +1,82 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/backup" +) + +func TestVerifyCommandIsReadOnlyAndOffline(t *testing.T) { + dir := t.TempDir() + t.Setenv("PATH", dir) // no pg_dump, psql or shell helper is available + t.Setenv("PGDATABASE", "") + archive := filepath.Join(dir, "private.tar.gz") + m := backup.Manifest{FormatVersion: 1, CreatedAt: time.Now().UTC(), DatabaseFormat: "postgresql-plain-sql", ToolVersion: "PRIVATE_CANARY"} + data := map[string][]byte{"database.sql": []byte("SELECT 'PRIVATE_CANARY';"), "config.yaml": {}} + for _, name := range []string{"database.sql", "config.yaml"} { + hash := sha256.Sum256(data[name]) + m.Files = append(m.Files, backup.File{Name: name, Size: int64(len(data[name])), SHA256: hex.EncodeToString(hash[:])}) + } + data["manifest.json"], _ = json.Marshal(m) + var raw bytes.Buffer + gz := gzip.NewWriter(&raw) + tw := tar.NewWriter(gz) + for _, name := range []string{"manifest.json", "database.sql", "config.yaml"} { + if err := tw.WriteHeader(&tar.Header{Name: name, Size: int64(len(data[name])), Mode: 0600}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(data[name]); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(archive, raw.Bytes(), 0600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + args []string + code int + }{ + {[]string{"-verify", archive}, 0}, + {[]string{"-verify", archive, "-max-bytes", "1"}, 1}, + {[]string{"-verify", archive, "-timeout", "0"}, 2}, + {[]string{"-verify", archive, "-output", filepath.Join(dir, "must-not-exist")}, 2}, + {[]string{"-verify", archive, "-config", "missing.yaml"}, 2}, + {[]string{"-verify", "", "-output", filepath.Join(dir, "must-not-exist")}, 2}, + {[]string{"-verify", dir}, 1}, + {[]string{"-verify", filepath.Join(dir, "PRIVATE_CANARY-missing")}, 1}, + {[]string{"-verify", archive, "unexpected"}, 2}, + } { + var stdout, stderr bytes.Buffer + if got := run(context.Background(), tc.args, &stdout, &stderr); got != tc.code || strings.Contains(stdout.String()+stderr.String(), "PRIVATE_CANARY") { + t.Fatalf("exit %d, want %d; diagnostic %s", got, tc.code, stderr.String()) + } + } + files, err := os.ReadDir(dir) + if err != nil || len(files) != 1 || files[0].Name() != "private.tar.gz" { + t.Fatal("verification extracted files or created output") + } + unchanged, err := os.ReadFile(archive) + if err != nil || !bytes.Equal(unchanged, raw.Bytes()) { + t.Fatal("input archive changed") + } +} diff --git a/docs/backup-export.md b/docs/backup-export.md index 160561f6..54551f05 100644 --- a/docs/backup-export.md +++ b/docs/backup-export.md @@ -1,13 +1,14 @@ -# Database and saved-config export +# Database and saved-config export and verification -`beacon-backup` is the export foundation for issue #72. It produces a private, -versioned `.tar.gz` using PostgreSQL's `pg_dump`. It does not yet provide a web -interface, account login, scheduled/remote storage or automatic import. +`beacon-backup` is the export and archive-validation foundation for issue #72. It +produces a private, versioned `.tar.gz` using PostgreSQL's `pg_dump` and can verify +an existing bundle offline. It does not yet provide a web interface, account +login, scheduled/remote storage or automatic import. -Build it with `go build ./cmd/beacon-backup`. Install `pg_dump` in the same runtime -as this command; an installation on the Docker host does not install it inside an -app container. Use a client of the same major version as the source PostgreSQL -server; an older client cannot dump a newer server. +Build it with `go build ./cmd/beacon-backup`. For exports, install `pg_dump` in the +same runtime as this command; an installation on the Docker host does not install +it inside an app container. Use a client of the same major version as the source +PostgreSQL server; an older client cannot dump a newer server. Set libpq's standard `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER` and TLS settings. `PGDATABASE` is required and must be a plain database name. Prefer a private @@ -115,12 +116,43 @@ waiting indefinitely behind schema changes. Export failure publishes no backup; check the client version, privileges, connection settings and available capacity privately. The command deliberately does not expose raw client diagnostics. +## Verify an existing archive + +```sh +beacon-backup -verify /private/beacon-20260913.tar.gz +``` + +Verification reads a regular file without extracting members, writing files, +executing SQL or connecting to PostgreSQL, Redis or MQTT. It needs neither a +configuration file nor `pg_dump` or connection credentials. Do not combine +`-verify` with the export-only `-config` or `-output` flags. + +The verifier accepts the native format-1 bundle: exactly the three fixed regular +members with 0600 modes and USTAR headers (or the size-only PAX header needed for +SQL over 8 GiB). It rejects duplicates, unexpected paths, links, devices, other +extended metadata, incomplete tar/gzip endings and trailing or concatenated +streams. Manifest fields must be known, with no duplicate keys or case aliases; +the payload sizes and SHA-256 hashes must match the bytes actually read. + +The same `-max-bytes` SQL limit applies, with a separate 1 MiB config limit and +64 KiB manifest limit. Compressed input is capped at the SQL limit plus 1% and +2 MiB, matching the exporter. Headers and decompressed data are bounded too; +SQL and config are hashed as streams, without keeping their contents in memory. +The ten-minute default `-timeout` and interrupt handling check cancellation +between reads; they cannot interrupt a blocked filesystem read. + +Exit status 0 means the archive structure and checksums passed. Failure returns +a nonzero status with no payload, manifest values or input path in its diagnostic. +The manifest is not signed: verification does **not** establish authenticity, +safe SQL, configuration validity or restorability. It does not make an untrusted +dump safe to restore, and does not provide import/overwrite or deployment recovery. + ## Restore verification -Use trusted bundles only: PostgreSQL dumps can contain executable SQL. Inspect -the fixed members, verify the gzip stream and the manifest's payload sizes and -hashes, and extract into private staging. For a **new, empty disposable database**, -with its own explicit `PGDATABASE` and target-role connection settings: +Use trusted bundles only: PostgreSQL dumps can contain executable SQL. Run the +archive verification above, then extract into private staging. For a +**new, empty disposable database**, with its own explicit `PGDATABASE` and +target-role connection settings: ```sh psql -X --set ON_ERROR_STOP=on --single-transaction --file database.sql @@ -137,8 +169,9 @@ PostgreSQL references: [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.h [password files](https://www.postgresql.org/docs/16/libpq-pgpass.html). CI runs the compiled command and its PostgreSQL round-trip test with a dedicated -PostgreSQL 16 service. To repeat it privately, build the command, set the `PG*` -connection variables for an isolated test server and set +PostgreSQL 16 service. The test also verifies the export offline with no client on +PATH or connection settings, and rejects a truncated copy. To repeat it privately, +build the command, set the `PG*` connection variables for an isolated test server and set `BEACON_BACKUP_TEST_POSTGRES=1` plus `BEACON_BACKUP_TEST_BINARY` to the command's absolute path. Run `go test ./internal/backup -run '^TestExportPostgres$' -v`. The test role needs permission to create/drop its two randomly named databases; diff --git a/internal/backup/export_integration_test.go b/internal/backup/export_integration_test.go index 886a16cb..42830573 100644 --- a/internal/backup/export_integration_test.go +++ b/internal/backup/export_integration_test.go @@ -115,6 +115,19 @@ CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM ba if err != nil { t.Fatal(err) } + verify := exec.CommandContext(ctx, binary, "-verify", output, "-max-bytes", "16777216", "-timeout", "30s") + verify.Env = []string{"PATH=" + dir} // verification needs no PostgreSQL client or connection + message, err := verify.CombinedOutput() + if err != nil || !bytes.Contains(message, []byte("structure and checksums verified")) || bytes.Contains(message, []byte("café")) { + t.Fatal("exact command could not verify its export offline") + } + corrupt := filepath.Join(dir, "corrupt.tar.gz") + if err := os.WriteFile(corrupt, original[:len(original)-1], 0600); err != nil { + t.Fatal(err) + } + if err := exec.CommandContext(ctx, binary, "-verify", corrupt).Run(); err == nil { + t.Fatal("exact command accepted a truncated export") + } for _, failure := range []string{"existing", "size", "connection", "timeout"} { t.Run(failure, func(t *testing.T) { failedOutput := filepath.Join(dir, failure+".tar.gz") @@ -212,5 +225,5 @@ CREATE MATERIALIZED VIEW backup_materialized AS SELECT count(*) AS count FROM ba if err := target.QueryRow(ctx, "INSERT INTO backup_fixture (message) VALUES ('next') RETURNING id").Scan(&nextID); err != nil || nextID != 3 { t.Fatal("identity sequence did not survive restore") } - t.Log("exact command restored all migrations, columns, Unicode/bytea/JSON data, relationships, views and identity sequence") + t.Log("exact command verified its export offline and restored all migrations, columns, Unicode/bytea/JSON data, relationships, views and identity sequence") } diff --git a/internal/backup/verify.go b/internal/backup/verify.go new file mode 100644 index 00000000..f2bf52ae --- /dev/null +++ b/internal/backup/verify.go @@ -0,0 +1,205 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "strings" +) + +const maxManifestBytes = 64 << 10 + +var ErrInvalidArchive = errors.New("invalid backup archive") + +// Verify streams a format-1 bundle without extracting files or executing SQL. +// maxBytes bounds the SQL payload (up to 1 TiB); the saved YAML and manifest have +// separate fixed limits. The caller owns input and its deadline. Cancellation is +// checked between reads; the supplied Reader must not block indefinitely. +// The returned manifest is untrusted metadata, not proof of origin or restorability. +func Verify(ctx context.Context, input io.Reader, maxBytes int64) (*Manifest, error) { + if input == nil || maxBytes <= 0 || maxBytes > 1<<40 { + return nil, errors.New("input and positive max-bytes (at most 1 TiB) are required") + } + // Match the exporter's compressed-output allowance. ByteReader prevents gzip + // from swallowing a following stream before we can reject it. + compressed := &verifyReader{ctx: ctx, r: input, limit: maxBytes + maxBytes/100 + 2*maxConfigBytes} + buffered := bufio.NewReader(compressed) + gz, err := gzip.NewReader(buffered) + if err != nil { + return nil, verificationError(ctx, err, "gzip header") + } + defer gz.Close() + gz.Multistream(false) + decoded := &verifyReader{ctx: ctx, r: gz} + tr := tar.NewReader(decoded) + files := make(map[string]File, 3) + var metadata []byte + var padding int64 + for { + before := decoded.n + // Native headers are USTAR, or one size-only PAX record for SQL >8 GiB. + // Bound Next itself: it otherwise hides and allocates extension records. + decoded.limit = before + padding + 3*512 + h, err := tr.Next() + if err == io.EOF { + // tar.Reader also accepts EOF without both closing zero blocks. + if decoded.n-before != padding+2*512 { + return nil, verificationError(ctx, nil, "missing tar terminator") + } + break + } + if err != nil { + return nil, verificationError(ctx, err, "tar header") + } + headerBytes := decoded.n - before - padding + ordinary := h.Format == tar.FormatUSTAR && headerBytes == 512 && len(h.PAXRecords) == 0 + large := h.Format == tar.FormatPAX && headerBytes == 3*512 && len(h.PAXRecords) == 1 && h.PAXRecords["size"] == strconv.FormatInt(h.Size, 10) + if (!ordinary && !large) || (h.Typeflag != tar.TypeReg && h.Typeflag != tar.TypeRegA) || h.Linkname != "" || h.Mode != 0600 { + return nil, verificationError(ctx, nil, "unsupported member type or metadata") + } + if _, duplicate := files[h.Name]; duplicate || len(files) == 3 { + return nil, verificationError(ctx, nil, "duplicate or extra member") + } + var limit int64 + switch h.Name { + case "manifest.json": + limit = maxManifestBytes + case "database.sql": + limit = maxBytes + if h.Size == 0 { + return nil, verificationError(ctx, nil, "empty database dump") + } + case "config.yaml": + limit = maxConfigBytes + default: + return nil, verificationError(ctx, nil, "unexpected member name") + } + if h.Size < 0 || h.Size > limit { + return nil, ErrTooLarge + } + decoded.limit = decoded.n + h.Size + digest := sha256.New() + if h.Name == "manifest.json" { + metadata, err = io.ReadAll(tr) + } else { + _, err = io.Copy(digest, tr) + } + if err != nil { + return nil, verificationError(ctx, err, "member data") + } + files[h.Name] = File{Name: h.Name, Size: h.Size, SHA256: hex.EncodeToString(digest.Sum(nil))} + padding = (512 - h.Size%512) % 512 + } + // Read through the gzip checksum/trailer, but allow no data after tar EOF. + decoded.limit = decoded.n + var tail [1]byte + if _, err := decoded.Read(tail[:]); err != io.EOF { + return nil, verificationError(ctx, err, "gzip completion or trailing tar data") + } + if _, err := buffered.ReadByte(); err != io.EOF { + return nil, verificationError(ctx, err, "trailing compressed data") + } + if len(files) != 3 || uniqueManifestKeys(json.NewDecoder(bytes.NewReader(metadata)), 0) != nil { + return nil, verificationError(ctx, nil, "missing members or invalid manifest") + } + var manifest Manifest + decoder := json.NewDecoder(bytes.NewReader(metadata)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&manifest); err != nil || decoder.Decode(new(any)) != io.EOF || manifest.FormatVersion != 1 || manifest.DatabaseFormat != "postgresql-plain-sql" || manifest.CreatedAt.IsZero() || len(manifest.Files) != 2 { + return nil, verificationError(ctx, nil, "unsupported or incomplete manifest") + } + for _, declared := range manifest.Files { + actual, exists := files[declared.Name] + digest, err := hex.DecodeString(declared.SHA256) + if !exists || declared.Name == "manifest.json" || declared.Size != actual.Size || err != nil || len(digest) != sha256.Size || !strings.EqualFold(declared.SHA256, actual.SHA256) { + return nil, verificationError(ctx, nil, "manifest size or checksum mismatch") + } + delete(files, declared.Name) // duplicate manifest file records fail as well. + } + if err := ctx.Err(); err != nil { + return nil, err + } + return &manifest, nil +} + +// Avoid returning filenames, manifest values, reader errors or payload text. +func verificationError(ctx context.Context, err error, stage string) error { + if ctx.Err() != nil { + return ctx.Err() + } + if errors.Is(err, ErrTooLarge) { + return ErrTooLarge + } + return fmt.Errorf("%w: %s", ErrInvalidArchive, stage) +} + +type verifyReader struct { + ctx context.Context + r io.Reader + n, limit int64 +} + +func (r *verifyReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + if r.n > r.limit { + return 0, ErrTooLarge + } + if int64(len(p)) > r.limit-r.n+1 { + p = p[:r.limit-r.n+1] + } + n, err := r.r.Read(p) + r.n += int64(n) + if r.n > r.limit { + return 0, ErrTooLarge + } + return n, err +} + +// encoding/json accepts duplicate keys and case aliases. Reject both before +// decoding the fixed schema, so another reader cannot interpret different values. +func uniqueManifestKeys(d *json.Decoder, depth int) error { + if depth > 8 { + return ErrInvalidArchive + } + token, err := d.Token() + if err != nil { + return err + } + if delim, ok := token.(json.Delim); ok { + keys := map[string]bool{} + for d.More() { + if delim == '{' { + key, err := d.Token() + if err != nil { + return err + } + name, ok := key.(string) + // Schema keys are ASCII. Unicode simple-fold aliases (such as + // long-s for s) also match encoding/json fields, even in lowercase. + if !ok || keys[name] || strings.IndexFunc(name, func(r rune) bool { return r > 127 || r >= 'A' && r <= 'Z' }) >= 0 { + return ErrInvalidArchive + } + keys[name] = true + } + if err := uniqueManifestKeys(d, depth+1); err != nil { + return err + } + } + _, err = d.Token() + } + return err +} diff --git a/internal/backup/verify_test.go b/internal/backup/verify_test.go new file mode 100644 index 00000000..fb0e9f1e --- /dev/null +++ b/internal/backup/verify_test.go @@ -0,0 +1,257 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package backup + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "strings" + "testing" + "time" +) + +type verifyEntry struct { + name string + data []byte + kind byte + pax map[string]string +} + +func verifyFixture(t testing.TB) []verifyEntry { + t.Helper() + sql, config := []byte(testSQL), []byte("secret: PRIVATE_CANARY\n") + m := Manifest{FormatVersion: 1, CreatedAt: time.Now().UTC(), ToolVersion: "fixture", DatabaseFormat: "postgresql-plain-sql", Excluded: []string{"deployment_environment"}} + for _, entry := range []verifyEntry{{name: "database.sql", data: sql}, {name: "config.yaml", data: config}} { + hash := sha256.Sum256(entry.data) + m.Files = append(m.Files, File{Name: entry.name, Size: int64(len(entry.data)), SHA256: hex.EncodeToString(hash[:])}) + } + metadata, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return []verifyEntry{{name: "manifest.json", data: metadata}, {name: "database.sql", data: sql}, {name: "config.yaml", data: config}} +} + +func verifyTar(t testing.TB, entries []verifyEntry) []byte { + t.Helper() + var data bytes.Buffer + tw := tar.NewWriter(&data) + for _, e := range entries { + h := &tar.Header{Name: e.name, Size: int64(len(e.data)), Mode: 0600, Typeflag: e.kind} + if e.kind == tar.TypeSymlink || e.kind == tar.TypeLink { + h.Linkname = "../PRIVATE_CANARY" + } + if e.pax != nil { + h.Format, h.PAXRecords = tar.FormatPAX, e.pax + } + if err := tw.WriteHeader(h); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(e.data); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + return data.Bytes() +} + +func verifyGzip(t testing.TB, data []byte) []byte { + t.Helper() + var encoded bytes.Buffer + gz := gzip.NewWriter(&encoded) + if _, err := gz.Write(data); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return encoded.Bytes() +} + +func TestVerifyExportAndReorderedMembers(t *testing.T) { + opts := setup(t) + if err := export(context.Background(), opts, helper(t, "ok")); err != nil { + t.Fatal(err) + } + original, err := os.ReadFile(opts.OutputPath) + if err != nil { + t.Fatal(err) + } + m, err := Verify(context.Background(), bytes.NewReader(original), opts.MaxBytes) + if err != nil || m.ToolVersion != opts.Version || len(m.Files) != 2 { + t.Fatalf("native export: %v %v", m, err) + } + entries := verifyFixture(t) + entries[0], entries[2] = entries[2], entries[0] + if _, err := Verify(context.Background(), bytes.NewReader(verifyGzip(t, verifyTar(t, entries))), int64(len(testSQL))); err != nil { + t.Fatal("valid reordered members:", err) + } + if current, err := os.ReadFile(opts.OutputPath); err != nil || !bytes.Equal(original, current) { + t.Fatal("verification changed input") + } +} + +func TestVerifyRejectsInvalidMembersAndManifest(t *testing.T) { + for _, tc := range []struct { + name string + change func([]verifyEntry) []verifyEntry + }{ + {"missing", func(e []verifyEntry) []verifyEntry { return e[:2] }}, + {"duplicate", func(e []verifyEntry) []verifyEntry { return append(e, e[1]) }}, + {"traversal", func(e []verifyEntry) []verifyEntry { e[2].name = "../PRIVATE_CANARY"; return e }}, + {"absolute", func(e []verifyEntry) []verifyEntry { e[2].name = "/config.yaml"; return e }}, + {"windows path", func(e []verifyEntry) []verifyEntry { e[2].name = `C:\config.yaml`; return e }}, + {"directory prefix", func(e []verifyEntry) []verifyEntry { e[2].name = "./config.yaml"; return e }}, + {"symlink", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeSymlink, nil; return e }}, + {"hardlink", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeLink, nil; return e }}, + {"device", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeChar, nil; return e }}, + {"directory", func(e []verifyEntry) []verifyEntry { e[2].kind, e[2].data = tar.TypeDir, nil; return e }}, + {"payload tamper", func(e []verifyEntry) []verifyEntry { e[1].data[0] ^= 1; return e }}, + {"large sql", func(e []verifyEntry) []verifyEntry { e[1].data = bytes.Repeat([]byte("x"), len(testSQL)+1); return e }}, + {"large config", func(e []verifyEntry) []verifyEntry { e[2].data = bytes.Repeat([]byte("x"), maxConfigBytes+1); return e }}, + {"large manifest", func(e []verifyEntry) []verifyEntry { + e[0].data = bytes.Repeat([]byte("x"), maxManifestBytes+1) + return e + }}, + {"extended attributes", func(e []verifyEntry) []verifyEntry { + e[1].pax = map[string]string{"SCHILY.xattr.secret": "PRIVATE_CANARY"} + return e + }}, + {"hidden oversized metadata", func(e []verifyEntry) []verifyEntry { + e[1].pax = map[string]string{"comment": strings.Repeat("x", 8000)} + return e + }}, + {"unknown version", func(e []verifyEntry) []verifyEntry { + e[0].data = bytes.Replace(e[0].data, []byte(`"format_version":1`), []byte(`"format_version":2`), 1) + return e + }}, + {"duplicate key", func(e []verifyEntry) []verifyEntry { + e[0].data = append([]byte(`{"format_version":2,`), e[0].data[1:]...) + return e + }}, + {"case alias", func(e []verifyEntry) []verifyEntry { + e[0].data = bytes.Replace(e[0].data, []byte("format_version"), []byte("FORMAT_VERSION"), 1) + return e + }}, + {"unicode case alias", func(e []verifyEntry) []verifyEntry { + e[0].data = bytes.Replace(e[0].data, []byte("database_format"), []byte(`databa\u017fe_format`), 1) + return e + }}, + {"duplicate file record", func(e []verifyEntry) []verifyEntry { + var m Manifest + _ = json.Unmarshal(e[0].data, &m) + m.Files[1] = m.Files[0] + e[0].data, _ = json.Marshal(m) + return e + }}, + {"claimed size", func(e []verifyEntry) []verifyEntry { + var m Manifest + _ = json.Unmarshal(e[0].data, &m) + m.Files[0].Size++ + e[0].data, _ = json.Marshal(m) + return e + }}, + {"unknown field", func(e []verifyEntry) []verifyEntry { + e[0].data = append([]byte(`{"private":"PRIVATE_CANARY",`), e[0].data[1:]...) + return e + }}, + {"second json value", func(e []verifyEntry) []verifyEntry { e[0].data = append(e[0].data, []byte(`{}`)...); return e }}, + } { + t.Run(tc.name, func(t *testing.T) { + data := verifyGzip(t, verifyTar(t, tc.change(verifyFixture(t)))) + if _, err := Verify(context.Background(), bytes.NewReader(data), int64(len(testSQL))); err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") { + t.Fatalf("expected a sanitized rejection, got %v", err) + } + }) + } +} + +func TestVerifyFraming(t *testing.T) { + raw := verifyTar(t, verifyFixture(t)) + valid := verifyGzip(t, raw) + badCRC := bytes.Clone(valid) + badCRC[len(badCRC)-8] ^= 1 + cases := map[string][]byte{ + "not gzip": []byte("PRIVATE_CANARY"), "truncated gzip": valid[:len(valid)-1], "bad checksum": badCRC, + "no tar terminator": verifyGzip(t, raw[:len(raw)-1024]), "one tar terminator": verifyGzip(t, raw[:len(raw)-512]), + "truncated payload": verifyGzip(t, raw[:1500]), "trailing tar bytes": verifyGzip(t, append(bytes.Clone(raw), 0)), + "trailing compressed bytes": append(bytes.Clone(valid), 0), "second gzip stream": append(bytes.Clone(valid), verifyGzip(t, nil)...), + } + for name, data := range cases { + t.Run(name, func(t *testing.T) { + if _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes); err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") { + t.Fatalf("expected sanitized framing error, got %v", err) + } + }) + } +} + +func TestVerifySizeOnlyPAX(t *testing.T) { + // A native dump over 8 GiB requires a size-only PAX header. Generate the + // real header without allocating its body; verification must reach the + // truncated member, rather than reject the exporter's metadata format. + var raw bytes.Buffer + tw := tar.NewWriter(&raw) + if err := tw.WriteHeader(&tar.Header{Name: "database.sql", Mode: 0600, Size: 1 << 34}); err != nil { + t.Fatal(err) + } + h, err := tar.NewReader(bytes.NewReader(raw.Bytes())).Next() + if err != nil || h.Format != tar.FormatPAX || len(h.PAXRecords) != 1 || h.PAXRecords["size"] != "17179869184" { + t.Fatalf("fixture must contain native size-only PAX metadata: %v %v", h, err) + } + data := verifyGzip(t, raw.Bytes()) + if _, err := Verify(context.Background(), bytes.NewReader(data), 1<<35); !errors.Is(err, ErrInvalidArchive) || !strings.Contains(err.Error(), "member data") { + t.Fatalf("native PAX header was rejected before reading the payload: %v", err) + } + if _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes); !errors.Is(err, ErrTooLarge) { + t.Fatalf("PAX size did not respect the SQL limit: %v", err) + } +} + +func TestVerifyBoundsAndCancellation(t *testing.T) { + for _, limit := range []int64{0, -1, 1<<40 + 1} { + if _, err := Verify(context.Background(), bytes.NewReader(nil), limit); err == nil { + t.Fatal("invalid limit accepted") + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := Verify(ctx, bytes.NewReader(nil), 1); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + for _, size := range []int{2, 3} { + r := &verifyReader{ctx: context.Background(), r: bytes.NewReader(make([]byte, size)), limit: 2} + _, err := io.Copy(io.Discard, r) + if (size == 3 && !errors.Is(err, ErrTooLarge)) || (size == 2 && err != nil) { + t.Fatalf("read limit boundary: %d %v", size, err) + } + } +} + +func FuzzVerify(f *testing.F) { + f.Add([]byte("invalid gzip")) + f.Add(verifyGzip(f, verifyTar(f, verifyFixture(f)))) + f.Add(verifyTar(f, verifyFixture(f))) + f.Fuzz(func(t *testing.T, data []byte) { + if len(data) > 2<<20 { + t.Skip() + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, _ = Verify(ctx, bytes.NewReader(data), 4096) + // Fuzz tar/manifest parsing too, without every mutation first failing + // the outer gzip checksum. + _, _ = Verify(ctx, bytes.NewReader(verifyGzip(t, data)), 4096) + }) +} From c02317a4ac7228d19cab498edfa1d61186c84626 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 23 Sep 2026 20:18:48 -0400 Subject: [PATCH 050/141] feat(packets): summarize ACK and trace references (#161) --- db/packet_summary_integration_test.go | 39 +++++++++--- db/queries/queries.sql | 42 +++++++++++-- db/sqlc/queries.sql.go | 42 +++++++++++-- docs/packet-summaries.md | 28 +++++++++ internal/ingest/packet.go | 2 + internal/ingest/packet_summary_test.go | 83 ++++++++++++++++++++++++++ 6 files changed, 216 insertions(+), 20 deletions(-) create mode 100644 docs/packet-summaries.md diff --git a/db/packet_summary_integration_test.go b/db/packet_summary_integration_test.go index 7f3003d0..f27cf63a 100644 --- a/db/packet_summary_integration_test.go +++ b/db/packet_summary_integration_test.go @@ -59,7 +59,26 @@ FROM (VALUES (6,4,'{"type":"ADVERT","appData":{"name":null}}'), (7,4,'{"type":"ADVERT","appData":{"name":42}}'), (8,2,'{"appData":{"name":"not an advert"}}'), - (9,4,'{"type":"ADVERT","appData":[]}') + (9,4,'{"type":"ADVERT","appData":[]}'), + (10,3,'{"type":"ACK","checksum":"01020304"}'), + (11,3,'{"type":"ACK","checksum":"AABBCCDD"}'), + (12,3,'{"type":"ACK","checksum":"00000000"}'), + (13,3,'{"type":"ACK"}'), + (14,3,'{"type":"ACK","checksum":12345678}'), + (15,3,'{"type":"ACK","checksum":"gg112233"}'), + (16,3,'{"type":"ACK","checksum":"1234"}'), + (17,3,'{"type":"ACK","checksum":"123456789"}'), + (18,3,'{"type":"TRACE","checksum":"01020304"}'), + (19,9,'{"type":"TRACE","traceTag":"efbeadde","authCode":4277009102}'), + (20,9,'{"type":"PING","traceTag":"deadBEEF"}'), + (21,9,'{"type":"OTHER","traceTag":"01020304"}'), + (22,9,'{"traceTag":"01020304"}'), + (23,9,'{"type":"TRACE","traceTag":12345678}'), + (24,9,'{"type":"TRACE","traceTag":"01020304\n"}'), + (25,9,'{"type":"TRACE","traceTag":"gg112233"}'), + (26,9,'{"type":"TRACE","traceTag":null}'), + (27,2,'{"type":"ACK","checksum":"01020304"}'), + (28,9,'{"type":"TRACE","traceTag":"00000000"}') ) v(id,kind,parsed); `) if err != nil { @@ -69,11 +88,15 @@ FROM (VALUES store := &Store{q: sqlc.New(counter)} check := func(items []api.PacketSummary) { t.Helper() - if len(items) != 9 { - t.Fatalf("got %d packets, want 9", len(items)) + if len(items) != 28 { + t.Fatalf("got %d packets, want 28", len(items)) } for _, item := range items { - want := map[string]string{"01": "MD00-Repeater", "02": "Relay 📡"}[item.PacketHash] + want := map[string]string{ + "01": "MD00-Repeater", "02": "Relay 📡", + "0a": "ACK 01020304", "0b": "ACK aabbccdd", "0c": "ACK 00000000", + "13": "TRACE efbeadde", "14": "PING deadbeef", "1c": "TRACE 00000000", + }[item.PacketHash] if want == "" { if item.Summary != nil { t.Fatalf("unexpected summary for %s: %q", item.PacketHash, *item.Summary) @@ -87,7 +110,7 @@ FROM (VALUES } counter.calls = 0 } - page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 20) + page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 50) if err != nil { t.Fatal(err) } @@ -96,18 +119,18 @@ FROM (VALUES INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) SELECT id,decode(lpad(to_hex(id),2,'0'),'hex'),'00000000-0000-0000-0000-000000000001','YVR', - '2026-01-01'::timestamptz+id*interval '1 second',0,1,0 FROM generate_series(1,9) id; + '2026-01-01'::timestamptz+id*interval '1 second',0,1,0 FROM generate_series(1,28) id; `) if err != nil { t.Fatal(err) } for _, iatas := range [][]string{nil, {"YVR"}} { - page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 20) + page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 50) if err != nil { t.Fatal(err) } check(page.Items) - rows, err := store.ListPacketsAfterID(ctx, 0, -1, -1, iatas, "", 20) + rows, err := store.ListPacketsAfterID(ctx, 0, -1, -1, iatas, "", 50) if err != nil { t.Fatal(err) } diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 7ca704d2..614c92c1 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -463,8 +463,18 @@ SELECT COUNT(*) FROM packet_observations WHERE packet_hash = $1; SELECT p.packet_hash, p.payload_type, - COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' - THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, @@ -560,8 +570,18 @@ page AS ( SELECT p.packet_hash, p.payload_type, - COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' - THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, @@ -600,8 +620,18 @@ ORDER BY sh.site_heard_at DESC; SELECT p.packet_hash, p.payload_type, - COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' - THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index e912b9f1..16e76c4f 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -3157,8 +3157,18 @@ const listPackets = `-- name: ListPackets :many SELECT p.packet_hash, p.payload_type, - COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' - THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, @@ -3278,8 +3288,18 @@ const listPacketsAfterID = `-- name: ListPacketsAfterID :many SELECT p.packet_hash, p.payload_type, - COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' - THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, @@ -3427,8 +3447,18 @@ page AS ( SELECT p.packet_hash, p.payload_type, - COALESCE(CASE WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' - THEN p.parsed_payload #>> '{appData,name}' END, '')::text AS summary, + COALESCE(CASE + WHEN p.payload_type = 4 AND jsonb_typeof(p.parsed_payload #> '{appData,name}') = 'string' + THEN p.parsed_payload #>> '{appData,name}' + WHEN p.payload_type = 3 AND p.parsed_payload ->> 'type' = 'ACK' + AND jsonb_typeof(p.parsed_payload -> 'checksum') = 'string' + AND p.parsed_payload ->> 'checksum' ~ '^[0-9a-fA-F]{8}$' + THEN 'ACK ' || lower(p.parsed_payload ->> 'checksum') + WHEN p.payload_type = 9 AND p.parsed_payload ->> 'type' IN ('TRACE', 'PING') + AND jsonb_typeof(p.parsed_payload -> 'traceTag') = 'string' + AND p.parsed_payload ->> 'traceTag' ~ '^[0-9a-fA-F]{8}$' + THEN (p.parsed_payload ->> 'type') || ' ' || lower(p.parsed_payload ->> 'traceTag') + END, '')::text AS summary, p.route_type, p.first_heard_at, p.last_heard_at, diff --git a/docs/packet-summaries.md b/docs/packet-summaries.md new file mode 100644 index 00000000..e954765c --- /dev/null +++ b/docs/packet-summaries.md @@ -0,0 +1,28 @@ +# Packet summaries + +Packet lists, regional lists, reconnect backfill and live `packetObservation` +events share an optional `summary` string. It describes metadata carried by that +packet, rather than looking up a node's current name or decoding history again. + +| Payload | Summary | Source | +|---|---|---| +| ADVERT | Advertised name | Saved `appData.name` / verified decoded advert | +| ACK | `ACK 01020304` | Four-byte acknowledgement checksum | +| TRACE | `TRACE efbeadde` | Trace tag | +| TRACE classified as PING | `PING efbeadde` | Trace tag, with the existing single-destination classification | + +References use eight lowercase hexadecimal characters in the same byte order +as packet detail's `checksum` / `traceTag`. Extended ACK retry bytes do not change +the displayed checksum. Zero is a valid reference. These short references are +not globally unique packet identifiers or proof of delivery/authentication; no +message body, trace auth code or inferred sender/recipient is added to the summary. + +Historical reads require the expected stored type and a string containing +exactly eight hex characters. Missing, malformed or unsupported references omit +the field. Advert behavior is unchanged. List/backfill queries project the text +from saved JSON in their existing query, without a schema change, backfill job +or per-packet database lookup. The existing web summary display needs no new API +field or client release to render these values. + +Other packet types remain separate follow-ups under issue #99. A missing summary +does not mean the packet is invalid or unreadable in packet detail. diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 3242844b..d5176c1d 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -564,6 +564,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ SNRValues: snrValues, } parsedPayload, _ = json.Marshal(pt) + summary = new(traceType + " " + pt.TraceTag) // Each consecutive hop pair becomes a node_neighbors edge: // hop[i]'s measured SNR of receiving from hop[i-1]. hop[0]'s @@ -600,6 +601,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ Checksum: hex.EncodeToString(uint32ToBytes(ack.CRC())), } parsedPayload, _ = json.Marshal(pa) + summary = new("ACK " + pa.Checksum) } case meshcore.PayloadTypeMultiPart: diff --git a/internal/ingest/packet_summary_test.go b/internal/ingest/packet_summary_test.go index 07003355..887c3960 100644 --- a/internal/ingest/packet_summary_test.go +++ b/internal/ingest/packet_summary_test.go @@ -114,3 +114,86 @@ func TestAdvertSummaryLive(t *testing.T) { }) } } + +func TestPacketReferenceSummaryLive(t *testing.T) { + trace := func(hashes []byte) []byte { + t.Helper() + payload, err := (&meshcore.Trace{Tag: 0xdeadbeef, AuthCode: 0xfeedface, PathHashes: hashes}).ToBytes() + if err != nil { + t.Fatal(err) + } + return payload + } + for _, tc := range []struct { + name string + kind byte + payload []byte + want string + field string + }{ + {"ack", meshcore.PayloadTypeAck, []byte{1, 2, 3, 4}, "ACK 01020304", "checksum"}, + {"extended ack", meshcore.PayloadTypeAck, []byte{1, 2, 3, 4, 5, 6}, "ACK 01020304", "checksum"}, + {"zero ack", meshcore.PayloadTypeAck, []byte{0, 0, 0, 0}, "ACK 00000000", "checksum"}, + {"short ack", meshcore.PayloadTypeAck, []byte{1, 2, 3}, "", ""}, + {"trace", meshcore.PayloadTypeTrace, trace([]byte{1, 2}), "TRACE efbeadde", "traceTag"}, + {"empty trace path", meshcore.PayloadTypeTrace, trace(nil), "TRACE efbeadde", "traceTag"}, + {"ping", meshcore.PayloadTypeTrace, trace([]byte{1}), "PING efbeadde", "traceTag"}, + {"short trace", meshcore.PayloadTypeTrace, trace(nil)[:8], "", ""}, + {"unsupported", meshcore.PayloadTypeReq, []byte{1, 2, 3, 4}, "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} + w.db = db + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + w.hub.AddScope(client, "summary", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + defer w.hub.Remove(client) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, tc.kind, 0), Payload: tc.payload} + w.handlePacket(ctx, "YVR", "0102", packetEnvelope(t, packet)) + if len(db.packets) != 1 { + t.Fatal("packet was not stored") + } + if tc.want != "" { + var saved map[string]any + if err := json.Unmarshal(db.packets[0].ParsedPayload, &saved); err != nil { + t.Fatal(err) + } + if saved["type"].(string)+" "+saved[tc.field].(string) != tc.want { + t.Fatal("stored reference differs from the expected wire value") + } + } + for { + select { + case event := <-client.Send: + if event.Type != hub.EventPacketObservation { + continue + } + for _, raw := range []json.RawMessage{event.Payload, event.PayloadResolved} { + var got struct { + Packet struct { + Summary *string `json:"summary"` + } `json:"packet"` + } + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatal(err) + } + if tc.want == "" { + if got.Packet.Summary != nil { + t.Fatalf("unexpected summary %q", *got.Packet.Summary) + } + } else if got.Packet.Summary == nil || *got.Packet.Summary != tc.want { + t.Fatalf("live summary = %v, want %q", got.Packet.Summary, tc.want) + } + } + return + case <-ctx.Done(): + t.Fatal("packet observation event missing") + } + } + }) + } +} From 3dee2098bae6855ef8c9ab21df8b0d3feec89d4b Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Fri, 25 Sep 2026 11:38:04 -0400 Subject: [PATCH 051/141] perf(db): batch retention deletes and tune autovacuum (#162) Default packets.retention drops to 7 days. --- README.md | 4 +- config.yaml.example | 5 +- db/migrations/037_autovacuum_tuning.sql | 14 +++ db/packets.go | 10 ++- db/packets_test.go | 20 +++++ db/queries/queries.sql | 36 +++++--- db/retention_integration_test.go | 111 ++++++++++++++++++++++++ db/routes.go | 13 ++- db/routes_test.go | 5 +- db/sqlc/mock/querier.go | 20 +++-- db/sqlc/querier.go | 11 ++- db/sqlc/queries.sql.go | 75 +++++++++++----- db/store.go | 16 ++++ db/store_test.go | 28 ++++++ internal/config/config.go | 10 +-- internal/config/config_test.go | 6 +- 16 files changed, 317 insertions(+), 67 deletions(-) create mode 100644 db/migrations/037_autovacuum_tuning.sql create mode 100644 db/retention_integration_test.go diff --git a/README.md b/README.md index becb8bc4..5418223c 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ telemetry: # Packet and observation retention. packets: - retention: 720h # how long to keep packets and observations (default: 30 days) + retention: 168h # packets, observations, channel messages (default: 7 days) # Presence write coalescing. # Observer last_seen and packet last_heard_at bumps are batched in memory and @@ -260,7 +260,7 @@ websocket: nodes: mark_foreign: false # optional indication for repeaters outside configured IATA borders stale_threshold: 24h # mark a node "stale" in the API after this long unseen (default: 24h) - delete_after: 720h # delete a node entirely after this long unseen (default: 30 days, same default as packets.retention) + delete_after: 720h # delete a node entirely after this long unseen (default: 30 days) clock_drift_threshold: 5m # |device clock - server clock| above which clockOutOfSync=true for a repeater/room server (default: 5m) # Optional observer age-out (disabled by default; set e.g. 720h to opt in). diff --git a/config.yaml.example b/config.yaml.example index dd7bfac4..de12bef4 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -118,8 +118,9 @@ telemetry: #backup: # enabled: false +# Also covers observations and channel messages. packets: - retention: 720h # 30 days + retention: 168h # 7 days # Known-route retention. Routes are distilled packet history (the path a packet # took, hop by hop), so they may outlive packets.retention; there is no required @@ -155,7 +156,7 @@ websocket: # # uses the union of configured iatas.*.borderFile polygons; # # requires at least one valid border; no packet filtering # stale_threshold: 24h # default: 24h -# delete_after: 720h # default: 30 days, same default as packets.retention +# delete_after: 720h # default: 30 days # clock_drift_threshold: 5m # default: 5m # Optional observer age-out. Omitted or nonpositive leaves observers intact. diff --git a/db/migrations/037_autovacuum_tuning.sql b/db/migrations/037_autovacuum_tuning.sql new file mode 100644 index 00000000..f8db967d --- /dev/null +++ b/db/migrations/037_autovacuum_tuning.sql @@ -0,0 +1,14 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Default 20% trigger never fired on known_routes, bloating its indexes. +ALTER TABLE known_routes SET ( + autovacuum_vacuum_scale_factor = 0.02, + autovacuum_analyze_scale_factor = 0.02, + autovacuum_vacuum_cost_limit = 1000 +); +ALTER TABLE packets SET (autovacuum_vacuum_scale_factor = 0.05, autovacuum_vacuum_cost_limit = 1000); +ALTER TABLE packet_observations SET (autovacuum_vacuum_scale_factor = 0.05, autovacuum_vacuum_cost_limit = 1000); +ALTER TABLE nodes SET (autovacuum_vacuum_scale_factor = 0.05); +ALTER TABLE node_iatas SET (autovacuum_vacuum_scale_factor = 0.05); +ALTER TABLE node_short_ids SET (autovacuum_vacuum_scale_factor = 0.05); diff --git a/db/packets.go b/db/packets.go index e6512694..8172c66f 100644 --- a/db/packets.go +++ b/db/packets.go @@ -650,6 +650,14 @@ func (s *Store) GetPacketObservationCount(ctx context.Context, packetHash []byte return s.q.GetPacketObservationCount(ctx, packetHash) } +// Small because each packet cascades to its observations. +const packetDeleteBatch = 1000 + func (s *Store) DeleteOldPackets(ctx context.Context, cutoff time.Time) error { - return s.q.DeleteOldPackets(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) + return deleteInBatches(ctx, packetDeleteBatch, func(ctx context.Context, n int32) (int64, error) { + return s.q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{ + Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, + BatchSize: n, + }) + }) } diff --git a/db/packets_test.go b/db/packets_test.go index 6541ba35..d93aac57 100644 --- a/db/packets_test.go +++ b/db/packets_test.go @@ -642,3 +642,23 @@ func TestListPackets_UnfilteredKeepsGlobalQuery(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestDeleteOldPackets_LoopsUntilShortBatch(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + cutoff := time.Date(2026, 9, 18, 0, 0, 0, 0, time.UTC) + + byCutoff := gomock.Cond(func(p sqlc.DeleteOldPacketsParams) bool { + return p.Cutoff.Time.Equal(cutoff) && p.BatchSize == packetDeleteBatch + }) + gomock.InOrder( + mock.EXPECT().DeleteOldPackets(gomock.Any(), byCutoff).Return(int64(packetDeleteBatch), nil), + mock.EXPECT().DeleteOldPackets(gomock.Any(), byCutoff).Return(int64(packetDeleteBatch), nil), + mock.EXPECT().DeleteOldPackets(gomock.Any(), byCutoff).Return(int64(7), nil), + ) + + if err := store.DeleteOldPackets(context.Background(), cutoff); err != nil { + t.Fatal(err) + } +} diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 614c92c1..1f84bdf3 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -658,10 +658,18 @@ ORDER BY po.id ASC LIMIT $6; --- name: DeleteOldPackets :exec --- Deletes packets and their observations older than the given cutoff. --- packet_observations cascade-delete via FK. -DELETE FROM packets WHERE last_heard_at < $1; +-- name: DeleteOldPackets :execrows +-- One batch of expired packets; observations and channel messages cascade. +WITH expired AS ( + SELECT ep.packet_hash + FROM packets ep + WHERE ep.last_heard_at < @cutoff + ORDER BY ep.last_heard_at + LIMIT @batch_size + FOR UPDATE OF ep SKIP LOCKED +) +DELETE FROM packets p USING expired e +WHERE p.packet_hash = e.packet_hash; -- name: DeleteOldNodes :exec -- Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- @@ -675,12 +683,20 @@ DELETE FROM nodes WHERE last_seen < $1 AND id NOT IN (SELECT owner_node_id FROM observer_owners WHERE owner_node_id IS NOT NULL); --- name: DeleteOldRoutes :exec --- Deletes routes not observed since the retention cutoff ($1), and rarely-observed --- routes (observation_count < $2) not observed since the grace cutoff ($3). -DELETE FROM known_routes -WHERE last_seen < $1 - OR (observation_count < $2 AND last_seen < $3); +-- name: DeleteOldRoutes :execrows +-- One batch of routes past retention, or past grace with too few observations. +-- GREATEST keeps the scan on idx_known_routes_last_seen. +WITH expired AS ( + SELECT r.iata, r.path_key + FROM known_routes r + WHERE r.last_seen < GREATEST(@retention_cutoff::timestamptz, @grace_cutoff::timestamptz) + AND (r.last_seen < @retention_cutoff OR + (r.observation_count < @min_observations AND r.last_seen < @grace_cutoff)) + LIMIT @batch_size + FOR UPDATE OF r SKIP LOCKED +) +DELETE FROM known_routes kr USING expired e +WHERE kr.iata = e.iata AND kr.path_key = e.path_key; -- name: DeleteOldChannelIATAs :exec -- Keeps the channel IATA filter in step with packet retention. diff --git a/db/retention_integration_test.go b/db/retention_integration_test.go new file mode 100644 index 00000000..5e40dbd2 --- /dev/null +++ b/db/retention_integration_test.go @@ -0,0 +1,111 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +func retentionTx(t *testing.T) (context.Context, pgx.Tx) { + t.Helper() + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close(context.Background()) }) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { tx.Rollback(context.Background()) }) + return ctx, tx +} + +func countRows(t *testing.T, ctx context.Context, tx pgx.Tx, sql string) int { + t.Helper() + var n int + if err := tx.QueryRow(ctx, sql).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// More than one batch of old packets; observations go with them. +func TestDeleteOldPacketsBatchesPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + _, err := tx.Exec(ctx, ` +CREATE TEMP TABLE packets (LIKE public.packets INCLUDING ALL) ON COMMIT DROP; +CREATE TEMP TABLE packet_observations (LIKE public.packet_observations INCLUDING ALL) ON COMMIT DROP; +ALTER TABLE packet_observations ADD FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +SELECT int4send(i), 4, 0, 1, '\x00', '\x00', + CASE WHEN i <= 2500 THEN '2026-01-01'::timestamptz ELSE '2026-02-01'::timestamptz END, + CASE WHEN i <= 2500 THEN '2026-01-01'::timestamptz ELSE '2026-02-01'::timestamptz END +FROM generate_series(1, 2510) i; +INSERT INTO packet_observations (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +SELECT packet_hash, '00000000-0000-0000-0000-000000000001', 'YVR', last_heard_at, 0, 1, 0 FROM packets;`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + if err := store.DeleteOldPackets(ctx, time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC)); err != nil { + t.Fatal(err) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM packets`); n != 10 { + t.Errorf("packets left = %d, want 10", n) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM packet_observations`); n != 10 { + t.Errorf("observations left = %d, want 10", n) + } +} + +// Covers grace set longer than retention too. +func TestDeleteOldRoutesBatchesPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + // 1-10500 past retention, 10501-10600 rare, 10601-10700 well observed, rest fresh. + _, err := tx.Exec(ctx, ` +CREATE TEMP TABLE known_routes (LIKE public.known_routes INCLUDING ALL) ON COMMIT DROP; +INSERT INTO known_routes (id, path_key, node_ids, hash_prefix, iata, hop_count, last_seen, observation_count) +SELECT i, decode(md5(i::text), 'hex'), ARRAY[md5(i::text)::uuid], ARRAY['\x01'::bytea], 'YYZ', 2, + CASE WHEN i <= 10500 THEN '2026-01-01'::timestamptz + WHEN i <= 10700 THEN '2026-01-20'::timestamptz + ELSE '2026-02-01'::timestamptz END, + CASE WHEN i BETWEEN 10501 AND 10600 THEN 1 ELSE 10 END +FROM generate_series(1, 10710) i;`) + if err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + retention := time.Date(2026, 1, 10, 0, 0, 0, 0, time.UTC) + grace := time.Date(2026, 1, 25, 0, 0, 0, 0, time.UTC) + if err := store.DeleteOldRoutes(ctx, retention, 3, grace); err != nil { + t.Fatal(err) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM known_routes`); n != 110 { + t.Errorf("routes left = %d, want 110 (well-observed + fresh)", n) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM known_routes WHERE observation_count = 1`); n != 0 { + t.Errorf("rare routes past grace left = %d, want 0", n) + } + + // Grace longer than retention: retention alone decides. + if err := store.DeleteOldRoutes(ctx, time.Date(2026, 1, 25, 0, 0, 0, 0, time.UTC), 3, retention); err != nil { + t.Fatal(err) + } + if n := countRows(t, ctx, tx, `SELECT count(*) FROM known_routes`); n != 10 { + t.Errorf("routes left = %d, want 10 fresh", n) + } +} diff --git a/db/routes.go b/db/routes.go index 047bfec6..a629bcbc 100644 --- a/db/routes.go +++ b/db/routes.go @@ -292,13 +292,18 @@ func (s *Store) ReconfirmRoutes(ctx context.Context, batchSize int32) error { return s.q.ReconfirmRoutes(ctx, batchSize) } +const routeDeleteBatch = 10000 + // DeleteOldRoutes prunes routes per the retention rule: unconditionally past // retentionCutoff, and past graceCutoff when observed fewer than minObservations times. func (s *Store) DeleteOldRoutes(ctx context.Context, retentionCutoff time.Time, minObservations int64, graceCutoff time.Time) error { - return s.q.DeleteOldRoutes(ctx, sqlc.DeleteOldRoutesParams{ - LastSeen: pgtype.Timestamptz{Time: retentionCutoff, Valid: true}, - ObservationCount: minObservations, - LastSeen_2: pgtype.Timestamptz{Time: graceCutoff, Valid: true}, + return deleteInBatches(ctx, routeDeleteBatch, func(ctx context.Context, n int32) (int64, error) { + return s.q.DeleteOldRoutes(ctx, sqlc.DeleteOldRoutesParams{ + RetentionCutoff: pgtype.Timestamptz{Time: retentionCutoff, Valid: true}, + GraceCutoff: pgtype.Timestamptz{Time: graceCutoff, Valid: true}, + MinObservations: minObservations, + BatchSize: n, + }) }) } diff --git a/db/routes_test.go b/db/routes_test.go index 977b541a..034d57f4 100644 --- a/db/routes_test.go +++ b/db/routes_test.go @@ -60,8 +60,9 @@ func TestDeleteOldRoutes_PassesCutoffs(t *testing.T) { grace := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC) mock.EXPECT().DeleteOldRoutes(gomock.Any(), gomock.Cond(func(p sqlc.DeleteOldRoutesParams) bool { - return p.LastSeen.Time.Equal(retention) && p.ObservationCount == 3 && p.LastSeen_2.Time.Equal(grace) - })).Return(nil) + return p.RetentionCutoff.Time.Equal(retention) && p.MinObservations == 3 && p.GraceCutoff.Time.Equal(grace) && + p.BatchSize == routeDeleteBatch + })).Return(int64(0), nil) if err := store.DeleteOldRoutes(context.Background(), retention, 3, grace); err != nil { t.Fatal(err) diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index d941dc96..4efbfaa5 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -117,25 +117,27 @@ func (mr *MockQuerierMockRecorder) DeleteOldObservers(ctx, lastSeen any) *gomock } // DeleteOldPackets mocks base method. -func (m *MockQuerier) DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { +func (m *MockQuerier) DeleteOldPackets(ctx context.Context, arg db.DeleteOldPacketsParams) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteOldPackets", ctx, lastHeardAt) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "DeleteOldPackets", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 } // DeleteOldPackets indicates an expected call of DeleteOldPackets. -func (mr *MockQuerierMockRecorder) DeleteOldPackets(ctx, lastHeardAt any) *gomock.Call { +func (mr *MockQuerierMockRecorder) DeleteOldPackets(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldPackets", reflect.TypeOf((*MockQuerier)(nil).DeleteOldPackets), ctx, lastHeardAt) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldPackets", reflect.TypeOf((*MockQuerier)(nil).DeleteOldPackets), ctx, arg) } // DeleteOldRoutes mocks base method. -func (m *MockQuerier) DeleteOldRoutes(ctx context.Context, arg db.DeleteOldRoutesParams) error { +func (m *MockQuerier) DeleteOldRoutes(ctx context.Context, arg db.DeleteOldRoutesParams) (int64, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteOldRoutes", ctx, arg) - ret0, _ := ret[0].(error) - return ret0 + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 } // DeleteOldRoutes indicates an expected call of DeleteOldRoutes. diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index aa4bd874..d3913c42 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -29,12 +29,11 @@ type Querier interface { // Opt-in age-out: preserve retained history and manually recorded ownership. // Bound deletions per cleanup tick and skip observers being updated by ingest. DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) - // Deletes packets and their observations older than the given cutoff. - // packet_observations cascade-delete via FK. - DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error - // Deletes routes not observed since the retention cutoff ($1), and rarely-observed - // routes (observation_count < $2) not observed since the grace cutoff ($3). - DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) error + // One batch of expired packets; observations and channel messages cascade. + DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) + // One batch of routes past retention, or past grace with too few observations. + // GREATEST keeps the scan on idx_known_routes_last_seen. + DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) (int64, error) // Deletes telemetry rows older than the given cutoff. Called by the cleanup goroutine. DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error // Keeps the trace IATA filter in step with packet retention. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 16e76c4f..d454af2d 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -124,34 +124,67 @@ func (q *Queries) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timest return items, nil } -const deleteOldPackets = `-- name: DeleteOldPackets :exec -DELETE FROM packets WHERE last_heard_at < $1 +const deleteOldPackets = `-- name: DeleteOldPackets :execrows +WITH expired AS ( + SELECT ep.packet_hash + FROM packets ep + WHERE ep.last_heard_at < $1 + ORDER BY ep.last_heard_at + LIMIT $2 + FOR UPDATE OF ep SKIP LOCKED +) +DELETE FROM packets p USING expired e +WHERE p.packet_hash = e.packet_hash ` -// Deletes packets and their observations older than the given cutoff. -// packet_observations cascade-delete via FK. -func (q *Queries) DeleteOldPackets(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { - _, err := q.db.Exec(ctx, deleteOldPackets, lastHeardAt) - return err +type DeleteOldPacketsParams struct { + Cutoff pgtype.Timestamptz `json:"cutoff"` + BatchSize int32 `json:"batch_size"` } -const deleteOldRoutes = `-- name: DeleteOldRoutes :exec -DELETE FROM known_routes -WHERE last_seen < $1 - OR (observation_count < $2 AND last_seen < $3) +// One batch of expired packets; observations and channel messages cascade. +func (q *Queries) DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) { + result, err := q.db.Exec(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const deleteOldRoutes = `-- name: DeleteOldRoutes :execrows +WITH expired AS ( + SELECT r.iata, r.path_key + FROM known_routes r + WHERE r.last_seen < GREATEST($1::timestamptz, $2::timestamptz) + AND (r.last_seen < $1 OR + (r.observation_count < $3 AND r.last_seen < $2)) + LIMIT $4 + FOR UPDATE OF r SKIP LOCKED +) +DELETE FROM known_routes kr USING expired e +WHERE kr.iata = e.iata AND kr.path_key = e.path_key ` type DeleteOldRoutesParams struct { - LastSeen pgtype.Timestamptz `json:"last_seen"` - ObservationCount int64 `json:"observation_count"` - LastSeen_2 pgtype.Timestamptz `json:"last_seen_2"` -} - -// Deletes routes not observed since the retention cutoff ($1), and rarely-observed -// routes (observation_count < $2) not observed since the grace cutoff ($3). -func (q *Queries) DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) error { - _, err := q.db.Exec(ctx, deleteOldRoutes, arg.LastSeen, arg.ObservationCount, arg.LastSeen_2) - return err + RetentionCutoff pgtype.Timestamptz `json:"retention_cutoff"` + GraceCutoff pgtype.Timestamptz `json:"grace_cutoff"` + MinObservations int64 `json:"min_observations"` + BatchSize int32 `json:"batch_size"` +} + +// One batch of routes past retention, or past grace with too few observations. +// GREATEST keeps the scan on idx_known_routes_last_seen. +func (q *Queries) DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) (int64, error) { + result, err := q.db.Exec(ctx, deleteOldRoutes, + arg.RetentionCutoff, + arg.GraceCutoff, + arg.MinObservations, + arg.BatchSize, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil } const deleteOldTelemetry = `-- name: DeleteOldTelemetry :exec diff --git a/db/store.go b/db/store.go index a22b9600..20866d76 100644 --- a/db/store.go +++ b/db/store.go @@ -149,3 +149,19 @@ func toChannelMessage(id int64, packetHashHex string, channelHash []byte, sender ObservationCount: observationCount, } } + +// deleteInBatches repeats a delete until a batch comes back short. +func deleteInBatches(ctx context.Context, batchSize int32, del func(context.Context, int32) (int64, error)) error { + for { + n, err := del(ctx, batchSize) + if err != nil { + return err + } + if n < int64(batchSize) { + return nil + } + if err := ctx.Err(); err != nil { + return err + } + } +} diff --git a/db/store_test.go b/db/store_test.go index 8bfab391..4d0521be 100644 --- a/db/store_test.go +++ b/db/store_test.go @@ -179,3 +179,31 @@ func TestResolvePathHashes_Mapping(t *testing.T) { t.Errorf("expected Name %s, got %v", name, entries[0].Name) } } + +func TestDeleteInBatches_StopsOnError(t *testing.T) { + boom := errors.New("boom") + calls := 0 + err := deleteInBatches(context.Background(), 10, func(context.Context, int32) (int64, error) { + calls++ + if calls == 2 { + return 0, boom + } + return 10, nil + }) + if !errors.Is(err, boom) || calls != 2 { + t.Fatalf("err=%v calls=%d, want boom after 2 calls", err, calls) + } +} + +func TestDeleteInBatches_StopsWhenCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + err := deleteInBatches(ctx, 10, func(context.Context, int32) (int64, error) { + calls++ + cancel() + return 10, nil + }) + if !errors.Is(err, context.Canceled) || calls != 1 { + t.Fatalf("err=%v calls=%d, want context.Canceled after 1 call", err, calls) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 61786364..5f532726 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -245,8 +245,7 @@ type WebSocketConfig struct { // PacketsConfig controls packet retention behaviour. type PacketsConfig struct { - // Retention is how long packet and observation rows are kept. - // Defaults to 720h (30 days) if not set. + // Retention covers packets, observations and channel messages. Defaults to 168h. Retention duration `yaml:"retention"` } @@ -276,8 +275,7 @@ type NodesConfig struct { // stale=true for it. Defaults to 24h if not set. StaleThreshold duration `yaml:"stale_threshold"` // DeleteAfter is how long since a node's last_seen before the cleanup job deletes the - // node entirely. Defaults to the same 30-day default as packets.retention if not set -- - // independently configurable from it, just the same starting point. + // node entirely. Defaults to 720h (30 days) if not set. DeleteAfter duration `yaml:"delete_after"` } @@ -449,7 +447,7 @@ func Resolve(cfg *Config) ResolvedConfig { r.TelemetryRetention = 28 * 24 * time.Hour } if r.PacketRetention == 0 { - r.PacketRetention = 30 * 24 * time.Hour + r.PacketRetention = 7 * 24 * time.Hour } if r.RouteRetention == 0 { r.RouteRetention = 14 * 24 * time.Hour @@ -488,8 +486,6 @@ func Resolve(cfg *Config) ResolvedConfig { r.NodeStaleThreshold = 24 * time.Hour } if r.NodeDeleteAfter == 0 { - // Same default as packets.retention (30 days) -- independently configurable, just - // the same starting point, not tied to whatever PacketRetention resolves to. r.NodeDeleteAfter = 30 * 24 * time.Hour } return r diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 8ce2828c..4ba8b238 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -130,8 +130,8 @@ func TestResolve_Defaults(t *testing.T) { if r.TelemetryRetention != 28*24*time.Hour { t.Errorf("expected TelemetryRetention 672h, got %v", r.TelemetryRetention) } - if r.PacketRetention != 30*24*time.Hour { - t.Errorf("expected PacketRetention 720h, got %v", r.PacketRetention) + if r.PacketRetention != 7*24*time.Hour { + t.Errorf("expected PacketRetention 168h, got %v", r.PacketRetention) } if r.MaxConnsPerIP != 5 { t.Errorf("expected MaxConnsPerIP 5, got %d", r.MaxConnsPerIP) @@ -152,7 +152,7 @@ func TestResolve_Defaults(t *testing.T) { t.Errorf("expected NodeStaleThreshold 24h, got %v", r.NodeStaleThreshold) } if r.NodeDeleteAfter != 30*24*time.Hour { - t.Errorf("expected NodeDeleteAfter 720h (same default as PacketRetention), got %v", r.NodeDeleteAfter) + t.Errorf("expected NodeDeleteAfter 720h, got %v", r.NodeDeleteAfter) } if r.ObserverDeleteAfter != 0 { t.Errorf("observer deletion must be disabled by default, got %v", r.ObserverDeleteAfter) From 91b4b457b5f233e9b90b4030e30623095e950714 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:59:07 -0400 Subject: [PATCH 052/141] perf(db): resolve packet endpoints at read time (#163) Storing a full candidate snapshot on every observation was the wrong call. The same node lists got copied thousands of times, about 22 GB on prod. Resolve endpoints live like path hops and drop the column. --- .github/workflows/ci.yml | 4 +- db/endpoints.go | 151 +++++++++ db/endpoints_test.go | 140 +++++++++ db/migrations/038_drop_resolved_endpoints.sql | 6 + db/packet_endpoints_integration_test.go | 287 ++++++------------ db/packets.go | 126 ++------ db/queries/queries.sql | 34 ++- db/sqlc/mock/querier.go | 30 ++ db/sqlc/models.go | 1 - db/sqlc/querier.go | 4 + db/sqlc/queries.sql.go | 240 ++++++++++----- internal/api/packets.go | 15 - internal/api/packets_test.go | 22 -- internal/ingest/packet.go | 13 +- internal/ingest/packet_endpoints_test.go | 128 ++++---- 15 files changed, 713 insertions(+), 488 deletions(-) create mode 100644 db/endpoints.go create mode 100644 db/endpoints_test.go create mode 100644 db/migrations/038_drop_resolved_endpoints.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43b9d5c5..beb7284f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,10 +52,10 @@ jobs: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable run: go test ./cmd/beacon -run '^TestBackupConfigurationPostgres$' -count=1 -v - - name: Verify signal and path aggregates against PostgreSQL 16 + - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/db/endpoints.go b/db/endpoints.go new file mode 100644 index 00000000..457ff421 --- /dev/null +++ b/db/endpoints.go @@ -0,0 +1,151 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/hex" + "log/slog" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/meshcore-go/meshcore-go" +) + +// packetEndpoints is what a packet's payload says about its sender and recipient. +type packetEndpoints struct { + advert bool // source is the advert's own key, not a hash + advertKey []byte // nil for legacy adverts stored without one + source, destination []byte // one-byte hashes +} + +func parsePacketEndpoints(payloadType int16, raw, originPubkey []byte) packetEndpoints { + var ep packetEndpoints + switch payloadType { + case int16(meshcore.PayloadTypeAdvert): + ep.advert, ep.advertKey = true, originPubkey + case int16(meshcore.PayloadTypeAnonReq): + if p, err := meshcore.AnonReqFromBytes(raw); err == nil { + ep.destination = []byte{p.Destination} + } + case int16(meshcore.PayloadTypeReq): + if p, err := meshcore.RequestFromBytes(raw); err == nil { + ep.source, ep.destination = []byte{p.Source}, []byte{p.Destination} + } + case int16(meshcore.PayloadTypeResponse): + if p, err := meshcore.ResponseFromBytes(raw); err == nil { + ep.source, ep.destination = []byte{p.Source}, []byte{p.Destination} + } + case int16(meshcore.PayloadTypeTxtMsg): + if p, err := meshcore.TextMessageFromBytes(raw); err == nil { + ep.source, ep.destination = []byte{p.Source}, []byte{p.Destination} + } + case int16(meshcore.PayloadTypePath): + if p, err := meshcore.PathFromBytes(raw); err == nil { + ep.source, ep.destination = []byte{p.Source}, []byte{p.Destination} + } + } + return ep +} + +type endpointLookup struct { + iata string + ep packetEndpoints +} + +// resolveEndpoints resolves a whole page of endpoints in at most two queries. +// Results line up with lookups; nil means no such endpoint or a failed lookup. +func (s *Store) resolveEndpoints(ctx context.Context, lookups []endpointLookup) (sources, destinations []*api.ResolvedHop) { + iatas, hashes, keys := map[string]bool{}, map[byte]bool{}, map[string][]byte{} + for _, l := range lookups { + for _, h := range [][]byte{l.ep.source, l.ep.destination} { + if len(h) == 1 { + iatas[l.iata], hashes[h[0]] = true, true + } + } + if len(l.ep.advertKey) > 0 { + keys[string(l.ep.advertKey)] = l.ep.advertKey + } + } + + candidates := map[string]map[string][]api.ResolvedPathEntry{} // iata -> hash hex + hashesOK := len(hashes) > 0 + if hashesOK { + params := sqlc.ResolveEndpointHashPairsParams{} + for iata := range iatas { + params.Iatas = append(params.Iatas, iata) + } + for h := range hashes { + params.Hashes = append(params.Hashes, []byte{h}) + } + rows, err := s.q.ResolveEndpointHashPairs(ctx, params) + if err != nil { + slog.Error("store: endpoint resolution failed", "component", "db", "error", err) + hashesOK = false + } + for _, r := range rows { + if candidates[r.Iata] == nil { + candidates[r.Iata] = map[string][]api.ResolvedPathEntry{} + } + key := hex.EncodeToString(r.Hash) + candidates[r.Iata][key] = append(candidates[r.Iata][key], api.ResolvedPathEntry{ + NodeID: r.NodeID, Name: r.Name, Latitude: r.Latitude, + Longitude: r.Longitude, PublicKey: r.PublicKey, + }) + } + } + + adverts := map[string]*api.ResolvedNode{} + advertsOK := true + if len(keys) > 0 { + pubkeys := make([][]byte, 0, len(keys)) + for _, k := range keys { + pubkeys = append(pubkeys, k) + } + rows, err := s.q.GetNodesByPubkeys(ctx, pubkeys) + if err != nil { + slog.Error("store: advert source lookup failed", "component", "db", "error", err) + advertsOK = false + } + for _, r := range rows { + adverts[string(r.PublicKey)] = &api.ResolvedNode{ + ID: r.ID, Name: r.Name, PublicKey: hex.EncodeToString(r.PublicKey), + Latitude: r.Latitude, Longitude: r.Longitude, + } + } + } + + hop := func(iata string, h []byte) *api.ResolvedHop { + if len(h) != 1 || !hashesOK { + return nil + } + r := api.BuildResolvedPath([][]byte{h}, candidates[iata])[0] + return &r + } + sources = make([]*api.ResolvedHop, len(lookups)) + destinations = make([]*api.ResolvedHop, len(lookups)) + for i, l := range lookups { + if l.ep.advert { + if advertsOK { + r := api.ResolveExactNode(adverts[string(l.ep.advertKey)]) + sources[i] = &r + } + } else { + sources[i] = hop(l.iata, l.ep.source) + } + destinations[i] = hop(l.iata, l.ep.destination) + } + return sources, destinations +} + +// fillLatestEndpoints resolves the latest observer's endpoints for a page of packets. +// lookups line up with items; rows without a latest observer are skipped. +func (s *Store) fillLatestEndpoints(ctx context.Context, items []api.PacketSummary, lookups []endpointLookup) { + sources, destinations := s.resolveEndpoints(ctx, lookups) + for i := range items { + if lo := items[i].LatestObserver; lo != nil { + lo.ResolvedSource, lo.ResolvedDestination = sources[i], destinations[i] + } + } +} diff --git a/db/endpoints_test.go b/db/endpoints_test.go new file mode 100644 index 00000000..a896d2e4 --- /dev/null +++ b/db/endpoints_test.go @@ -0,0 +1,140 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "sort" + "testing" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" + "go.uber.org/mock/gomock" +) + +func TestParsePacketEndpoints(t *testing.T) { + msg := append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...) + key := []byte{1, 2, 3} + for _, tc := range []struct { + name string + kind uint8 + raw, key []byte + want packetEndpoints + }{ + {"direct message", meshcore.PayloadTypeTxtMsg, msg, nil, packetEndpoints{source: []byte{0xaa}, destination: []byte{0xbb}}}, + {"request", meshcore.PayloadTypeReq, msg, nil, packetEndpoints{source: []byte{0xaa}, destination: []byte{0xbb}}}, + {"advert", meshcore.PayloadTypeAdvert, nil, key, packetEndpoints{advert: true, advertKey: key}}, + {"channel message", meshcore.PayloadTypeGrpTxt, msg, nil, packetEndpoints{}}, + {"truncated", meshcore.PayloadTypeTxtMsg, []byte{0xbb}, nil, packetEndpoints{}}, + } { + t.Run(tc.name, func(t *testing.T) { + got := parsePacketEndpoints(int16(tc.kind), tc.raw, tc.key) + if got.advert != tc.want.advert || string(got.advertKey) != string(tc.want.advertKey) || + string(got.source) != string(tc.want.source) || string(got.destination) != string(tc.want.destination) { + t.Fatalf("got %+v, want %+v", got, tc.want) + } + }) + } +} + +func hopNames(h *api.ResolvedHop) []string { + if h == nil { + return nil + } + names := []string{h.Confidence} + for _, n := range h.Nodes { + names = append(names, *n.Name) + } + sort.Strings(names[1:]) + return names +} + +func TestResolveEndpointsBatchesAPage(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + name := func(s string) *string { return &s } + alice, bob, carol := uuid.New(), uuid.New(), uuid.New() + advertKey := []byte{0xaa, 0x11} + + mock.EXPECT().ResolveEndpointHashPairs(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, p sqlc.ResolveEndpointHashPairsParams) ([]sqlc.ResolveEndpointHashPairsRow, error) { + if len(p.Iatas) != 2 || len(p.Hashes) != 2 { + t.Errorf("want distinct IATAs and hashes, got %v %v", p.Iatas, p.Hashes) + } + return []sqlc.ResolveEndpointHashPairsRow{ + {Iata: "YYZ", Hash: []byte{0xaa}, NodeID: alice, Name: name("Alice")}, + {Iata: "YYZ", Hash: []byte{0xaa}, NodeID: bob, Name: name("Bob")}, + {Iata: "YYZ", Hash: []byte{0xbb}, NodeID: carol, Name: name("Carol")}, + {Iata: "YVR", Hash: []byte{0xaa}, NodeID: alice, Name: name("Alice")}, + {Iata: "YVR", Hash: []byte{0xbb}, NodeID: carol, Name: name("Carol")}, // cross-product extra + }, nil + }) + mock.EXPECT().GetNodesByPubkeys(gomock.Any(), [][]byte{advertKey}).Return( + []sqlc.GetNodesByPubkeysRow{{ID: alice, PublicKey: advertKey, Name: name("Alice")}}, nil) + + sources, destinations := store.resolveEndpoints(context.Background(), []endpointLookup{ + {iata: "YYZ", ep: packetEndpoints{source: []byte{0xaa}, destination: []byte{0xbb}}}, + {iata: "YVR", ep: packetEndpoints{source: []byte{0xaa}}}, + {iata: "YYZ", ep: packetEndpoints{advert: true, advertKey: advertKey}}, + {iata: "YYZ", ep: packetEndpoints{advert: true, advertKey: advertKey}}, + {iata: "YYZ", ep: packetEndpoints{advert: true}}, + {}, + }) + + for i, tc := range []struct{ source, destination []string }{ + {[]string{"ambiguous", "Alice", "Bob"}, []string{"high", "Carol"}}, + {[]string{"high", "Alice"}, nil}, + {[]string{"high", "Alice"}, nil}, + {[]string{"high", "Alice"}, nil}, + {[]string{"none"}, nil}, + {nil, nil}, + } { + if got := hopNames(sources[i]); !equalStrings(got, tc.source) { + t.Errorf("row %d source = %v, want %v", i, got, tc.source) + } + if got := hopNames(destinations[i]); !equalStrings(got, tc.destination) { + t.Errorf("row %d destination = %v, want %v", i, got, tc.destination) + } + } +} + +func TestResolveEndpointsFailedLookupLeavesHopsEmpty(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + store := &Store{q: mock} + mock.EXPECT().ResolveEndpointHashPairs(gomock.Any(), gomock.Any()).Return(nil, errors.New("boom")) + + sources, destinations := store.resolveEndpoints(context.Background(), []endpointLookup{ + {iata: "YYZ", ep: packetEndpoints{source: []byte{0xaa}, destination: []byte{0xbb}}}, + }) + if sources[0] != nil || destinations[0] != nil { + t.Fatal("a failed lookup must not report endpoints as unresolved") + } +} + +func TestResolveEndpointsSkipsQueriesWhenNothingToResolve(t *testing.T) { + ctrl := gomock.NewController(t) + store := &Store{q: mockdb.NewMockQuerier(ctrl)} // no EXPECT: any query fails the test + sources, _ := store.resolveEndpoints(context.Background(), []endpointLookup{{}, {iata: "YYZ"}}) + if len(sources) != 2 || sources[0] != nil || sources[1] != nil { + t.Fatal("rows without endpoints must stay empty") + } +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/db/migrations/038_drop_resolved_endpoints.sql b/db/migrations/038_drop_resolved_endpoints.sql new file mode 100644 index 00000000..e69190ec --- /dev/null +++ b/db/migrations/038_drop_resolved_endpoints.sql @@ -0,0 +1,6 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Endpoints now resolve at read time like path hops. The per-observation +-- snapshots were ~60% of packet_observations on prod. +ALTER TABLE packet_observations DROP COLUMN IF EXISTS resolved_endpoints; diff --git a/db/packet_endpoints_integration_test.go b/db/packet_endpoints_integration_test.go index 756d9cdf..50d2c0bb 100644 --- a/db/packet_endpoints_integration_test.go +++ b/db/packet_endpoints_integration_test.go @@ -5,18 +5,18 @@ package db import ( "context" - "encoding/json" + "crypto/rand" "fmt" "os" - "reflect" + "strings" "testing" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" - "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" ) type endpointQueryCounter struct { @@ -33,225 +33,134 @@ func (q *endpointQueryCounter) QueryRow(ctx context.Context, sql string, args .. return q.DBTX.QueryRow(ctx, sql, args...) } -func TestPacketEndpointSnapshotsPostgres(t *testing.T) { +// Endpoints resolve live against today's nodes, in a fixed number of queries per page. +func TestPacketEndpointsResolveLivePostgres(t *testing.T) { dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") if dsn == "" { t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") } - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - conn, err := pgx.Connect(ctx, dsn) + // Own throwaway database so CI can run it against an empty server. + admin, err := pgx.Connect(ctx, dsn) if err != nil { t.Fatal(err) } - defer conn.Close(context.Background()) - tx, err := conn.Begin(ctx) + defer admin.Close(context.Background()) + name := "beacon_endpoints_" + strings.ToLower(rand.Text()) + ident := pgx.Identifier{name}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE DATABASE "+ident); err != nil { + t.Fatal(err) + } + defer admin.Exec(context.Background(), "DROP DATABASE "+ident+" WITH (FORCE)") + cfg, err := pgxpool.ParseConfig(dsn) if err != nil { t.Fatal(err) } - defer tx.Rollback(context.Background()) + cfg.ConnConfig.Database = name + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } exec := func(query string, args ...any) { t.Helper() - if _, err := tx.Exec(ctx, query, args...); err != nil { + if _, err := pool.Exec(ctx, query, args...); err != nil { t.Fatal(err) } } - for _, table := range []string{"packets", "packet_observations", "observers", "transport_scopes", "nodes", "channel_messages"} { - exec("CREATE TEMP TABLE " + table + " (LIKE public." + table + " INCLUDING ALL) ON COMMIT DROP") - } - // Also lets the unchanged Store run against the regression fixture before migration. - exec("ALTER TABLE pg_temp.packet_observations ADD COLUMN IF NOT EXISTS resolved_endpoints JSONB") - exec("CREATE TEMP SEQUENCE endpoint_observation_id START 1000") - exec("ALTER TABLE pg_temp.packet_observations ALTER COLUMN id SET DEFAULT nextval('pg_temp.endpoint_observation_id')") - observer1 := uuid.MustParse("00000000-0000-0000-0000-000000000001") + exec("INSERT INTO iata_codes(iata) VALUES ('YYZ'),('YVR')") + + observer := uuid.MustParse("00000000-0000-0000-0000-000000000001") + alice, bob, carol := uuid.New(), uuid.New(), uuid.New() + aliceKey := "aa11" + fmt.Sprintf("%060d", 0) observer2 := uuid.MustParse("00000000-0000-0000-0000-000000000002") - nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000003") - key := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - name, earlier := "Captured 👋 companion", "Earlier companion" - high := &api.ResolvedHop{Confidence: "high", Nodes: []api.ResolvedNode{{ID: nodeID, Name: &name, PublicKey: key}}} - old := &api.ResolvedHop{Confidence: "high", Nodes: []api.ResolvedNode{{ID: nodeID, Name: &earlier, PublicKey: key}}} - ambiguous := &api.ResolvedHop{Confidence: "ambiguous", Nodes: []api.ResolvedNode{high.Nodes[0], {ID: observer2, PublicKey: "bb"}}} - none := &api.ResolvedHop{Confidence: "none", Nodes: []api.ResolvedNode{}} - snapshot := func(source, destination *api.ResolvedHop) []byte { - value, err := json.Marshal(struct { - Source *api.ResolvedHop `json:"source,omitempty"` - Destination *api.ResolvedHop `json:"destination,omitempty"` - }{source, destination}) - if err != nil { - t.Fatal(err) - } - return value - } - check := func(source, destination, wantSource, wantDestination *api.ResolvedHop) { - t.Helper() - if !reflect.DeepEqual(source, wantSource) || !reflect.DeepEqual(destination, wantDestination) { - t.Error("historical packet lost endpoint snapshot") - } - } - exec("INSERT INTO observers(id,public_key) VALUES ($1,'\\x01'),($2,'\\x02')", observer1, observer2) - exec("INSERT INTO nodes(id,public_key,name,node_type) VALUES ($1,decode($2,'hex'),$3,1)", nodeID, key, name) + exec("INSERT INTO observers(id,public_key) VALUES ($1,'\\x01'),($2,'\\x02')", observer, observer2) + exec(`INSERT INTO nodes(id,public_key,name,node_type) VALUES + ($1,decode($4,'hex'),'Alice',1), ($2,'\xaa22','Bob',1), ($3,'\xbb33','Carol',2)`, alice, bob, carol, aliceKey) + // Alice and Bob share prefix aa in YYZ; YVR only knows Alice. + exec(`INSERT INTO node_short_ids(node_id,iata,prefix_4) VALUES + ($1,'YYZ','\xaa110000'), ($2,'YYZ','\xaa220000'), ($3,'YYZ','\xbb330000'), ($1,'YVR','\xaa110000')`, alice, bob, carol) + anchor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) -SELECT decode(lpad(to_hex(i),64,'0'),'hex'),kind,0,1,'\x00','\x00',decode($2,'hex'), - $1::timestamptz+i*interval '1 second',$1::timestamptz+i*interval '1 second' -FROM (VALUES (1,4),(2,2),(3,3),(4,4),(5,3)) v(i,kind)`, anchor, key) - for _, obs := range []struct { - id, packet int - observer uuid.UUID - iata string - data []byte + msg := append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...) + packets := []struct { + kind int + raw []byte + origin any + iata string }{ - {1, 1, observer1, "YYZ", snapshot(old, nil)}, - {2, 1, observer2, "YVR", snapshot(high, nil)}, - {3, 2, observer1, "YYZ", snapshot(ambiguous, none)}, - {4, 3, observer2, "YVR", snapshot(nil, nil)}, - {5, 4, observer1, "YYZ", nil}, - } { - exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,source_broker,resolved_endpoints) -VALUES ($1,decode($2,'hex'),$3,$4,$5,0,1,0,'fixture',$6)`, obs.id, fmt.Sprintf("%064x", obs.packet), obs.observer, obs.iata, anchor.Add(time.Duration(obs.id)*time.Second), obs.data) - } - exec("UPDATE nodes SET name='Current name'") - q := &endpointQueryCounter{DBTX: tx} + {2, msg, nil, "YYZ"}, // 1: direct message, ambiguous source + {4, []byte{0}, aliceKey, "YYZ"}, // 2: advert from Alice + {4, []byte{0}, "cc" + aliceKey[2:], "YYZ"}, // 3: advert from an unknown node + {2, msg, nil, "YVR"}, // 4: same message bytes, other IATA + } + for i, p := range packets { + n := i + 1 + exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) +VALUES (decode($1,'hex'),$2,0,1,$3,'\x00',decode($4,'hex'),$5,$5)`, fmt.Sprintf("%064x", n), p.kind, p.raw, p.origin, anchor.Add(time.Duration(n)*time.Second)) + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +VALUES ($1,decode($2,'hex'),$3,$4,$5,0,1,0)`, n, fmt.Sprintf("%064x", n), observer, p.iata, anchor.Add(time.Duration(n)*time.Second)) + } + exec("UPDATE nodes SET name='Alice renamed' WHERE id=$1", alice) + + q := &endpointQueryCounter{DBTX: pool} store := &Store{q: sqlc.New(q)} - for _, iatas := range [][]string{nil, {"YYZ"}, {"YVR"}, {"YYZ", "YVR", "YYZ"}} { - q.calls = 0 - page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 50) - if err != nil { - t.Fatal(err) - } - if q.calls != 1 || len(page.Items) == 0 { - t.Fatalf("list used %d queries or lost its page", q.calls) + check := func(label string, got *api.ResolvedHop, want ...string) { + t.Helper() + if names := hopNames(got); !equalStrings(names, want) { + t.Errorf("%s = %v, want %v", label, names, want) } - for _, item := range page.Items { + } + checkPage := func(items []api.PacketSummary) { + t.Helper() + for _, item := range items { + lo := item.LatestObserver switch item.PacketHash { case fmt.Sprintf("%064x", 1): - // The existing regional roll-up also selects the global latest observer. - if item.LatestObserver == nil || item.LatestObserver.ID != observer2 || item.LatestObserver.IATA != "YVR" { - t.Fatal("latest observer selection changed") - } - check(item.LatestObserver.ResolvedSource, item.LatestObserver.ResolvedDestination, high, nil) + check("dm source", lo.ResolvedSource, "ambiguous", "Alice renamed", "Bob") + check("dm destination", lo.ResolvedDestination, "high", "Carol") case fmt.Sprintf("%064x", 2): - check(item.LatestObserver.ResolvedSource, item.LatestObserver.ResolvedDestination, ambiguous, none) - case fmt.Sprintf("%064x", 3), fmt.Sprintf("%064x", 4): - check(item.LatestObserver.ResolvedSource, item.LatestObserver.ResolvedDestination, nil, nil) - case fmt.Sprintf("%064x", 5): - if item.LatestObserver != nil { - t.Fatal("invented an observation for a packet without one") - } + check("advert source", lo.ResolvedSource, "high", "Alice renamed") + case fmt.Sprintf("%064x", 3): + check("unknown advert", lo.ResolvedSource, "none") + case fmt.Sprintf("%064x", 4): + check("YVR source", lo.ResolvedSource, "high", "Alice renamed") + check("YVR destination", lo.ResolvedDestination, "none") } } } - q.calls = 0 - backfill, err := store.ListPacketsAfterID(ctx, 0, -1, -1, nil, "", 50) - if err != nil || len(backfill) != 5 || q.calls != 1 { - t.Fatalf("backfill: rows=%d queries=%d err=%v", len(backfill), q.calls, err) - } - check(backfill[0].LatestObserver.ResolvedSource, backfill[0].LatestObserver.ResolvedDestination, old, nil) - check(backfill[1].LatestObserver.ResolvedSource, backfill[1].LatestObserver.ResolvedDestination, high, nil) - check(backfill[2].LatestObserver.ResolvedSource, backfill[2].LatestObserver.ResolvedDestination, ambiguous, none) - packet1 := append(make([]byte, 31), 1) - for pass := 0; pass < 2; pass++ { + + // One page query plus one hash batch and one advert batch. + for _, iatas := range [][]string{nil, {"YYZ", "YVR"}} { q.calls = 0 - packet, err := store.GetPacket(ctx, packet1) - if err != nil || len(packet.Observations) != 2 { - t.Fatalf("packet detail: %v", err) - } - check(packet.Observations[0].ResolvedSource, packet.Observations[0].ResolvedDestination, old, nil) - check(packet.Observations[1].ResolvedSource, packet.Observations[1].ResolvedDestination, high, nil) - if q.calls != 2 { - t.Errorf("snapshot detail used %d queries, want packet plus observations only", q.calls) - } - if pass == 0 { - legacy, err := store.GetPacket(ctx, append(make([]byte, 31), 4)) - if err != nil || legacy.Observations[0].ResolvedSource == nil || *legacy.Observations[0].ResolvedSource.Nodes[0].Name != "Current name" { - t.Fatalf("legacy detail lookup changed: %v", err) - } - exec("DELETE FROM nodes") + page, err := store.ListPackets(ctx, nil, nil, iatas, nil, time.Time{}, time.Time{}, 0, 50) + if err != nil || len(page.Items) != 4 || q.calls != 3 { + t.Fatalf("list %v: rows=%d queries=%d err=%v", iatas, len(page.Items), q.calls, err) } + checkPage(page.Items) } - exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) -SELECT decode(lpad(to_hex(i),64,'0'),'hex'),4,0,1,'\x00','\x00',$1::timestamptz+i*interval '1 second',$1::timestamptz+i*interval '1 second' -FROM generate_series(6,55) i`, anchor) - exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,source_broker,resolved_endpoints) -SELECT i,decode(lpad(to_hex(i),64,'0'),'hex'),$1,'YYZ',$2::timestamptz+i*interval '1 second',0,1,0,'fixture',$3 -FROM generate_series(6,55) i`, observer1, anchor, snapshot(high, nil)) q.calls = 0 - page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 50) - if err != nil || len(page.Items) != 50 || q.calls != 1 { - t.Fatalf("50-row page: queries=%d err=%v", q.calls, err) - } - for _, item := range page.Items { - check(item.LatestObserver.ResolvedSource, item.LatestObserver.ResolvedDestination, high, nil) + backfill, err := store.ListPacketsAfterID(ctx, 0, -1, -1, nil, "", 50) + if err != nil || len(backfill) != 4 || q.calls != 3 { + t.Fatalf("backfill: rows=%d queries=%d err=%v", len(backfill), q.calls, err) } - t.Logf("50 historical rows use %d database query", q.calls) + checkPage(backfill) - // Capture uses the ordinary insert; duplicate delivery cannot replace its snapshot. - exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) -VALUES (decode($1,'hex'),4,0,1,'\x00','\x00',NOW(),NOW())`, fmt.Sprintf("%064x", 56)) - observation := ingest.InsertObservationParams{PacketHash: append(make([]byte, 31), 56), - ObserverID: observer1, IATA: "YYZ", HeardAt: anchor, HashSize: 1, SourceBroker: "fixture", - PayloadType: 4, ResolvedEndpoints: snapshot(high, none)} - for _, wantInserted := range []bool{true, false} { - q.calls = 0 - inserted, err := store.InsertObservation(ctx, observation) - if err != nil || inserted != wantInserted || q.calls != 1 { - t.Fatalf("snapshot insert: inserted=%v queries=%d err=%v", inserted, q.calls, err) - } - observation.ResolvedEndpoints = snapshot(old, nil) - observation.IATA = "YVR" - } - stored, err := store.GetPacket(ctx, observation.PacketHash) - if err != nil || len(stored.Observations) != 1 || stored.Observations[0].IATA != "YYZ" { - t.Fatalf("duplicate observation changed: %v", err) - } - check(stored.Observations[0].ResolvedSource, stored.Observations[0].ResolvedDestination, high, none) - // First adverts can be captured before their node exists. Empty old captures - // must allow detail-time lookup once that node has been advertised. - exec("INSERT INTO nodes(id,public_key,name,node_type) VALUES ($1,decode($2,'hex'),'Later discovered companion',1)", nodeID, key) - exec("UPDATE packets SET origin_pubkey=decode($2,'hex') WHERE packet_hash=$1", observation.PacketHash, key) - for _, raw := range []string{`{}`, `{"source":{"confidence":"none","nodes":[]}}`} { - exec("UPDATE packet_observations SET resolved_endpoints=$2::jsonb,source_broker=NULL WHERE packet_hash=$1", observation.PacketHash, raw) - q.calls = 0 - stored, err = store.GetPacket(ctx, observation.PacketHash) - if err != nil || q.calls != 4 || stored.Observations[0].SourceBroker != "" { - t.Fatalf("empty snapshot fallback or nullable legacy broker: queries=%d err=%v", q.calls, err) - } - if source := stored.Observations[0].ResolvedSource; source == nil || len(source.Nodes) != 1 || source.Nodes[0].Name == nil || *source.Nodes[0].Name != "Later discovered companion" { - t.Fatal("empty capture suppressed the later advertised node") - } - } - exec(`INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) -VALUES (decode($1,'hex'),4,0,1,'\x00','\x00',decode($2,'hex'),$3,$3)`, fmt.Sprintf("%064x", 57), key, anchor) - observation.PacketHash, observation.ResolvedEndpoints = append(make([]byte, 31), 57), nil - inserted, err := store.InsertObservation(ctx, observation) - if err != nil || !inserted { - t.Fatalf("uncaptured observation insert: %v", err) - } - var isNull bool - if err := tx.QueryRow(ctx, "SELECT resolved_endpoints IS NULL FROM packet_observations WHERE packet_hash=$1", observation.PacketHash).Scan(&isNull); err != nil || !isNull { - t.Fatalf("uncaptured resolution was not SQL NULL: %v", err) - } - empty, err := store.GetPacket(ctx, append(make([]byte, 31), 5)) - if err != nil || len(empty.Observations) != 0 { - t.Fatalf("packet awaiting its first observation: %v", err) + // Detail resolves every observation in one batch, per its own IATA. + exec(`INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +VALUES (5,decode($1,'hex'),$2,'YVR',$3,0,1,0)`, fmt.Sprintf("%064x", 1), observer2, anchor.Add(10*time.Second)) + q.calls = 0 + packet, err := store.GetPacket(ctx, append(make([]byte, 31), 1)) + if err != nil || len(packet.Observations) != 2 { + t.Fatalf("detail: %v", err) } -} - -func TestDecodePacketEndpointSnapshot(t *testing.T) { - for _, tc := range []struct { - raw string - captured bool - }{ - {"", false}, {"null", false}, {"[", false}, {"[]", false}, {"{}", false}, - {`{"source":{"confidence":"none","nodes":[]}}`, false}, - {`{"destination":{"confidence":"none","nodes":[]}}`, false}, - {`{"source":{},"destination":null}`, false}, - {`{"source":{"confidence":"high","nodes":[{"publicKey":"aa"}]}}`, true}, - {`{"destination":{"confidence":"ambiguous","nodes":[{"publicKey":"aa"},{"publicKey":"ab"}]}}`, true}, - } { - _, captured := decodePacketEndpointSnapshot(json.RawMessage(tc.raw)) - if captured != tc.captured { - t.Errorf("snapshot %q: captured=%v, want %v", tc.raw, captured, tc.captured) - } + if q.calls != 3 { + t.Errorf("detail used %d queries, want packet, observations and one endpoint batch", q.calls) } + check("YYZ observation", packet.Observations[0].ResolvedSource, "ambiguous", "Alice renamed", "Bob") + check("YVR observation", packet.Observations[1].ResolvedSource, "high", "Alice renamed") } diff --git a/db/packets.go b/db/packets.go index 8172c66f..2acc5d42 100644 --- a/db/packets.go +++ b/db/packets.go @@ -84,14 +84,6 @@ func buildLatestObserverPath(pathLengthByte, hashSize, hopCount *int16, pathByte return pathLength, pathBytesHex } -func decodePacketEndpointSnapshot(raw json.RawMessage) (api.PacketEndpointSnapshot, bool) { - var snapshot *api.PacketEndpointSnapshot - if len(raw) == 0 || json.Unmarshal(raw, &snapshot) != nil || snapshot == nil || !snapshot.HasResolvedNodes() { - return api.PacketEndpointSnapshot{}, false - } - return *snapshot, true -} - func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int16, iatas []string, scopes []string, since, until time.Time, cursor int64, limit int32) (api.Page[api.PacketSummary], error) { if len(iatas) > 0 { return s.listPacketsByIATAs(ctx, payloadTypes, routeTypes, iatas, scopes, since, until, cursor, limit) @@ -125,7 +117,9 @@ func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int1 rows = rows[:limit] } items := make([]api.PacketSummary, 0, len(rows)) + lookups := make([]endpointLookup, 0, len(rows)) for _, v := range rows { + var lookup endpointLookup item := api.PacketSummary{ PacketHash: hex.EncodeToString(v.PacketHash), PayloadType: v.PayloadType, @@ -141,20 +135,20 @@ func (s *Store) ListPackets(ctx context.Context, payloadTypes, routeTypes []int1 item.Summary = &v.Summary } if v.LatestObserverID != (uuid.UUID{}) { - endpoints, _ := decodePacketEndpointSnapshot(v.LatestObserverResolvedEndpoints) item.LatestObserver = &api.PacketLatestObserver{ - ID: v.LatestObserverID, - DisplayName: v.LatestObserverName, - IATA: v.LatestObserverIata, - ResolvedSource: endpoints.Source, - ResolvedDestination: endpoints.Destination, + ID: v.LatestObserverID, + DisplayName: v.LatestObserverName, + IATA: v.LatestObserverIata, } + lookup = endpointLookup{iata: v.LatestObserverIata, ep: parsePacketEndpoints(v.PayloadType, v.RawPayload, v.OriginPubkey)} item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, ) } items = append(items, item) + lookups = append(lookups, lookup) } + s.fillLatestEndpoints(ctx, items, lookups) var nextCursor *int64 if hasMore && len(items) > 0 { last := items[len(items)-1].LastHeardAt @@ -210,7 +204,9 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes scanFloor = rows[0].ScanFloor } items := make([]api.PacketSummary, 0, len(rows)) + lookups := make([]endpointLookup, 0, len(rows)) for _, v := range rows { + var lookup endpointLookup item := api.PacketSummary{ PacketHash: hex.EncodeToString(v.PacketHash), PayloadType: v.PayloadType, @@ -226,20 +222,20 @@ func (s *Store) listPacketsByIATAs(ctx context.Context, payloadTypes, routeTypes item.Summary = &v.Summary } if v.LatestObserverID != (uuid.UUID{}) { - endpoints, _ := decodePacketEndpointSnapshot(v.LatestObserverResolvedEndpoints) item.LatestObserver = &api.PacketLatestObserver{ - ID: v.LatestObserverID, - DisplayName: v.LatestObserverName, - IATA: v.LatestObserverIata, - ResolvedSource: endpoints.Source, - ResolvedDestination: endpoints.Destination, + ID: v.LatestObserverID, + DisplayName: v.LatestObserverName, + IATA: v.LatestObserverIata, } + lookup = endpointLookup{iata: v.LatestObserverIata, ep: parsePacketEndpoints(v.PayloadType, v.RawPayload, v.OriginPubkey)} item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( &v.LatestObserverPathLengthByte, &v.LatestObserverHashSize, &v.LatestObserverHopCount, v.LatestObserverPathBytes, ) } items = append(items, item) + lookups = append(lookups, lookup) } + s.fillLatestEndpoints(ctx, items, lookups) // Cursor follows site-local recency, not the packet's global last_heard_at. var nextCursor *int64 if hasMore && len(rows) > 0 { @@ -272,7 +268,9 @@ func (s *Store) ListPacketsAfterID(ctx context.Context, afterObservationID int64 return nil, err } items := make([]api.PacketSummary, 0, len(rows)) + lookups := make([]endpointLookup, 0, len(rows)) for _, v := range rows { + var lookup endpointLookup item := api.PacketSummary{ PacketHash: hex.EncodeToString(v.PacketHash), PayloadType: v.PayloadType, @@ -288,14 +286,12 @@ func (s *Store) ListPacketsAfterID(ctx context.Context, afterObservationID int64 item.Summary = &v.Summary } if v.LatestObserverID != (uuid.UUID{}) { - endpoints, _ := decodePacketEndpointSnapshot(v.LatestObserverResolvedEndpoints) item.LatestObserver = &api.PacketLatestObserver{ - ID: v.LatestObserverID, - DisplayName: v.LatestObserverName, - IATA: v.LatestObserverIata, - ResolvedSource: endpoints.Source, - ResolvedDestination: endpoints.Destination, + ID: v.LatestObserverID, + DisplayName: v.LatestObserverName, + IATA: v.LatestObserverIata, } + lookup = endpointLookup{iata: v.LatestObserverIata, ep: parsePacketEndpoints(v.PayloadType, v.RawPayload, v.OriginPubkey)} // Inner join here (unlike ListPackets/listPacketsByIATAs' LEFT JOIN LATERAL), so // these are never nil when an observer was joined at all. item.LatestObserver.PathLength, item.LatestObserver.PathBytes = buildLatestObserverPath( @@ -303,7 +299,9 @@ func (s *Store) ListPacketsAfterID(ctx context.Context, afterObservationID int64 ) } items = append(items, item) + lookups = append(lookups, lookup) } + s.fillLatestEndpoints(ctx, items, lookups) return items, nil } @@ -417,42 +415,14 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, } } } - // 1-byte source/destination hashes for the payload types that carry a resolvable - // endpoint (REQUEST, RESPONSE, TEXT_MESSAGE, PATH, ANON_REQ's destination). Constant - // for this packet hash, so parsed once; resolution itself still happens per-observation - // below since candidates depend on the observation's IATA, same as the intermediate - // hop resolution already does. - var sourceHashByte, destHashByte []byte - switch row.PayloadType { - case int16(meshcore.PayloadTypeAnonReq): - if anonReq, err := meshcore.AnonReqFromBytes(row.RawPayload); err == nil { - destHashByte = []byte{anonReq.Destination} - } - case int16(meshcore.PayloadTypeReq): - if req, err := meshcore.RequestFromBytes(row.RawPayload); err == nil { - sourceHashByte = []byte{req.Source} - destHashByte = []byte{req.Destination} - } - case int16(meshcore.PayloadTypeResponse): - if resp, err := meshcore.ResponseFromBytes(row.RawPayload); err == nil { - sourceHashByte = []byte{resp.Source} - destHashByte = []byte{resp.Destination} - } - case int16(meshcore.PayloadTypeTxtMsg): - if txt, err := meshcore.TextMessageFromBytes(row.RawPayload); err == nil { - sourceHashByte = []byte{txt.Source} - destHashByte = []byte{txt.Destination} - } - case int16(meshcore.PayloadTypePath): - if path, err := meshcore.PathFromBytes(row.RawPayload); err == nil { - sourceHashByte = []byte{path.Source} - destHashByte = []byte{path.Destination} - } + // Endpoints resolve against each observation's IATA, all in one batch. + ep := parsePacketEndpoints(row.PayloadType, row.RawPayload, row.OriginPubkey) + lookups := make([]endpointLookup, len(obsRows)) + for i, v := range obsRows { + lookups[i] = endpointLookup{iata: v.Iata, ep: ep} } - // Legacy ADVERT sources need an exact lookup, at most once for this packet. - var resolvedAdvertSource *api.ResolvedNode - advertSourceLookedUp := false - for _, v := range obsRows { + sources, destinations := s.resolveEndpoints(ctx, lookups) + for i, v := range obsRows { obs := api.PacketObservationDetail{ ID: v.ID, ObserverID: v.ObserverID, @@ -496,38 +466,7 @@ func (s *Store) GetPacket(ctx context.Context, packetHash []byte) (*api.Packet, } } obs.ResolvedPath = resolvedPath - if endpoints, captured := decodePacketEndpointSnapshot(v.ResolvedEndpoints); captured { - obs.ResolvedSource = endpoints.Source - obs.ResolvedDestination = endpoints.Destination - } else { - if row.PayloadType == int16(meshcore.PayloadTypeAdvert) { - if !advertSourceLookedUp && row.OriginPubkey != nil { - if nodeID, err := s.GetNodeByPubkey(ctx, row.OriginPubkey); err == nil { - if nodes, err := s.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { - resolvedAdvertSource = nodes[nodeID] - } - } - advertSourceLookedUp = true - } - hop := api.ResolveExactNode(resolvedAdvertSource) - obs.ResolvedSource = &hop - } else if len(sourceHashByte) == 1 { - if r, err := s.ResolveEndpointHashes(ctx, v.Iata, [][]byte{sourceHashByte}); err != nil { - slog.Error(fmt.Sprintf("store: source resolution failed for observation %d", v.ID), "component", "db", "error", err) - } else { - hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] - obs.ResolvedSource = &hop - } - } - if len(destHashByte) == 1 { - if r, err := s.ResolveEndpointHashes(ctx, v.Iata, [][]byte{destHashByte}); err != nil { - slog.Error(fmt.Sprintf("store: destination resolution failed for observation %d", v.ID), "component", "db", "error", err) - } else { - hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] - obs.ResolvedDestination = &hop - } - } - } + obs.ResolvedSource, obs.ResolvedDestination = sources[i], destinations[i] if row.PayloadType == int16(meshcore.PayloadTypeTrace) && len(traceRawHashes) > 0 { // Swap in the trace's own path hashes so PathData's hop-block split (driven by // pathBytes + hashSize) lines up 1:1 with resolvedPath -- the raw SNR bytes @@ -594,7 +533,6 @@ func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservatio CodingRate: &o.CodingRate, SourceBroker: &o.SourceBroker, PayloadType: &o.PayloadType, - ResolvedEndpoints: o.ResolvedEndpoints, AirtimeMs: o.AirtimeMs, } row, err := s.q.InsertObservation(ctx, params) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 1f84bdf3..9c467944 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -489,10 +489,11 @@ SELECT COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes, - po.resolved_endpoints AS latest_observer_resolved_endpoints + p.raw_payload, + p.origin_pubkey FROM packets p LEFT JOIN LATERAL ( - SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes, resolved_endpoints + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -599,12 +600,13 @@ SELECT COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes, - po.resolved_endpoints AS latest_observer_resolved_endpoints + p.raw_payload, + p.origin_pubkey FROM page sh CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( - SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes, resolved_endpoints + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -643,7 +645,8 @@ SELECT po.hash_size AS latest_observer_hash_size, po.hop_count AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes, - po.resolved_endpoints AS latest_observer_resolved_endpoints, + p.raw_payload, + p.origin_pubkey, ts.name AS scope_name FROM packets p JOIN packet_observations po ON po.packet_hash = p.packet_hash @@ -729,10 +732,9 @@ INSERT INTO packet_observations ( coding_rate, source_broker, payload_type, - resolved_endpoints, airtime_ms ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING RETURNING *; @@ -795,6 +797,11 @@ WHERE id = ANY($1::uuid[]); -- name: GetNodeByPubkey :one SELECT id FROM nodes WHERE public_key = $1; +-- name: GetNodesByPubkeys :many +SELECT id, public_key, name, latitude, longitude +FROM nodes +WHERE public_key = ANY(@pubkeys::bytea[]); + -- name: ListNodes :many -- Limit the filtered node page before enriching IATA membership and neighbours. WITH page AS ( @@ -1413,6 +1420,19 @@ CROSS JOIN LATERAL ( WHERE ns.iata = $1 AND ns.prefix_1 = ANY($2::bytea[]); +-- name: ResolveEndpointHashPairs :many +-- Batch form of ResolveEndpointHashes for a page of packets. Matches the cross +-- product of IATAs and hashes; callers pick out the pairs they asked for. +SELECT ns.iata, ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM node_short_ids ns +CROSS JOIN LATERAL ( + SELECT id, name, latitude, longitude, public_key + FROM nodes WHERE id = ns.node_id + LIMIT 1 +) n +WHERE ns.iata = ANY(@iatas::bpchar[]) + AND ns.prefix_1 = ANY(@hashes::bytea[]); + -- name: ResolvePathHashesP2 :many SELECT ns.prefix_4 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key FROM node_short_ids ns diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 4efbfaa5..6a665e77 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -339,6 +339,21 @@ func (mr *MockQuerierMockRecorder) GetNodesByIDs(ctx, dollar_1 any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodesByIDs", reflect.TypeOf((*MockQuerier)(nil).GetNodesByIDs), ctx, dollar_1) } +// GetNodesByPubkeys mocks base method. +func (m *MockQuerier) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]db.GetNodesByPubkeysRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetNodesByPubkeys", ctx, pubkeys) + ret0, _ := ret[0].([]db.GetNodesByPubkeysRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetNodesByPubkeys indicates an expected call of GetNodesByPubkeys. +func (mr *MockQuerierMockRecorder) GetNodesByPubkeys(ctx, pubkeys any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNodesByPubkeys", reflect.TypeOf((*MockQuerier)(nil).GetNodesByPubkeys), ctx, pubkeys) +} + // GetObserverActivityHourly mocks base method. func (m *MockQuerier) GetObserverActivityHourly(ctx context.Context, arg db.GetObserverActivityHourlyParams) ([]db.GetObserverActivityHourlyRow, error) { m.ctrl.T.Helper() @@ -1391,6 +1406,21 @@ func (mr *MockQuerierMockRecorder) RefreshTopTalkers(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopTalkers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopTalkers), ctx) } +// ResolveEndpointHashPairs mocks base method. +func (m *MockQuerier) ResolveEndpointHashPairs(ctx context.Context, arg db.ResolveEndpointHashPairsParams) ([]db.ResolveEndpointHashPairsRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveEndpointHashPairs", ctx, arg) + ret0, _ := ret[0].([]db.ResolveEndpointHashPairsRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveEndpointHashPairs indicates an expected call of ResolveEndpointHashPairs. +func (mr *MockQuerierMockRecorder) ResolveEndpointHashPairs(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveEndpointHashPairs", reflect.TypeOf((*MockQuerier)(nil).ResolveEndpointHashPairs), ctx, arg) +} + // ResolveEndpointHashes mocks base method. func (m *MockQuerier) ResolveEndpointHashes(ctx context.Context, arg db.ResolveEndpointHashesParams) ([]db.ResolveEndpointHashesRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 126b37a4..b5ffb054 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -335,7 +335,6 @@ type PacketObservation struct { CodingRate *int16 `json:"coding_rate"` SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` - ResolvedEndpoints []byte `json:"resolved_endpoints"` AirtimeMs *float32 `json:"airtime_ms"` } diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index d3913c42..232eae3b 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -54,6 +54,7 @@ type Querier interface { // Returns the neighbors of a node with details, ordered by most recently seen. GetNodeNeighbors(ctx context.Context, nodeID uuid.UUID) ([]GetNodeNeighborsRow, error) GetNodesByIDs(ctx context.Context, dollar_1 []uuid.UUID) ([]GetNodesByIDsRow, error) + GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]GetNodesByPubkeysRow, error) // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) @@ -224,6 +225,9 @@ type Querier interface { RefreshTopNodes(ctx context.Context) error RefreshTopObservers(ctx context.Context) error RefreshTopTalkers(ctx context.Context) error + // Batch form of ResolveEndpointHashes for a page of packets. Matches the cross + // product of IATAs and hashes; callers pick out the pairs they asked for. + ResolveEndpointHashPairs(ctx context.Context, arg ResolveEndpointHashPairsParams) ([]ResolveEndpointHashPairsRow, error) // Logical endpoints can be any advertised role, unlike intermediate relay hops. // Endpoint hashes are always one byte; use the existing (iata, prefix_1) index. // LIMIT 1 keeps generic plans on a node PK lookup per candidate instead of diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index d454af2d..38d299c6 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -608,6 +608,46 @@ func (q *Queries) GetNodesByIDs(ctx context.Context, dollar_1 []uuid.UUID) ([]Ge return items, nil } +const getNodesByPubkeys = `-- name: GetNodesByPubkeys :many +SELECT id, public_key, name, latitude, longitude +FROM nodes +WHERE public_key = ANY($1::bytea[]) +` + +type GetNodesByPubkeysRow struct { + ID uuid.UUID `json:"id"` + PublicKey []byte `json:"public_key"` + Name *string `json:"name"` + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` +} + +func (q *Queries) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]GetNodesByPubkeysRow, error) { + rows, err := q.db.Query(ctx, getNodesByPubkeys, pubkeys) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetNodesByPubkeysRow{} + for rows.Next() { + var i GetNodesByPubkeysRow + if err := rows.Scan( + &i.ID, + &i.PublicKey, + &i.Name, + &i.Latitude, + &i.Longitude, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getObserverActivityHourly = `-- name: GetObserverActivityHourly :many SELECT date_bin($3::interval, bucket, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, @@ -2049,13 +2089,12 @@ INSERT INTO packet_observations ( coding_rate, source_broker, payload_type, - resolved_endpoints, airtime_ms ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING -RETURNING id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type, resolved_endpoints, airtime_ms +RETURNING id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type, airtime_ms ` type InsertObservationParams struct { @@ -2076,7 +2115,6 @@ type InsertObservationParams struct { CodingRate *int16 `json:"coding_rate"` SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` - ResolvedEndpoints []byte `json:"resolved_endpoints"` AirtimeMs *float32 `json:"airtime_ms"` } @@ -2102,7 +2140,6 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa arg.CodingRate, arg.SourceBroker, arg.PayloadType, - arg.ResolvedEndpoints, arg.AirtimeMs, ) var i PacketObservation @@ -2125,7 +2162,6 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa &i.CodingRate, &i.SourceBroker, &i.PayloadType, - &i.ResolvedEndpoints, &i.AirtimeMs, ) return i, err @@ -2923,7 +2959,7 @@ func (q *Queries) ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNod } const listObservationsForPacket = `-- name: ListObservationsForPacket :many -SELECT po.id, po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, po.path_bytes, po.rssi, po.snr, po.propagation_time_ms, po.radio_freq_mhz, po.spread_factor, po.bandwidth_khz, po.coding_rate, po.source_broker, po.payload_type, po.resolved_endpoints, po.airtime_ms, o.display_name AS observer_name +SELECT po.id, po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, po.path_bytes, po.rssi, po.snr, po.propagation_time_ms, po.radio_freq_mhz, po.spread_factor, po.bandwidth_khz, po.coding_rate, po.source_broker, po.payload_type, po.airtime_ms, o.display_name AS observer_name FROM packet_observations po LEFT JOIN observers o ON o.id = po.observer_id WHERE po.packet_hash = $1 @@ -2949,7 +2985,6 @@ type ListObservationsForPacketRow struct { CodingRate *int16 `json:"coding_rate"` SourceBroker *string `json:"source_broker"` PayloadType *int16 `json:"payload_type"` - ResolvedEndpoints []byte `json:"resolved_endpoints"` AirtimeMs *float32 `json:"airtime_ms"` ObserverName *string `json:"observer_name"` } @@ -2982,7 +3017,6 @@ func (q *Queries) ListObservationsForPacket(ctx context.Context, packetHash []by &i.CodingRate, &i.SourceBroker, &i.PayloadType, - &i.ResolvedEndpoints, &i.AirtimeMs, &i.ObserverName, ); err != nil { @@ -3216,10 +3250,11 @@ SELECT COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes, - po.resolved_endpoints AS latest_observer_resolved_endpoints + p.raw_payload, + p.origin_pubkey FROM packets p LEFT JOIN LATERAL ( - SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes, resolved_endpoints + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -3249,23 +3284,24 @@ type ListPacketsParams struct { } type ListPacketsRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - Summary string `json:"summary"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` - LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` - LatestObserverHashSize int16 `json:"latest_observer_hash_size"` - LatestObserverHopCount int16 `json:"latest_observer_hop_count"` - LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` - LatestObserverResolvedEndpoints []byte `json:"latest_observer_resolved_endpoints"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + Summary string `json:"summary"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + RawPayload []byte `json:"raw_payload"` + OriginPubkey []byte `json:"origin_pubkey"` } // Returns packets with the latest observation rolled in for display. @@ -3305,7 +3341,8 @@ func (q *Queries) ListPackets(ctx context.Context, arg ListPacketsParams) ([]Lis &i.LatestObserverHashSize, &i.LatestObserverHopCount, &i.LatestObserverPathBytes, - &i.LatestObserverResolvedEndpoints, + &i.RawPayload, + &i.OriginPubkey, ); err != nil { return nil, err } @@ -3344,7 +3381,8 @@ SELECT po.hash_size AS latest_observer_hash_size, po.hop_count AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes, - po.resolved_endpoints AS latest_observer_resolved_endpoints, + p.raw_payload, + p.origin_pubkey, ts.name AS scope_name FROM packets p JOIN packet_observations po ON po.packet_hash = p.packet_hash @@ -3369,22 +3407,23 @@ type ListPacketsAfterIDParams struct { } type ListPacketsAfterIDRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - Summary string `json:"summary"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` - LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` - LatestObserverHashSize int16 `json:"latest_observer_hash_size"` - LatestObserverHopCount int16 `json:"latest_observer_hop_count"` - LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` - LatestObserverResolvedEndpoints []byte `json:"latest_observer_resolved_endpoints"` - ScopeName *string `json:"scope_name"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + Summary string `json:"summary"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + RawPayload []byte `json:"raw_payload"` + OriginPubkey []byte `json:"origin_pubkey"` + ScopeName *string `json:"scope_name"` } // Returns packets with observations after the given observation ID, ordered oldest first. @@ -3420,7 +3459,8 @@ func (q *Queries) ListPacketsAfterID(ctx context.Context, arg ListPacketsAfterID &i.LatestObserverHashSize, &i.LatestObserverHopCount, &i.LatestObserverPathBytes, - &i.LatestObserverResolvedEndpoints, + &i.RawPayload, + &i.OriginPubkey, &i.ScopeName, ); err != nil { return nil, err @@ -3509,12 +3549,13 @@ SELECT COALESCE(po.hash_size, 0::smallint) AS latest_observer_hash_size, COALESCE(po.hop_count, 0::smallint) AS latest_observer_hop_count, po.path_bytes AS latest_observer_path_bytes, - po.resolved_endpoints AS latest_observer_resolved_endpoints + p.raw_payload, + p.origin_pubkey FROM page sh CROSS JOIN saturation sat JOIN packets p ON p.packet_hash = sh.packet_hash LEFT JOIN LATERAL ( - SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes, resolved_endpoints + SELECT observer_id, iata, path_length_byte, hash_size, hop_count, path_bytes FROM packet_observations WHERE packet_hash = p.packet_hash ORDER BY heard_at DESC @@ -3538,26 +3579,27 @@ type ListPacketsByIATAsParams struct { } type ListPacketsByIATAsRow struct { - PacketHash []byte `json:"packet_hash"` - PayloadType int16 `json:"payload_type"` - Summary string `json:"summary"` - RouteType int16 `json:"route_type"` - FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` - LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` - ScopeID *int32 `json:"scope_id"` - ScopeName *string `json:"scope_name"` - SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` - ScanSaturated bool `json:"scan_saturated"` - ScanFloor pgtype.Timestamptz `json:"scan_floor"` - ObservationCount int64 `json:"observation_count"` - LatestObserverID uuid.UUID `json:"latest_observer_id"` - LatestObserverName *string `json:"latest_observer_name"` - LatestObserverIata string `json:"latest_observer_iata"` - LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` - LatestObserverHashSize int16 `json:"latest_observer_hash_size"` - LatestObserverHopCount int16 `json:"latest_observer_hop_count"` - LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` - LatestObserverResolvedEndpoints []byte `json:"latest_observer_resolved_endpoints"` + PacketHash []byte `json:"packet_hash"` + PayloadType int16 `json:"payload_type"` + Summary string `json:"summary"` + RouteType int16 `json:"route_type"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ScopeID *int32 `json:"scope_id"` + ScopeName *string `json:"scope_name"` + SiteHeardAt pgtype.Timestamptz `json:"site_heard_at"` + ScanSaturated bool `json:"scan_saturated"` + ScanFloor pgtype.Timestamptz `json:"scan_floor"` + ObservationCount int64 `json:"observation_count"` + LatestObserverID uuid.UUID `json:"latest_observer_id"` + LatestObserverName *string `json:"latest_observer_name"` + LatestObserverIata string `json:"latest_observer_iata"` + LatestObserverPathLengthByte int16 `json:"latest_observer_path_length_byte"` + LatestObserverHashSize int16 `json:"latest_observer_hash_size"` + LatestObserverHopCount int16 `json:"latest_observer_hop_count"` + LatestObserverPathBytes []byte `json:"latest_observer_path_bytes"` + RawPayload []byte `json:"raw_payload"` + OriginPubkey []byte `json:"origin_pubkey"` } // IATA-filtered packet list, driven from idx_observations_iata_heard. @@ -3612,7 +3654,8 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs &i.LatestObserverHashSize, &i.LatestObserverHopCount, &i.LatestObserverPathBytes, - &i.LatestObserverResolvedEndpoints, + &i.RawPayload, + &i.OriginPubkey, ); err != nil { return nil, err } @@ -3951,6 +3994,63 @@ func (q *Queries) RefreshTopTalkers(ctx context.Context) error { return err } +const resolveEndpointHashPairs = `-- name: ResolveEndpointHashPairs :many +SELECT ns.iata, ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key +FROM node_short_ids ns +CROSS JOIN LATERAL ( + SELECT id, name, latitude, longitude, public_key + FROM nodes WHERE id = ns.node_id + LIMIT 1 +) n +WHERE ns.iata = ANY($1::bpchar[]) + AND ns.prefix_1 = ANY($2::bytea[]) +` + +type ResolveEndpointHashPairsParams struct { + Iatas []string `json:"iatas"` + Hashes [][]byte `json:"hashes"` +} + +type ResolveEndpointHashPairsRow struct { + Iata string `json:"iata"` + Hash []byte `json:"hash"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + Latitude *float64 `json:"latitude"` + Longitude *float64 `json:"longitude"` + PublicKey []byte `json:"public_key"` +} + +// Batch form of ResolveEndpointHashes for a page of packets. Matches the cross +// product of IATAs and hashes; callers pick out the pairs they asked for. +func (q *Queries) ResolveEndpointHashPairs(ctx context.Context, arg ResolveEndpointHashPairsParams) ([]ResolveEndpointHashPairsRow, error) { + rows, err := q.db.Query(ctx, resolveEndpointHashPairs, arg.Iatas, arg.Hashes) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ResolveEndpointHashPairsRow{} + for rows.Next() { + var i ResolveEndpointHashPairsRow + if err := rows.Scan( + &i.Iata, + &i.Hash, + &i.NodeID, + &i.Name, + &i.Latitude, + &i.Longitude, + &i.PublicKey, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const resolveEndpointHashes = `-- name: ResolveEndpointHashes :many SELECT ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key FROM node_short_ids ns diff --git a/internal/api/packets.go b/internal/api/packets.go index bb4ac37e..2b2e4ebf 100644 --- a/internal/api/packets.go +++ b/internal/api/packets.go @@ -94,21 +94,6 @@ type ResolvedHop struct { Nodes []ResolvedNode `json:"nodes"` // empty for "none", one for "high", multiple for "ambiguous" } -// PacketEndpointSnapshot is the internal storage shape for an observation's -// endpoint resolution at ingest. Names and confidence are preserved even when -// the current node registry changes. REST/WS expose the existing per-endpoint fields. -type PacketEndpointSnapshot struct { - Source *ResolvedHop `json:"source,omitempty"` - Destination *ResolvedHop `json:"destination,omitempty"` -} - -// HasResolvedNodes distinguishes a historical capture from an unresolved lookup -// that should be retried when a node becomes known (including a first advert). -func (s PacketEndpointSnapshot) HasResolvedNodes() bool { - return (s.Source != nil && len(s.Source.Nodes) > 0) || - (s.Destination != nil && len(s.Destination.Nodes) > 0) -} - // ResolvedNode is a node reference within a resolved path hop. type ResolvedNode struct { ID uuid.UUID `json:"id"` diff --git a/internal/api/packets_test.go b/internal/api/packets_test.go index 2befa568..f430f60a 100644 --- a/internal/api/packets_test.go +++ b/internal/api/packets_test.go @@ -10,28 +10,6 @@ import ( "github.com/meshcore-go/meshcore-go" ) -func TestPacketEndpointSnapshotHasResolvedNodes(t *testing.T) { - none := &api.ResolvedHop{Confidence: "none", Nodes: []api.ResolvedNode{}} - known := &api.ResolvedHop{Confidence: "high", Nodes: []api.ResolvedNode{{PublicKey: "aa"}}} - for _, tc := range []struct { - name string - snapshot api.PacketEndpointSnapshot - want bool - }{ - {"absent", api.PacketEndpointSnapshot{}, false}, - {"unresolved source", api.PacketEndpointSnapshot{Source: none}, false}, - {"both unresolved", api.PacketEndpointSnapshot{Source: none, Destination: none}, false}, - {"source only", api.PacketEndpointSnapshot{Source: known, Destination: none}, true}, - {"destination only", api.PacketEndpointSnapshot{Destination: known}, true}, - } { - t.Run(tc.name, func(t *testing.T) { - if got := tc.snapshot.HasResolvedNodes(); got != tc.want { - t.Fatalf("resolved nodes = %v, want %v", got, tc.want) - } - }) - } -} - func TestPayloadTypeName(t *testing.T) { tests := []struct { input int16 diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index d5176c1d..657da999 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -55,7 +55,6 @@ type InsertObservationParams struct { CodingRate int16 SourceBroker string PayloadType int16 - ResolvedEndpoints json.RawMessage AirtimeMs *float32 // nil when the observer never reported costable radio settings } @@ -763,7 +762,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if err != nil { w.log.Error(fmt.Sprintf("db: get observer radio failed for %s", pubkeyHex), "error", err) } - // Save the same endpoint resolution used by the live event in the observation INSERT. + // Endpoints for the live event only; stored rows resolve them at read time. var resolvedSource, resolvedDestination *api.ResolvedHop if packet.PayloadType() == meshcore.PayloadTypeAdvert && originPubkey != nil { // Exact match: ADVERT carries the sender's real identity pubkey, not a @@ -786,15 +785,6 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ resolvedDestination = &hop } } - var resolvedEndpoints json.RawMessage - snapshot := api.PacketEndpointSnapshot{Source: resolvedSource, Destination: resolvedDestination} - if snapshot.HasResolvedNodes() { - resolvedEndpoints, err = json.Marshal(snapshot) - if err != nil { - w.log.Error("endpoint snapshot encoding failed", "error", err) - resolvedEndpoints = nil // optional enrichment must not discard the observation - } - } // Airtime is costed from the frame as received; zero radio columns mean the // observer never reported its settings, so there is nothing to cost. var airtimeMs *float32 @@ -820,7 +810,6 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ CodingRate: radio.CR, SourceBroker: w.cfg.BrokerName, PayloadType: int16(packet.PayloadType()), - ResolvedEndpoints: resolvedEndpoints, AirtimeMs: airtimeMs, } inserted, err := w.db.InsertObservation(ctx, oParams) diff --git a/internal/ingest/packet_endpoints_test.go b/internal/ingest/packet_endpoints_test.go index 61a47063..3611f6f2 100644 --- a/internal/ingest/packet_endpoints_test.go +++ b/internal/ingest/packet_endpoints_test.go @@ -5,118 +5,94 @@ package ingest import ( "context" - "encoding/hex" "encoding/json" - "errors" - "math" "reflect" "testing" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" "github.com/google/uuid" "github.com/meshcore-go/meshcore-go" ) type endpointCaptureDB struct { *stubDB - node api.ResolvedNode - observed []InsertObservationParams - lookups int - pathLookups int - missingNode bool + node api.ResolvedNode + observed int } func (s *endpointCaptureDB) GetNodeByPubkey(context.Context, []byte) (uuid.UUID, error) { - if s.missingNode { - return uuid.Nil, errors.New("node not yet advertised") - } return s.node.ID, nil } - func (s *endpointCaptureDB) UpsertNode(context.Context, UpsertNodeParams, RadioSettings) (uuid.UUID, error) { - s.missingNode = false return s.node.ID, nil } func (s *endpointCaptureDB) GetNodesByIDs(context.Context, []uuid.UUID) (map[uuid.UUID]*api.ResolvedNode, error) { return map[uuid.UUID]*api.ResolvedNode{s.node.ID: &s.node}, nil } -func (s *endpointCaptureDB) ResolvePathHashes(_ context.Context, _ string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { - if len(hashes) > 0 { - s.pathLookups++ - } - return nil, nil // a companion must not be found by the relay-only resolver -} - -func (s *endpointCaptureDB) ResolveEndpointHashes(_ context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { - if len(hashes) == 0 { - return nil, nil - } - s.lookups++ - if iata != "YYZ" || s.missingNode { - return nil, nil +func (s *endpointCaptureDB) ResolveEndpointHashes(_ context.Context, _ string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + if len(hashes) == 1 && hashes[0][0] == 0xaa { + return map[string][]api.ResolvedPathEntry{"aa": {{NodeID: s.node.ID, Name: s.node.Name, PublicKey: []byte{0xaa}}}}, nil } - return map[string][]api.ResolvedPathEntry{ - "aa": {{NodeID: s.node.ID, Name: s.node.Name, PublicKey: []byte{0xaa}}, {NodeID: uuid.Nil, PublicKey: []byte{0xab}}}, - }, nil + return nil, nil } - -func (s *endpointCaptureDB) InsertObservation(_ context.Context, observation InsertObservationParams) (bool, error) { - s.observed = append(s.observed, observation) +func (s *endpointCaptureDB) InsertObservation(context.Context, InsertObservationParams) (bool, error) { + s.observed++ return true, nil } -func TestHandlePacketCapturesEndpoints(t *testing.T) { - name := "Companion 👋" - for _, kind := range []string{"advert", "first advert", "direct message", "unresolved direct", "unaddressed", "encoding failure"} { +// Stored rows resolve endpoints at read time, but live events still carry them. +func TestHandlePacketBroadcastsEndpoints(t *testing.T) { + name := "Companion" + for _, kind := range []string{"advert", "direct message"} { t.Run(kind, func(t *testing.T) { w, base := newTestWorker() db := &endpointCaptureDB{stubDB: base, node: api.ResolvedNode{ID: uuid.New(), Name: &name, PublicKey: "aa"}} w.db = db + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + w.hub.AddScope(client, "endpoints", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + defer w.hub.Remove(client) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := buildAdvertPacket(t, false) - want := api.PacketEndpointSnapshot{} - switch kind { - case "advert": - hop := api.ResolveExactNode(&db.node) - want.Source = &hop - case "first advert": - db.missingNode = true - case "direct message", "unresolved direct": - // Destination, source, MAC and a minimal ciphertext envelope. + wantSource := api.ResolveExactNode(&db.node) + var wantDestination *api.ResolvedHop + if kind == "direct message" { packet = &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeTxtMsg, 0), Payload: append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...)} - db.missingNode = kind == "unresolved direct" - resolved, _ := db.ResolveEndpointHashes(context.Background(), "YYZ", [][]byte{{0xaa}}) - source := api.BuildResolvedPath([][]byte{{0xaa}}, resolved)[0] - destination := api.BuildResolvedPath([][]byte{{0xbb}}, nil)[0] - want.Source, want.Destination = &source, &destination - db.lookups = 0 - case "unaddressed": - packet = buildTracePacket(t) - case "encoding failure": - nan := math.NaN() - db.node.Latitude = &nan + wantSource = api.BuildResolvedPath([][]byte{{0xaa}}, map[string][]api.ResolvedPathEntry{"aa": {{NodeID: db.node.ID, Name: &name, PublicKey: []byte{0xaa}}}})[0] + d := api.BuildResolvedPath([][]byte{{0xbb}}, nil)[0] + wantDestination = &d } - w.handlePacket(context.Background(), "YYZ", hex.EncodeToString([]byte{1, 2}), packetEnvelope(t, packet)) - if len(db.observed) != 1 { - t.Fatalf("observation was lost or written twice: %d", len(db.observed)) + w.handlePacket(ctx, "YYZ", "0102", packetEnvelope(t, packet)) + if db.observed != 1 { + t.Fatalf("observation written %d times", db.observed) } - if kind == "first advert" || kind == "unresolved direct" || kind == "unaddressed" || kind == "encoding failure" { - if db.observed[0].ResolvedEndpoints != nil { - t.Fatal("empty or failed endpoint resolution must remain SQL NULL") - } - if kind == "first advert" && db.missingNode { - t.Fatal("advert side effects did not make the node available for a later lookup") + for { + select { + case event := <-client.Send: + if event.Type != hub.EventPacketObservation { + continue + } + var got struct { + Observation struct { + ResolvedSource *api.ResolvedHop `json:"resolvedSource"` + ResolvedDestination *api.ResolvedHop `json:"resolvedDestination"` + } `json:"observation"` + } + if err := json.Unmarshal(event.Payload, &got); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got.Observation.ResolvedSource, &wantSource) || !reflect.DeepEqual(got.Observation.ResolvedDestination, wantDestination) { + t.Fatalf("live endpoints: got %+v / %+v", got.Observation.ResolvedSource, got.Observation.ResolvedDestination) + } + return + case <-ctx.Done(): + t.Fatal("packet observation event missing") } - return - } - var got api.PacketEndpointSnapshot - if err := json.Unmarshal(db.observed[0].ResolvedEndpoints, &got); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("captured endpoints differ: got %+v, want %+v", got, want) - } - if kind == "direct message" && (db.lookups != 2 || db.pathLookups != 0) { - t.Fatalf("wrong resolver or repeated work: endpoint=%d relay=%d", db.lookups, db.pathLookups) } }) } From dec643a2ade712cd60feb2bc761a5654c7c82714 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Sun, 27 Sep 2026 11:41:11 -0400 Subject: [PATCH 053/141] feat(ws): opt-in allowed origins and observer key (#170) --- cmd/beacon/main.go | 1 + config.yaml.example | 3 + internal/api/router/options.go | 1 + internal/api/router/router.go | 2 +- internal/config/config.go | 23 +++++ internal/config/ws_connect_test.go | 32 +++++++ internal/hub/hub.go | 86 +++++++++++++------ internal/hub/hub_test.go | 65 ++++++++++++-- internal/ingest/ingest.go | 35 +++++--- internal/ingest/observer_key_test.go | 80 ++++++++++++++++++ internal/ingest/packet.go | 22 ++--- internal/ws/connect_limit_test.go | 4 +- internal/ws/handler.go | 35 ++++---- internal/ws/handler_test.go | 122 +++++++++++++++++++++++++++ 14 files changed, 440 insertions(+), 71 deletions(-) create mode 100644 internal/ingest/observer_key_test.go create mode 100644 internal/ws/handler_test.go diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 7c54b7c1..2f170439 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -309,6 +309,7 @@ func main() { r := router.New(h, reader, []*ingest.Worker{broker1, broker2}, router.Options{ MaxConnsPerIP: resolved.MaxConnsPerIP, MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, + WSAllowedOrigins: cfg.WebSocket.AllowedOrigins, CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, AdminRoutes: map[string]http.Handler{ "/accounts": handlers.AccountsRouter(store), diff --git a/config.yaml.example b/config.yaml.example index de12bef4..e50906ac 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -149,6 +149,9 @@ websocket: # Within this rate budget, a full concurrent cap accepts then closes with 1013 # before hello, allowing browsers to back off without evicting existing clients. max_connects_per_minute: 10 + # Other sites allowed to open /ws. Exact scheme://host[:port], no wildcards. + #allowed_origins: + # - https://example.com # Node staleness, deletion, and clock-drift thresholds. #nodes: diff --git a/internal/api/router/options.go b/internal/api/router/options.go index ec9e1fbf..757a6593 100644 --- a/internal/api/router/options.go +++ b/internal/api/router/options.go @@ -14,6 +14,7 @@ import ( type Options struct { MaxConnsPerIP int MaxConnectsPerMinute int + WSAllowedOrigins []string CORS config.CORSConfig Server config.ServerConfig Auth config.AuthConfig diff --git a/internal/api/router/router.go b/internal/api/router/router.go index 447d407e..58bbfac5 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -91,7 +91,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) )) // ── WebSocket ──────────────────────────────────────────────────────────── - r.Get("/ws", ws.Handler(h, reader, opts.MaxConnsPerIP, opts.MaxConnectsPerMinute)) + r.Get("/ws", ws.Handler(h, reader, opts.MaxConnsPerIP, opts.MaxConnectsPerMinute, opts.WSAllowedOrigins)) // ── Public REST API (v1) ───────────────────────────────────────────────── r.Route("/api/v1", func(r chi.Router) { diff --git a/internal/config/config.go b/internal/config/config.go index 5f532726..8473c771 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,8 +8,10 @@ package config import ( "fmt" "net/netip" + "net/url" "os" "path/filepath" + "strings" "time" "gopkg.in/yaml.v3" @@ -241,6 +243,8 @@ type WebSocketConfig struct { // MaxConnectsPerMinute limits upgrade attempts, including failed handshakes. // Zero/omitted defaults to 10; IPv6 addresses share a /64 attempt budget. MaxConnectsPerMinute int `yaml:"max_connects_per_minute"` + // AllowedOrigins are extra exact origins allowed to open /ws. Defaults to same-host only. + AllowedOrigins []string `yaml:"allowed_origins"` } // PacketsConfig controls packet retention behaviour. @@ -396,6 +400,11 @@ func Load(path string) (*Config, error) { if cfg.WebSocket.MaxConnectsPerMinute < 0 { return nil, fmt.Errorf("websocket.max_connects_per_minute must be positive or zero for the default") } + for i, origin := range cfg.WebSocket.AllowedOrigins { + if err := validateOrigin(origin); err != nil { + return nil, fmt.Errorf("websocket.allowed_origins[%d]: %w", i, err) + } + } configDir := filepath.Dir(path) for iata, details := range cfg.IATAs { if details.BorderFile != "" && !filepath.IsAbs(details.BorderFile) { @@ -406,6 +415,20 @@ func Load(path string) (*Config, error) { return cfg, nil } +// validateOrigin rejects wildcards, which the WebSocket library would treat as patterns. +func validateOrigin(origin string) error { + u, err := url.Parse(origin) + if err != nil { + return fmt.Errorf("invalid origin %q: %w", origin, err) + } + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || + u.Path != "" || u.RawQuery != "" || u.Fragment != "" || u.Opaque != "" || + strings.ContainsAny(origin, `*?[]\`) { + return fmt.Errorf("origin %q must be exactly scheme://host[:port] with an http or https scheme", origin) + } + return nil +} + // Resolve returns a ResolvedConfig with defaults applied for any zero values. func Resolve(cfg *Config) ResolvedConfig { r := ResolvedConfig{ diff --git a/internal/config/ws_connect_test.go b/internal/config/ws_connect_test.go index 5914707c..b285b82b 100644 --- a/internal/config/ws_connect_test.go +++ b/internal/config/ws_connect_test.go @@ -35,3 +35,35 @@ func TestWebSocketConnectConfig(t *testing.T) { }) } } + +func TestWebSocketAllowedOrigins(t *testing.T) { + for _, tc := range []struct { + name, origin string + wantError bool + }{ + {"https", "https://example.com", false}, + {"http with port", "http://localhost:5173", false}, + {"wildcard", "*", true}, + {"wildcard host", "https://*.example.com", true}, + {"no scheme", "example.com", true}, + {"other scheme", "ftp://example.com", true}, + {"path", "https://example.com/", true}, + {"query", "https://example.com?x=1", true}, + {"userinfo", "https://user@example.com", true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + yaml := "websocket: {allowed_origins: [\"" + tc.origin + "\"]}" + if err := os.WriteFile(path, []byte(yaml), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if (err != nil) != tc.wantError { + t.Fatalf("Load error = %v, want error = %v", err, tc.wantError) + } + if err == nil && (len(cfg.WebSocket.AllowedOrigins) != 1 || cfg.WebSocket.AllowedOrigins[0] != tc.origin) { + t.Fatalf("allowed_origins = %v", cfg.WebSocket.AllowedOrigins) + } + }) + } +} diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 3671afeb..d4086a1d 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -20,6 +20,7 @@ import ( "fmt" "log/slog" "slices" + "sync/atomic" ) // EventType identifies the kind of server-push event. These match the @@ -40,10 +41,13 @@ const ( // fields for clients that opted into them via configure (currently just // resolvedPath on packetObservation events). Left nil for event types that // don't have an opt-in variant; the hub falls back to Payload in that case. +// The WithKey variants also carry observerPublicKey. type Event struct { - Type EventType - Payload json.RawMessage - PayloadResolved json.RawMessage + Type EventType + Payload json.RawMessage + PayloadResolved json.RawMessage + PayloadWithKey json.RawMessage + PayloadResolvedWithKey json.RawMessage // Routing metadata used by the hub to match subscriptions. // Populated by the ingest layer before calling Broadcast. @@ -73,11 +77,35 @@ type Client struct { subscriptions map[string]Scope // OR semantics: event matches if it matches any scope entry // ResolvePath is a connection-wide opt-in (not per-subscription), set via - // SetResolvePath ("configure" WS messages). Freely toggleable at any + // Configure ("configure" WS messages). Freely toggleable at any // point during the connection's lifetime. Only ever read/written inside // Run(), so it needs no locking despite Client being shared with the WS // goroutines. ResolvePath bool + // IncludeObserverKey works like ResolvePath, for observerPublicKey. + IncludeObserverKey bool +} + +// ClientOptions are the connection-wide settings a "configure" message sets. +type ClientOptions struct { + ResolvePath bool + IncludeObserverKey bool +} + +// payloadFor falls back to a narrower variant if the event lacks one. +func (e Event) payloadFor(c *Client) json.RawMessage { + if c.IncludeObserverKey { + if c.ResolvePath && e.PayloadResolvedWithKey != nil { + return e.PayloadResolvedWithKey + } + if !c.ResolvePath && e.PayloadWithKey != nil { + return e.PayloadWithKey + } + } + if c.ResolvePath && e.PayloadResolved != nil { + return e.PayloadResolved + } + return e.Payload } // matches returns true if the event satisfies at least one of the client's @@ -122,10 +150,13 @@ type Hub struct { unsubscribe chan unsubscribeMsg remove chan *Client broadcast chan Event + + // observerKeyClients counts registered clients with IncludeObserverKey set. + observerKeyClients atomic.Int64 } // subscribeMsg carries a client registration, a scope subscription, or a -// configure (resolvePath toggle) request — all three go through this single +// configure request — all three go through this single // channel, not separate ones, specifically so that Go's same-channel FIFO // guarantee orders them relative to NewClient's registration message. A // separate "configure" channel raced against registration: select() has no @@ -139,7 +170,7 @@ type subscribeMsg struct { subscriptionID string isConfigure bool - resolvePath bool + options ClientOptions } type unsubscribeMsg struct { @@ -147,14 +178,6 @@ type unsubscribeMsg struct { subscriptionID string } -// configureMsg carries a connection-wide setting change, decoupled from the -// subscribe/unsubscribe scope mechanics so it can be toggled independently -// and repeatedly over the life of a connection. -type configureMsg struct { - client *Client - resolvePath bool -} - // New creates a Hub. Call Run() in a goroutine before using it. func New() *Hub { return &Hub{ @@ -192,13 +215,15 @@ func (h *Hub) RemoveScope(c *Client, id string) { h.unsubscribe <- unsubscribeMsg{client: c, subscriptionID: id} } -// SetResolvePath toggles a client's opt-in to the resolvedPath variant of -// packetObservation events. Unlike scopes, this is a single connection-wide -// flag (not additive/OR'd) and can be flipped on or off at any point during -// the connection's lifetime — takes effect on the next broadcast after the -// hub processes it. -func (h *Hub) SetResolvePath(c *Client, enabled bool) { - h.subscribe <- subscribeMsg{client: c, isConfigure: true, resolvePath: enabled} +// Configure replaces a client's connection-wide options. Unlike scopes, it is +// not additive. +func (h *Hub) Configure(c *Client, opts ClientOptions) { + h.subscribe <- subscribeMsg{client: c, isConfigure: true, options: opts} +} + +// ObserverKeyWanted lets ingest skip the key variants when nobody wants them. +func (h *Hub) ObserverKeyWanted() bool { + return h.observerKeyClients.Load() > 0 } // Remove deregisters a client and closes its Send channel. @@ -236,9 +261,17 @@ func (h *Hub) Run() { // Registration with no scope yet (NewClient path). clients[msg.client] = struct{}{} case msg.isConfigure: - // SetResolvePath path — client must already be registered. + // Configure path — client must already be registered. if _, ok := clients[msg.client]; ok { - msg.client.ResolvePath = msg.resolvePath + if msg.client.IncludeObserverKey != msg.options.IncludeObserverKey { + if msg.options.IncludeObserverKey { + h.observerKeyClients.Add(1) + } else { + h.observerKeyClients.Add(-1) + } + } + msg.client.ResolvePath = msg.options.ResolvePath + msg.client.IncludeObserverKey = msg.options.IncludeObserverKey } default: // AddScope path — client must already be registered. @@ -255,6 +288,9 @@ func (h *Hub) Run() { case c := <-h.remove: if _, ok := clients[c]; ok { delete(clients, c) + if c.IncludeObserverKey { + h.observerKeyClients.Add(-1) + } close(c.Send) close(c.laggedCH) } @@ -265,9 +301,7 @@ func (h *Hub) Run() { continue } outEvt := evt - if c.ResolvePath && evt.PayloadResolved != nil { - outEvt.Payload = evt.PayloadResolved - } + outEvt.Payload = evt.payloadFor(c) select { case c.Send <- outEvt: default: diff --git a/internal/hub/hub_test.go b/internal/hub/hub_test.go index 5ced37be..3a992238 100644 --- a/internal/hub/hub_test.go +++ b/internal/hub/hub_test.go @@ -269,7 +269,7 @@ func TestHub_ResolvePath_OptedIn_GetsResolvedPayload(t *testing.T) { h := runHub(t) c := h.NewClient() h.AddScope(c, "sub1", Scope{Events: []EventType{EventPacketObservation}}) - h.SetResolvePath(c, true) + h.Configure(c, ClientOptions{ResolvePath: true}) time.Sleep(10 * time.Millisecond) @@ -294,7 +294,7 @@ func TestHub_ResolvePath_DefaultOff_GetsBasePayload(t *testing.T) { h := runHub(t) c := h.NewClient() h.AddScope(c, "sub1", Scope{Events: []EventType{EventPacketObservation}}) - // no SetResolvePath call — default is off + // no Configure call — default is off time.Sleep(10 * time.Millisecond) @@ -320,7 +320,7 @@ func TestHub_ResolvePath_OptedIn_NoResolvedVariant_FallsBackToBase(t *testing.T) c := h.NewClient() // e.g. nodeUpdate events never carry a PayloadResolved variant h.AddScope(c, "sub1", Scope{Events: []EventType{EventNodeUpdate}}) - h.SetResolvePath(c, true) + h.Configure(c, ClientOptions{ResolvePath: true}) time.Sleep(10 * time.Millisecond) @@ -367,15 +367,70 @@ func TestHub_ResolvePath_ToggleableLive(t *testing.T) { t.Errorf("expected base payload before opting in, got %s", got) } - h.SetResolvePath(c, true) + h.Configure(c, ClientOptions{ResolvePath: true}) time.Sleep(10 * time.Millisecond) if got := broadcastAndRead(); got != `{"resolvedPath":[{"confidence":"high"}]}` { t.Errorf("expected resolved payload after opting in, got %s", got) } - h.SetResolvePath(c, false) + h.Configure(c, ClientOptions{}) time.Sleep(10 * time.Millisecond) if got := broadcastAndRead(); got != `{"resolvedPath":null}` { t.Errorf("expected base payload after opting back out, got %s", got) } } + +func TestEvent_PayloadFor(t *testing.T) { + full := Event{ + Payload: json.RawMessage(`base`), + PayloadResolved: json.RawMessage(`resolved`), + PayloadWithKey: json.RawMessage(`key`), + PayloadResolvedWithKey: json.RawMessage(`resolved+key`), + } + baseOnly := Event{Payload: json.RawMessage(`base`), PayloadResolved: json.RawMessage(`resolved`)} + for _, tc := range []struct { + name string + evt Event + opts ClientOptions + want string + }{ + {"default", full, ClientOptions{}, "base"}, + {"resolve", full, ClientOptions{ResolvePath: true}, "resolved"}, + {"key", full, ClientOptions{IncludeObserverKey: true}, "key"}, + {"both", full, ClientOptions{ResolvePath: true, IncludeObserverKey: true}, "resolved+key"}, + {"key missing", baseOnly, ClientOptions{IncludeObserverKey: true}, "base"}, + {"both, key missing", baseOnly, ClientOptions{ResolvePath: true, IncludeObserverKey: true}, "resolved"}, + } { + t.Run(tc.name, func(t *testing.T) { + c := &Client{ResolvePath: tc.opts.ResolvePath, IncludeObserverKey: tc.opts.IncludeObserverKey} + if got := string(tc.evt.payloadFor(c)); got != tc.want { + t.Fatalf("payloadFor = %q, want %q", got, tc.want) + } + }) + } +} + +func TestHub_ObserverKeyWanted_TracksOptIns(t *testing.T) { + h := runHub(t) + waitFor := func(want bool) { + t.Helper() + deadline := time.Now().Add(time.Second) + for h.ObserverKeyWanted() != want { + if time.Now().After(deadline) { + t.Fatalf("ObserverKeyWanted = %t, want %t", !want, want) + } + time.Sleep(time.Millisecond) + } + } + a, b := h.NewClient(), h.NewClient() + if h.ObserverKeyWanted() { + t.Fatal("no client opted in yet") + } + h.Configure(a, ClientOptions{IncludeObserverKey: true}) + h.Configure(a, ClientOptions{IncludeObserverKey: true}) // repeat must not double count + h.Configure(b, ClientOptions{IncludeObserverKey: true}) + waitFor(true) + h.Configure(a, ClientOptions{ResolvePath: true}) + h.Remove(b) + waitFor(false) +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index b322569b..4961344a 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -378,32 +378,43 @@ func (w *Worker) broadcast(eventType hub.EventType, iata string, payloadType uin }) } -// broadcastPacketObservation marshals evt twice: once as-is (the default +// broadcastPacketObservation marshals evt once as-is (the default // payload every packetObservation subscriber gets) and once with // resolvedPath populated (delivered only to connections that opted in via // the "configure" WS message; see hub.Client.ResolvePath). resolvedPath is // passed in rather than computed here because the caller already has the // path-hash resolution results in hand from other per-packet work (known // route detection, capability detection) — this adds no extra DB calls. -func (w *Worker) broadcastPacketObservation(iata string, payloadType uint8, evt packetObservationEvent, resolvedPath []api.ResolvedHop) { +func (w *Worker) broadcastPacketObservation(iata string, payloadType uint8, evt packetObservationEvent, resolvedPath []api.ResolvedHop, observerKey string) { base, err := json.Marshal(evt) if err != nil { w.log.Error("failed to marshal packetObservation event", "error", err) return } + out := hub.Event{ + Type: hub.EventPacketObservation, + Payload: base, + IATA: iata, + PayloadType: payloadType, + } evt.Observation.ResolvedPath = resolvedPath - resolved, err := json.Marshal(evt) - if err != nil { + if out.PayloadResolved, err = json.Marshal(evt); err != nil { w.log.Error("failed to marshal packetObservation event (resolved variant)", "error", err) - resolved = nil // fall back to base-only; not fatal + out.PayloadResolved = nil } - w.hub.Broadcast(hub.Event{ - Type: hub.EventPacketObservation, - Payload: base, - PayloadResolved: resolved, - IATA: iata, - PayloadType: payloadType, - }) + if w.hub.ObserverKeyWanted() { + evt.Observation.ObserverPublicKey = observerKey + if out.PayloadResolvedWithKey, err = json.Marshal(evt); err != nil { + w.log.Error("failed to marshal packetObservation event (resolved key variant)", "error", err) + out.PayloadResolvedWithKey = nil + } + evt.Observation.ResolvedPath = nil + if out.PayloadWithKey, err = json.Marshal(evt); err != nil { + w.log.Error("failed to marshal packetObservation event (key variant)", "error", err) + out.PayloadWithKey = nil + } + } + w.hub.Broadcast(out) } // parseNumber handles RSSI and SNR fields that different observer types send as diff --git a/internal/ingest/observer_key_test.go b/internal/ingest/observer_key_test.go new file mode 100644 index 00000000..afd04b01 --- /dev/null +++ b/internal/ingest/observer_key_test.go @@ -0,0 +1,80 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" +) + +func TestBroadcastPacketObservation_ObserverKeyOptIn(t *testing.T) { + const key = "ab01020304050607080910111213141516171819202122232425262728293031" + w, _ := newTestWorker() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "all", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + waitForSummarySubscriber(t, ctx, w.hub, client) + + broadcast := func() hub.Event { + t.Helper() + w.broadcastPacketObservation("YVR", 4, packetObservationEvent{}, []api.ResolvedHop{{}}, key) + for { + select { + case evt := <-client.Send: + if evt.Type == hub.EventPacketObservation { + return evt + } + case <-ctx.Done(): + t.Fatal("packetObservation not delivered") + } + } + } + observation := func(raw json.RawMessage) map[string]json.RawMessage { + t.Helper() + var evt struct { + Observation map[string]json.RawMessage `json:"observation"` + } + if err := json.Unmarshal(raw, &evt); err != nil { + t.Fatal(err) + } + return evt.Observation + } + + evt := broadcast() + if evt.PayloadWithKey != nil || evt.PayloadResolvedWithKey != nil { + t.Fatal("key variants built with no opted-in client") + } + if strings.Contains(string(evt.Payload)+string(evt.PayloadResolved), "observerPublicKey") { + t.Fatal("default payloads must not carry observerPublicKey") + } + + // client stays plain so its Payload is still the base variant. + keyed := w.hub.NewClient() + defer w.hub.Remove(keyed) + w.hub.Configure(keyed, hub.ClientOptions{IncludeObserverKey: true}) + for !w.hub.ObserverKeyWanted() { + time.Sleep(time.Millisecond) + } + evt = broadcast() + if _, ok := observation(evt.Payload)["observerPublicKey"]; ok { + t.Fatal("base payload must not carry observerPublicKey") + } + withKey := observation(evt.PayloadWithKey) + if string(withKey["observerPublicKey"]) != `"`+key+`"` || string(withKey["resolvedPath"]) != "null" { + t.Fatalf("key variant = %v", withKey) + } + both := observation(evt.PayloadResolvedWithKey) + if string(both["observerPublicKey"]) != `"`+key+`"` || string(both["resolvedPath"]) == "null" { + t.Fatalf("resolved key variant = %v", both) + } +} diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 657da999..e34b55d1 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -82,15 +82,17 @@ type packetObservationEvent struct { Summary *string `json:"summary,omitempty"` // same advert name as REST list/backfill rows } `json:"packet"` Observation struct { - ObserverID string `json:"observerId"` - ObserverName string `json:"observerName"` - IATA string `json:"iata"` - HeardAt int64 `json:"heardAt"` - RSSI int16 `json:"rssi"` - SNR float32 `json:"snr"` - SourceBroker string `json:"sourceBroker"` - PathBytes string `json:"pathBytes"` - PathLength struct { + ObserverID string `json:"observerId"` + ObserverName string `json:"observerName"` + // Only set for clients that sent includeObserverKey. + ObserverPublicKey string `json:"observerPublicKey,omitempty"` + IATA string `json:"iata"` + HeardAt int64 `json:"heardAt"` + RSSI int16 `json:"rssi"` + SNR float32 `json:"snr"` + SourceBroker string `json:"sourceBroker"` + PathBytes string `json:"pathBytes"` + PathLength struct { Raw string `json:"raw"` HashSize uint8 `json:"hashSize"` HopCount uint8 `json:"hopCount"` @@ -912,7 +914,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ resolvedPath := api.BuildResolvedPath(hashes, resolved) evt.Observation.ResolvedSource = resolvedSource evt.Observation.ResolvedDestination = resolvedDestination - w.broadcastPacketObservation(iata, packet.PayloadType(), evt, resolvedPath) + w.broadcastPacketObservation(iata, packet.PayloadType(), evt, resolvedPath, hex.EncodeToString(pubkeyBytes)) } } diff --git a/internal/ws/connect_limit_test.go b/internal/ws/connect_limit_test.go index 2e55a8b8..21d878be 100644 --- a/internal/ws/connect_limit_test.go +++ b/internal/ws/connect_limit_test.go @@ -20,7 +20,7 @@ import ( func connectTestServer(t *testing.T, maxConns, maxConnects int) (*httptest.Server, <-chan struct{}) { t.Helper() - handler := Handler(hub.New(), nil, maxConns, maxConnects) + handler := Handler(hub.New(), nil, maxConns, maxConnects, nil) done := make(chan struct{}, 64) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { handler(w, r) @@ -57,7 +57,7 @@ func TestFailedHandshakeDoesNotUseConnectionSlot(t *testing.T) { func TestConnectAttemptBudgetAndRecovery(t *testing.T) { synctest.Test(t, func(t *testing.T) { - handler := Handler(nil, nil, 1, 2) + handler := Handler(nil, nil, 1, 2, nil) attempt := func() *httptest.ResponseRecorder { request := httptest.NewRequest(http.MethodGet, "/ws", nil) request.RemoteAddr = "198.51.100.1:1234" diff --git a/internal/ws/handler.go b/internal/ws/handler.go index cb632eaa..c51a4b4a 100644 --- a/internal/ws/handler.go +++ b/internal/ws/handler.go @@ -10,14 +10,15 @@ // Client → Server: // subscribe { v, type, id, scope } → server replies subscribed { v, type, id, subscriptionId } // unsubscribe { v, type, id, subscriptionId } -// configure { v, type, id, resolvePath } → server replies configured { v, type, id, resolvePath } +// configure { v, type, id, resolvePath, includeObserverKey } +// → server replies configured { v, type, id, resolvePath, includeObserverKey } // ping { v, type, id } → server replies pong { v, type, id } // -// configure's resolvePath (bool) is a connection-wide setting, not -// per-subscription: enables/disables per-hop resolvedPath data on -// packetObservation events. Freely toggleable at any point during the -// connection; each configure call sets it to exactly the value sent -// (not additive across calls, unlike subscribe scopes). Default false. +// configure's flags are connection-wide settings, not per-subscription. +// resolvePath adds per-hop resolvedPath data to packetObservation events, +// includeObserverKey adds observation.observerPublicKey. Each configure +// sets both to exactly the values sent, so an omitted flag turns off. +// Both default false. // // Server → Client events (unsolicited): // packetObservation, observerStatus, nodeUpdate, channelMessage @@ -53,7 +54,11 @@ const ( // Handler returns an http.HandlerFunc that requires the hub to be injected. // Wire it via router.New(h) so the hub is available at startup. -func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP, maxConnectsPerMinute int) http.HandlerFunc { +func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP, maxConnectsPerMinute int, allowedOrigins []string) http.HandlerFunc { + var acceptOpts *websocket.AcceptOptions + if len(allowedOrigins) > 0 { + acceptOpts = &websocket.AcceptOptions{OriginPatterns: allowedOrigins} + } limiter := newIPLimiter(maxConnsPerIP) attempts := httprate.NewRateLimiter(maxConnectsPerMinute, time.Minute, httprate.WithResponseHeaders(httprate.ResponseHeaders{RetryAfter: "Retry-After"})) @@ -67,7 +72,7 @@ func Handler(h *hub.Hub, reader api.Reader, maxConnsPerIP, maxConnectsPerMinute if attempts.RespondOnLimit(w, r, httprate.CanonicalizeIP(ip)) { return } - conn, err := websocket.Accept(w, r, nil) + conn, err := websocket.Accept(w, r, acceptOpts) if err != nil { slog.Warn("ws: failed to accept connection", "component", "ws", "error", err) return @@ -172,10 +177,9 @@ type clientMessage struct { SubscriptionID string `json:"subscriptionId,omitempty"` Scope *subscribeScope `json:"scope,omitempty"` - // ResolvePath is only read for "configure" messages: enables/disables - // the resolvedPath variant of packetObservation events for the whole - // connection. See package doc. - ResolvePath bool `json:"resolvePath,omitempty"` + // Only read for "configure" messages. See package doc. + ResolvePath bool `json:"resolvePath,omitempty"` + IncludeObserverKey bool `json:"includeObserverKey,omitempty"` } // subscribeScope mirrors the scope object in the subscribe message. @@ -260,12 +264,13 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea } case "configure": - h.SetResolvePath(client, msg.ResolvePath) + h.Configure(client, hub.ClientOptions{ResolvePath: msg.ResolvePath, IncludeObserverKey: msg.IncludeObserverKey}) reply, _ := json.Marshal(map[string]any{ - "v": 1, "type": "configured", "id": msg.ID, "resolvePath": msg.ResolvePath, + "v": 1, "type": "configured", "id": msg.ID, + "resolvePath": msg.ResolvePath, "includeObserverKey": msg.IncludeObserverKey, }) if slog.Default().Enabled(context.Background(), slog.LevelDebug) { - slog.Debug(fmt.Sprintf("ws[%s]: configured resolvePath=%t", connID, msg.ResolvePath), "component", "ws") + slog.Debug(fmt.Sprintf("ws[%s]: configured resolvePath=%t includeObserverKey=%t", connID, msg.ResolvePath, msg.IncludeObserverKey), "component", "ws") } if err := conn.Write(ctx, websocket.MessageText, reply); err != nil { slog.Warn(fmt.Sprintf("ws[%s]: failed to send configured reply", connID), "component", "ws", "error", err) diff --git a/internal/ws/handler_test.go b/internal/ws/handler_test.go new file mode 100644 index 00000000..8ac911f8 --- /dev/null +++ b/internal/ws/handler_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ws + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/coder/websocket" +) + +func TestAllowedOrigins(t *testing.T) { + for _, tc := range []struct { + name string + allowed []string + origin string + wantOK bool + }{ + {"no origin header", nil, "", true}, + {"foreign origin by default", nil, "https://example.com", false}, + {"listed origin", []string{"https://example.com"}, "https://example.com", true}, + {"listed origin, other case", []string{"https://example.com"}, "https://Example.com", true}, + {"scheme mismatch", []string{"https://example.com"}, "http://example.com", false}, + {"port mismatch", []string{"https://example.com"}, "https://example.com:8443", false}, + {"unlisted origin", []string{"https://example.com"}, "https://other.example", false}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(Handler(hub.New(), nil, 5, 100, tc.allowed)) + t.Cleanup(server.Close) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + header := http.Header{} + if tc.origin != "" { + header.Set("Origin", tc.origin) + } + conn, resp, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), &websocket.DialOptions{HTTPHeader: header}) + if tc.wantOK { + if err != nil { + t.Fatalf("dial: %v", err) + } + conn.CloseNow() + return + } + if err == nil { + conn.CloseNow() + t.Fatal("expected handshake to be refused") + } + if resp == nil || resp.StatusCode != http.StatusForbidden { + t.Fatalf("expected 403, got %v (%v)", resp, err) + } + }) + } +} + +func TestConfigureIncludeObserverKey(t *testing.T) { + h := hub.New() + go h.Run() + server := httptest.NewServer(Handler(h, nil, 5, 100, nil)) + t.Cleanup(server.Close) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + read := func(conn *websocket.Conn) map[string]json.RawMessage { + t.Helper() + _, data, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + var msg map[string]json.RawMessage + if err := json.Unmarshal(data, &msg); err != nil { + t.Fatal(err) + } + return msg + } + send := func(conn *websocket.Conn, msg string) { + t.Helper() + if err := conn.Write(ctx, websocket.MessageText, []byte(msg)); err != nil { + t.Fatal(err) + } + } + connect := func(configure string) *websocket.Conn { + t.Helper() + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.CloseNow() }) + read(conn) // hello + if configure != "" { + send(conn, configure) + reply := read(conn) + if string(reply["type"]) != `"configured"` || string(reply["includeObserverKey"]) != "true" || string(reply["resolvePath"]) != "false" { + t.Fatalf("unexpected configured reply: %v", reply) + } + } + send(conn, `{"v":1,"type":"subscribe","id":"s","scope":{}}`) + read(conn) // subscribed + return conn + } + + plain := connect("") + keyed := connect(`{"v":1,"type":"configure","id":"c","includeObserverKey":true}`) + time.Sleep(20 * time.Millisecond) // let the hub apply both subscriptions + h.Broadcast(hub.Event{ + Type: hub.EventPacketObservation, + Payload: json.RawMessage(`{"variant":"base"}`), + PayloadWithKey: json.RawMessage(`{"variant":"key"}`), + }) + if got := string(read(plain)["data"]); got != `{"variant":"base"}` { + t.Errorf("plain client got %s", got) + } + if got := string(read(keyed)["data"]); got != `{"variant":"key"}` { + t.Errorf("opted-in client got %s", got) + } +} From 90f9b5061464890d92277aea018d09340b0f903b Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:03:49 -0400 Subject: [PATCH 054/141] feat(ws): allow subdomain wildcards in allowed origins (#177) --- config.yaml.example | 5 ++++- internal/config/config.go | 18 ++++++++++++++---- internal/config/ws_connect_test.go | 13 ++++++++++++- internal/ws/handler_test.go | 12 ++++++++++++ 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index e50906ac..a0a86cc9 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -149,9 +149,12 @@ websocket: # Within this rate budget, a full concurrent cap accepts then closes with 1013 # before hello, allowing browsers to back off without evicting existing clients. max_connects_per_minute: 10 - # Other sites allowed to open /ws. Exact scheme://host[:port], no wildcards. + # Other sites allowed to open /ws. Exact scheme://host[:port], or + # scheme://*.domain[:port] for every subdomain of domain at any depth. The + # wildcard never matches the apex itself, and scheme and port stay exact. #allowed_origins: # - https://example.com + # - https://*.example.com # Node staleness, deletion, and clock-drift thresholds. #nodes: diff --git a/internal/config/config.go b/internal/config/config.go index 8473c771..b3c6cd3f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "path/filepath" + "slices" "strings" "time" @@ -243,7 +244,8 @@ type WebSocketConfig struct { // MaxConnectsPerMinute limits upgrade attempts, including failed handshakes. // Zero/omitted defaults to 10; IPv6 addresses share a /64 attempt budget. MaxConnectsPerMinute int `yaml:"max_connects_per_minute"` - // AllowedOrigins are extra exact origins allowed to open /ws. Defaults to same-host only. + // AllowedOrigins are extra origins allowed to open /ws: exact, or scheme://*.domain[:port] + // for every subdomain of domain (never the apex). Defaults to same-host only. AllowedOrigins []string `yaml:"allowed_origins"` } @@ -415,16 +417,24 @@ func Load(path string) (*Config, error) { return cfg, nil } -// validateOrigin rejects wildcards, which the WebSocket library would treat as patterns. +// validateOrigin allows exact origins or a leading "*." subdomain wildcard; the WebSocket +// library treats entries as path.Match patterns, so every other pattern character is rejected. func validateOrigin(origin string) error { u, err := url.Parse(origin) if err != nil { return fmt.Errorf("invalid origin %q: %w", origin, err) } + host := u.Hostname() + // At least two labels after the wildcard, so it never spans a whole TLD. + if rest, ok := strings.CutPrefix(host, "*."); ok { + if labels := strings.Split(rest, "."); len(labels) >= 2 && !slices.Contains(labels, "") { + host = rest + } + } if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" || u.Opaque != "" || - strings.ContainsAny(origin, `*?[]\`) { - return fmt.Errorf("origin %q must be exactly scheme://host[:port] with an http or https scheme", origin) + strings.Contains(host, "*") || strings.ContainsAny(origin, `?[]\`) { + return fmt.Errorf("origin %q must be scheme://host[:port] or scheme://*.domain[:port] with an http or https scheme", origin) } return nil } diff --git a/internal/config/ws_connect_test.go b/internal/config/ws_connect_test.go index b285b82b..38463058 100644 --- a/internal/config/ws_connect_test.go +++ b/internal/config/ws_connect_test.go @@ -44,7 +44,18 @@ func TestWebSocketAllowedOrigins(t *testing.T) { {"https", "https://example.com", false}, {"http with port", "http://localhost:5173", false}, {"wildcard", "*", true}, - {"wildcard host", "https://*.example.com", true}, + {"wildcard subdomain", "https://*.example.com", false}, + {"wildcard subdomain with port", "http://*.example.com:8443", false}, + {"wildcard scheme only", "https://*", true}, + {"wildcard over tld", "https://*.com", true}, + {"wildcard label prefix", "https://a*.example.com", true}, + {"wildcard label suffix", "https://*a.example.com", true}, + {"wildcard inner label", "https://x.*.example.com", true}, + {"double wildcard", "https://*.*.example.com", true}, + {"wildcard empty label", "https://*..com", true}, + {"question mark", "https://ex?mple.com", true}, + {"character class", "https://[ab].example.com", true}, + {"backslash", `https://*.example.com\`, true}, {"no scheme", "example.com", true}, {"other scheme", "ftp://example.com", true}, {"path", "https://example.com/", true}, diff --git a/internal/ws/handler_test.go b/internal/ws/handler_test.go index 8ac911f8..176a3e6c 100644 --- a/internal/ws/handler_test.go +++ b/internal/ws/handler_test.go @@ -17,6 +17,8 @@ import ( ) func TestAllowedOrigins(t *testing.T) { + wildcard := []string{"https://*.example.com"} + apexAndWildcard := []string{"https://example.com", "https://*.example.com"} for _, tc := range []struct { name string allowed []string @@ -30,6 +32,16 @@ func TestAllowedOrigins(t *testing.T) { {"scheme mismatch", []string{"https://example.com"}, "http://example.com", false}, {"port mismatch", []string{"https://example.com"}, "https://example.com:8443", false}, {"unlisted origin", []string{"https://example.com"}, "https://other.example", false}, + {"wildcard subdomain", wildcard, "https://sub.example.com", true}, + {"wildcard deeper subdomain", wildcard, "https://a.b.example.com", true}, + {"wildcard other case", wildcard, "https://SUB.Example.com", true}, + {"wildcard apex", wildcard, "https://example.com", false}, + {"wildcard lookalike", wildcard, "https://evilexample.com", false}, + {"wildcard suffix host", wildcard, "https://example.com.evil.net", false}, + {"wildcard scheme mismatch", wildcard, "http://sub.example.com", false}, + {"wildcard port mismatch", wildcard, "https://sub.example.com:8443", false}, + {"apex and wildcard, apex", apexAndWildcard, "https://example.com", true}, + {"apex and wildcard, subdomain", apexAndWildcard, "https://sub.example.com", true}, } { t.Run(tc.name, func(t *testing.T) { server := httptest.NewServer(Handler(hub.New(), nil, 5, 100, tc.allowed)) From 876d99704fa9ae76fc8b57abb54a018b0499b880 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:38:49 -0400 Subject: [PATCH 055/141] feat(ws): opt-in includeRepeats streams later hearings over new paths (#178) Duplicate observations are dropped before the hub today, so a live map only sees each packet's first hearing per observer. Clients that configure includeRepeats also get later hearings whose path is new; nothing is stored, repeats skip the count query, and they are dropped first when the broadcast channel is half full. --- internal/hub/hub.go | 62 ++++++++++++- internal/hub/hub_test.go | 109 ++++++++++++++++++++++ internal/hub/sent.go | 59 ++++++++++++ internal/ingest/ingest.go | 7 +- internal/ingest/observer_key_test.go | 2 +- internal/ingest/packet.go | 30 ++++-- internal/ingest/repeats_test.go | 132 +++++++++++++++++++++++++++ internal/ws/handler.go | 19 ++-- internal/ws/handler_test.go | 65 +++++++++++++ 9 files changed, 465 insertions(+), 20 deletions(-) create mode 100644 internal/hub/sent.go create mode 100644 internal/ingest/repeats_test.go diff --git a/internal/hub/hub.go b/internal/hub/hub.go index d4086a1d..09c22077 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -21,6 +21,7 @@ import ( "log/slog" "slices" "sync/atomic" + "time" ) // EventType identifies the kind of server-push event. These match the @@ -54,6 +55,8 @@ type Event struct { IATA string PayloadType uint8 ChannelHash string // hex string, non-empty only for channelMessage events + // Repeat marks a later hearing of a stored observation; only IncludeRepeats clients get it. + Repeat bool } // Scope mirrors the client-side subscribe message. All fields are optional: @@ -84,12 +87,15 @@ type Client struct { ResolvePath bool // IncludeObserverKey works like ResolvePath, for observerPublicKey. IncludeObserverKey bool + // IncludeRepeats works like ResolvePath, for Repeat events. + IncludeRepeats bool } // ClientOptions are the connection-wide settings a "configure" message sets. type ClientOptions struct { ResolvePath bool IncludeObserverKey bool + IncludeRepeats bool } // payloadFor falls back to a narrower variant if the event lacks one. @@ -153,6 +159,12 @@ type Hub struct { // observerKeyClients counts registered clients with IncludeObserverKey set. observerKeyClients atomic.Int64 + // repeatClients counts registered clients with IncludeRepeats set. + repeatClients atomic.Int64 + // repeatDrops counts repeats refused because the broadcast channel was busy. + repeatDrops atomic.Int64 + repeatDropLogAt atomic.Int64 // unix nanos of the last drop log line + sent *sentPaths } // subscribeMsg carries a client registration, a scope subscription, or a @@ -185,6 +197,7 @@ func New() *Hub { unsubscribe: make(chan unsubscribeMsg, 64), remove: make(chan *Client, 64), broadcast: make(chan Event, 512), + sent: newSentPaths(sentPathsTTL, sentPathsMax), } } @@ -226,6 +239,17 @@ func (h *Hub) ObserverKeyWanted() bool { return h.observerKeyClients.Load() > 0 } +// RepeatsWanted lets ingest skip repeat work when nobody wants it. +func (h *Hub) RepeatsWanted() bool { + return h.repeatClients.Load() > 0 +} + +// MarkSent records a hearing's path and reports whether it was not already sent recently, +// so broker copies and same-path duplicates go out once. +func (h *Hub) MarkSent(packetHash, observerID, path []byte) bool { + return h.sent.mark(packetHash, observerID, path, time.Now()) +} + // Remove deregisters a client and closes its Send channel. // Safe to call from any goroutine (e.g. the WS handler's defer). func (h *Hub) Remove(c *Client) { @@ -241,6 +265,31 @@ func (h *Hub) Broadcast(e Event) { } } +// BroadcastRepeat enqueues a repeat event, dropping it once the broadcast channel is half +// full so repeats never crowd out first hearings. +func (h *Hub) BroadcastRepeat(e Event) { + e.Repeat = true + if len(h.broadcast) >= cap(h.broadcast)/2 { + h.dropRepeat() + return + } + select { + case h.broadcast <- e: + default: + h.dropRepeat() + } +} + +// dropRepeat counts a dropped repeat and logs the running total at most once a minute. +func (h *Hub) dropRepeat() { + total := h.repeatDrops.Add(1) + now := time.Now().UnixNano() + last := h.repeatDropLogAt.Load() + if now-last >= int64(time.Minute) && h.repeatDropLogAt.CompareAndSwap(last, now) { + slog.Warn("hub: broadcast channel busy, dropping repeat events", "component", "hub", "dropped_total", total) + } +} + // Run is the hub's single-goroutine event loop. Call it in a dedicated // goroutine: go hub.Run(). // @@ -270,8 +319,16 @@ func (h *Hub) Run() { h.observerKeyClients.Add(-1) } } + if msg.client.IncludeRepeats != msg.options.IncludeRepeats { + if msg.options.IncludeRepeats { + h.repeatClients.Add(1) + } else { + h.repeatClients.Add(-1) + } + } msg.client.ResolvePath = msg.options.ResolvePath msg.client.IncludeObserverKey = msg.options.IncludeObserverKey + msg.client.IncludeRepeats = msg.options.IncludeRepeats } default: // AddScope path — client must already be registered. @@ -291,13 +348,16 @@ func (h *Hub) Run() { if c.IncludeObserverKey { h.observerKeyClients.Add(-1) } + if c.IncludeRepeats { + h.repeatClients.Add(-1) + } close(c.Send) close(c.laggedCH) } case evt := <-h.broadcast: for c := range clients { - if !c.matches(evt) { + if (evt.Repeat && !c.IncludeRepeats) || !c.matches(evt) { continue } outEvt := evt diff --git a/internal/hub/hub_test.go b/internal/hub/hub_test.go index 3a992238..94be9687 100644 --- a/internal/hub/hub_test.go +++ b/internal/hub/hub_test.go @@ -434,3 +434,112 @@ func TestHub_ObserverKeyWanted_TracksOptIns(t *testing.T) { h.Remove(b) waitFor(false) } + +func TestHub_Repeat_OnlyReachesOptedInClients(t *testing.T) { + h := runHub(t) + plain, opted := h.NewClient(), h.NewClient() + h.AddScope(plain, "all", Scope{}) + h.AddScope(opted, "all", Scope{}) + h.Configure(opted, ClientOptions{IncludeRepeats: true}) + time.Sleep(10 * time.Millisecond) + + read := func(c *Client) (Event, bool) { + select { + case evt := <-c.Send: + return evt, true + case <-time.After(50 * time.Millisecond): + return Event{}, false + } + } + h.BroadcastRepeat(Event{Type: EventPacketObservation}) + if evt, ok := read(opted); !ok || !evt.Repeat { + t.Fatalf("opted-in client: got %+v, %t", evt, ok) + } + if evt, ok := read(plain); ok { + t.Fatalf("plain client got repeat %+v", evt) + } + h.Broadcast(Event{Type: EventPacketObservation}) + for _, c := range []*Client{plain, opted} { + if evt, ok := read(c); !ok || evt.Repeat { + t.Fatalf("normal event: got %+v, %t", evt, ok) + } + } +} + +func TestHub_RepeatsWanted_TracksOptIns(t *testing.T) { + h := runHub(t) + waitFor := func(want bool) { + t.Helper() + deadline := time.Now().Add(time.Second) + for h.RepeatsWanted() != want { + if time.Now().After(deadline) { + t.Fatalf("RepeatsWanted = %t, want %t", !want, want) + } + time.Sleep(time.Millisecond) + } + } + a, b := h.NewClient(), h.NewClient() + if h.RepeatsWanted() { + t.Fatal("no client opted in yet") + } + h.Configure(a, ClientOptions{IncludeRepeats: true}) + h.Configure(a, ClientOptions{IncludeRepeats: true}) // repeat must not double count + h.Configure(b, ClientOptions{IncludeRepeats: true}) + waitFor(true) + h.Configure(a, ClientOptions{ResolvePath: true}) + h.Remove(b) + waitFor(false) +} + +func TestHub_BroadcastRepeat_DropsFirstWhenBusy(t *testing.T) { + h := New() // not running, so the broadcast channel only fills + h.BroadcastRepeat(Event{Type: EventPacketObservation}) + if len(h.broadcast) != 1 || h.repeatDrops.Load() != 0 { + t.Fatalf("repeat on an idle hub: queued %d, dropped %d", len(h.broadcast), h.repeatDrops.Load()) + } + for len(h.broadcast) < cap(h.broadcast)/2 { + h.Broadcast(Event{Type: EventPacketObservation}) + } + queued := len(h.broadcast) + h.BroadcastRepeat(Event{Type: EventPacketObservation}) + h.BroadcastRepeat(Event{Type: EventPacketObservation}) + if len(h.broadcast) != queued || h.repeatDrops.Load() != 2 { + t.Fatalf("repeats past half full: queued %d (want %d), dropped %d", len(h.broadcast), queued, h.repeatDrops.Load()) + } + h.Broadcast(Event{Type: EventPacketObservation}) + if len(h.broadcast) != queued+1 { + t.Fatal("normal event not enqueued past half full") + } +} + +func TestSentPaths(t *testing.T) { + start := time.Now() + s := newSentPaths(time.Minute, 3) + hash, observer := []byte{1, 2}, []byte{3} + if !s.mark(hash, observer, []byte{0xaa}, start) { + t.Fatal("first hearing refused") + } + if s.mark(hash, observer, []byte{0xaa}, start.Add(time.Second)) { + t.Fatal("exact copy within the window allowed") + } + if !s.mark(hash, observer, []byte{0xaa, 0xbb}, start.Add(time.Second)) { + t.Fatal("new path refused") + } + if !s.mark(hash, []byte{4}, []byte{0xaa}, start.Add(time.Second)) { + t.Fatal("other observer refused") + } + if !s.mark(hash, observer, []byte{0xaa}, start.Add(time.Minute)) { + t.Fatal("copy after the window refused") + } + + s = newSentPaths(time.Hour, 3) + for i := range 4 { + s.mark(hash, observer, []byte{byte(i)}, start) + } + if len(s.at) != 3 || len(s.order) != 3 { + t.Fatalf("size bound: %d keys, %d queued", len(s.at), len(s.order)) + } + if !s.mark(hash, observer, []byte{0}, start) { + t.Fatal("oldest key not evicted at the size bound") + } +} diff --git a/internal/hub/sent.go b/internal/hub/sent.go new file mode 100644 index 00000000..f1e04112 --- /dev/null +++ b/internal/hub/sent.go @@ -0,0 +1,59 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package hub + +import ( + "hash/maphash" + "sync" + "time" +) + +const ( + sentPathsTTL = 2 * time.Minute + sentPathsMax = 200_000 +) + +// sentPaths remembers recently sent (packet, observer, path) hearings. It is shared by all +// broker workers; keys are hashed so memory stays small at the size bound. +type sentPaths struct { + mu sync.Mutex + seed maphash.Seed + ttl time.Duration + max int + at map[uint64]time.Time + order []sentPath // oldest first +} + +type sentPath struct { + key uint64 + at time.Time +} + +func newSentPaths(ttl time.Duration, max int) *sentPaths { + return &sentPaths{seed: maphash.MakeSeed(), ttl: ttl, max: max, at: make(map[uint64]time.Time)} +} + +// mark records the hearing and reports whether it was not already recorded within the TTL. +func (s *sentPaths) mark(packetHash, observerID, path []byte, now time.Time) bool { + var h maphash.Hash + h.SetSeed(s.seed) + for _, part := range [][]byte{packetHash, observerID, path} { + h.WriteByte(byte(len(part))) + h.Write(part) + } + key := h.Sum64() + + s.mu.Lock() + defer s.mu.Unlock() + for len(s.order) > 0 && (now.Sub(s.order[0].at) >= s.ttl || len(s.order) >= s.max) { + delete(s.at, s.order[0].key) + s.order = s.order[1:] + } + if _, ok := s.at[key]; ok { + return false + } + s.at[key] = now + s.order = append(s.order, sentPath{key, now}) + return true +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 4961344a..f19e403e 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -385,7 +385,8 @@ func (w *Worker) broadcast(eventType hub.EventType, iata string, payloadType uin // passed in rather than computed here because the caller already has the // path-hash resolution results in hand from other per-packet work (known // route detection, capability detection) — this adds no extra DB calls. -func (w *Worker) broadcastPacketObservation(iata string, payloadType uint8, evt packetObservationEvent, resolvedPath []api.ResolvedHop, observerKey string) { +// Repeats go through the hub's drop-first path. +func (w *Worker) broadcastPacketObservation(iata string, payloadType uint8, evt packetObservationEvent, resolvedPath []api.ResolvedHop, observerKey string, repeat bool) { base, err := json.Marshal(evt) if err != nil { w.log.Error("failed to marshal packetObservation event", "error", err) @@ -414,6 +415,10 @@ func (w *Worker) broadcastPacketObservation(iata string, payloadType uint8, evt out.PayloadWithKey = nil } } + if repeat { + w.hub.BroadcastRepeat(out) + return + } w.hub.Broadcast(out) } diff --git a/internal/ingest/observer_key_test.go b/internal/ingest/observer_key_test.go index afd04b01..c2021212 100644 --- a/internal/ingest/observer_key_test.go +++ b/internal/ingest/observer_key_test.go @@ -27,7 +27,7 @@ func TestBroadcastPacketObservation_ObserverKeyOptIn(t *testing.T) { broadcast := func() hub.Event { t.Helper() - w.broadcastPacketObservation("YVR", 4, packetObservationEvent{}, []api.ResolvedHop{{}}, key) + w.broadcastPacketObservation("YVR", 4, packetObservationEvent{}, []api.ResolvedHop{{}}, key, false) for { select { case evt := <-client.Send: diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index e34b55d1..ad1bfcbd 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -80,6 +80,8 @@ type packetObservationEvent struct { ObservationCount int64 `json:"observationCount"` Scope *string `json:"scope,omitempty"` Summary *string `json:"summary,omitempty"` // same advert name as REST list/backfill rows + // Only set on later hearings sent to includeRepeats clients; those carry observationCount 0. + IsRepeat bool `json:"isRepeat,omitempty"` } `json:"packet"` Observation struct { ObserverID string `json:"observerId"` @@ -880,15 +882,23 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } w.runCapabilityDetection(ctx, packet.PayloadType(), packet.PathHashSize(), resolvedIDs) - if inserted { - w.handlePayloadTypeSideEffects(ctx, packet, iata, packetHash[:], radio, scopeID, matchedScope, pubkeyBytes, float32(parseNumber(envelope.SNR))) + // A duplicate is streamed, never stored, to includeRepeats clients when its path is new. + repeat := !inserted && w.hub.RepeatsWanted() && w.hub.MarkSent(packetHash[:], id[:], packet.Path) + if inserted || repeat { + if inserted { + w.handlePayloadTypeSideEffects(ctx, packet, iata, packetHash[:], radio, scopeID, matchedScope, pubkeyBytes, float32(parseNumber(envelope.SNR))) + if w.hub.RepeatsWanted() { + w.hub.MarkSent(packetHash[:], id[:], packet.Path) // so broker copies of it aren't repeats + } + } evt := packetObservationEvent{} evt.PacketHash = hex.EncodeToString(packetHash[:]) evt.Packet.PayloadType = packet.PayloadType() evt.Packet.PayloadTypeName = packet.PayloadTypeString() evt.Packet.RouteType = packet.RouteType() evt.Packet.RouteTypeName = api.RouteTypeName(int16(packet.RouteType())) - evt.Packet.IsFirstObservation = isNew + evt.Packet.IsFirstObservation = isNew && !repeat + evt.Packet.IsRepeat = repeat evt.Packet.Summary = summary evt.Observation.ObserverID = id.String() evt.Observation.ObserverName = observerName @@ -902,19 +912,21 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ evt.Observation.PathLength.HashSize = packet.PathHashSize() evt.Observation.PathLength.HopCount = packet.PathHashCount() evt.Observation.PropagationTimeMs = 0 // not yet calculated - count, err := w.db.GetPacketObservationCount(ctx, packetHash[:]) - if err != nil { - w.log.Error("failed to get observation count", "error", err) - count = 0 + if !repeat { + count, err := w.db.GetPacketObservationCount(ctx, packetHash[:]) + if err != nil { + w.log.Error("failed to get observation count", "error", err) + count = 0 + } + evt.Packet.ObservationCount = count } - evt.Packet.ObservationCount = count if matchedScope != nil { evt.Packet.Scope = matchedScope } resolvedPath := api.BuildResolvedPath(hashes, resolved) evt.Observation.ResolvedSource = resolvedSource evt.Observation.ResolvedDestination = resolvedDestination - w.broadcastPacketObservation(iata, packet.PayloadType(), evt, resolvedPath, hex.EncodeToString(pubkeyBytes)) + w.broadcastPacketObservation(iata, packet.PayloadType(), evt, resolvedPath, hex.EncodeToString(pubkeyBytes), repeat) } } diff --git a/internal/ingest/repeats_test.go b/internal/ingest/repeats_test.go new file mode 100644 index 00000000..5766c099 --- /dev/null +++ b/internal/ingest/repeats_test.go @@ -0,0 +1,132 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" +) + +type countQueryDB struct { + *stubDB + countQueries int +} + +func (s *countQueryDB) GetPacketObservationCount(context.Context, []byte) (int64, error) { + s.countQueries++ + return 1, nil +} + +type repeatHarness struct { + t *testing.T + ctx context.Context + w *Worker + db *countQueryDB + client *hub.Client +} + +func newRepeatHarness(t *testing.T, includeRepeats bool) *repeatHarness { + w, base := newTestWorker() + db := &countQueryDB{stubDB: base} + w.db = db + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) + go w.hub.Run() + client := w.hub.NewClient() + t.Cleanup(func() { w.hub.Remove(client) }) + w.hub.AddScope(client, "all", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + if includeRepeats { + w.hub.Configure(client, hub.ClientOptions{IncludeRepeats: true}) + for !w.hub.RepeatsWanted() { + time.Sleep(time.Millisecond) + } + } + waitForSummarySubscriber(t, ctx, w.hub, client) + return &repeatHarness{t: t, ctx: ctx, w: w, db: db, client: client} +} + +// hear feeds one hearing of the same group text over path and returns the packetObservation +// payloads it produced. +func (r *repeatHarness) hear(inserted bool, path ...byte) []json.RawMessage { + r.t.Helper() + packet := buildGrpTxtPacket(r.t, 0x1a, make([]byte, 16)) + packet.Path = path + packet.PathLength = byte(len(path)) + r.db.observationInserted = inserted + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(r.t, packet)) + + // Hub delivery is FIFO, so everything handlePacket sent arrives before this marker. + r.w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + var got []json.RawMessage + for { + select { + case evt := <-r.client.Send: + if evt.Type == hub.EventObserverStatus { + return got + } + got = append(got, evt.Payload) + case <-r.ctx.Done(): + r.t.Fatal("marker event not delivered") + } + } +} + +func packetFields(t *testing.T, raw json.RawMessage) map[string]json.RawMessage { + t.Helper() + var evt struct { + Packet map[string]json.RawMessage `json:"packet"` + } + if err := json.Unmarshal(raw, &evt); err != nil { + t.Fatal(err) + } + return evt.Packet +} + +func TestHandlePacket_Repeats(t *testing.T) { + r := newRepeatHarness(t, true) + + first := r.hear(true, 0x11) + if len(first) != 1 || strings.Contains(string(first[0]), "isRepeat") { + t.Fatalf("first hearing: %s", first) + } + if r.db.countQueries != 1 { + t.Fatalf("first hearing ran %d count queries", r.db.countQueries) + } + if got := r.hear(false, 0x11); len(got) != 0 { + t.Fatalf("broker copy of the first hearing sent: %s", got) + } + + repeat := r.hear(false, 0x22, 0x11) + if len(repeat) != 1 { + t.Fatalf("new path sent %d events", len(repeat)) + } + fields := packetFields(t, repeat[0]) + if string(fields["isRepeat"]) != "true" || string(fields["isFirstObservation"]) != "false" || string(fields["observationCount"]) != "0" { + t.Fatalf("repeat packet fields: %v", fields) + } + if r.db.countQueries != 1 { + t.Fatalf("repeat ran a count query (%d total)", r.db.countQueries) + } + if got := r.hear(false, 0x22, 0x11); len(got) != 0 { + t.Fatalf("same path sent twice: %s", got) + } +} + +func TestHandlePacket_RepeatsOffByDefault(t *testing.T) { + r := newRepeatHarness(t, false) + if got := r.hear(true, 0x11); len(got) != 1 || strings.Contains(string(got[0]), "isRepeat") { + t.Fatalf("first hearing: %s", got) + } + if got := r.hear(false, 0x22, 0x11); len(got) != 0 { + t.Fatalf("repeat sent with nobody opted in: %s", got) + } + if r.db.countQueries != 1 { + t.Fatalf("duplicate ran a count query (%d total)", r.db.countQueries) + } +} diff --git a/internal/ws/handler.go b/internal/ws/handler.go index c51a4b4a..dd9c5453 100644 --- a/internal/ws/handler.go +++ b/internal/ws/handler.go @@ -10,15 +10,17 @@ // Client → Server: // subscribe { v, type, id, scope } → server replies subscribed { v, type, id, subscriptionId } // unsubscribe { v, type, id, subscriptionId } -// configure { v, type, id, resolvePath, includeObserverKey } -// → server replies configured { v, type, id, resolvePath, includeObserverKey } +// configure { v, type, id, resolvePath, includeObserverKey, includeRepeats } +// → server replies configured { v, type, id, resolvePath, includeObserverKey, includeRepeats } // ping { v, type, id } → server replies pong { v, type, id } // // configure's flags are connection-wide settings, not per-subscription. // resolvePath adds per-hop resolvedPath data to packetObservation events, -// includeObserverKey adds observation.observerPublicKey. Each configure -// sets both to exactly the values sent, so an omitted flag turns off. -// Both default false. +// includeObserverKey adds observation.observerPublicKey, and includeRepeats +// also streams later hearings of an already-stored observation over a new +// path, as packetObservation events with packet.isRepeat true and +// observationCount 0. Each configure sets all three to exactly the values +// sent, so an omitted flag turns off. All default false. // // Server → Client events (unsolicited): // packetObservation, observerStatus, nodeUpdate, channelMessage @@ -180,6 +182,7 @@ type clientMessage struct { // Only read for "configure" messages. See package doc. ResolvePath bool `json:"resolvePath,omitempty"` IncludeObserverKey bool `json:"includeObserverKey,omitempty"` + IncludeRepeats bool `json:"includeRepeats,omitempty"` } // subscribeScope mirrors the scope object in the subscribe message. @@ -264,13 +267,13 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea } case "configure": - h.Configure(client, hub.ClientOptions{ResolvePath: msg.ResolvePath, IncludeObserverKey: msg.IncludeObserverKey}) + h.Configure(client, hub.ClientOptions{ResolvePath: msg.ResolvePath, IncludeObserverKey: msg.IncludeObserverKey, IncludeRepeats: msg.IncludeRepeats}) reply, _ := json.Marshal(map[string]any{ "v": 1, "type": "configured", "id": msg.ID, - "resolvePath": msg.ResolvePath, "includeObserverKey": msg.IncludeObserverKey, + "resolvePath": msg.ResolvePath, "includeObserverKey": msg.IncludeObserverKey, "includeRepeats": msg.IncludeRepeats, }) if slog.Default().Enabled(context.Background(), slog.LevelDebug) { - slog.Debug(fmt.Sprintf("ws[%s]: configured resolvePath=%t includeObserverKey=%t", connID, msg.ResolvePath, msg.IncludeObserverKey), "component", "ws") + slog.Debug(fmt.Sprintf("ws[%s]: configured resolvePath=%t includeObserverKey=%t includeRepeats=%t", connID, msg.ResolvePath, msg.IncludeObserverKey, msg.IncludeRepeats), "component", "ws") } if err := conn.Write(ctx, websocket.MessageText, reply); err != nil { slog.Warn(fmt.Sprintf("ws[%s]: failed to send configured reply", connID), "component", "ws", "error", err) diff --git a/internal/ws/handler_test.go b/internal/ws/handler_test.go index 176a3e6c..3e9c5900 100644 --- a/internal/ws/handler_test.go +++ b/internal/ws/handler_test.go @@ -132,3 +132,68 @@ func TestConfigureIncludeObserverKey(t *testing.T) { t.Errorf("opted-in client got %s", got) } } + +func TestConfigureIncludeRepeats(t *testing.T) { + h := hub.New() + go h.Run() + server := httptest.NewServer(Handler(h, nil, 5, 100, nil)) + t.Cleanup(server.Close) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(server.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.CloseNow() }) + read := func() map[string]json.RawMessage { + t.Helper() + _, data, err := conn.Read(ctx) + if err != nil { + t.Fatal(err) + } + var msg map[string]json.RawMessage + if err := json.Unmarshal(data, &msg); err != nil { + t.Fatal(err) + } + return msg + } + send := func(msg string) { + t.Helper() + if err := conn.Write(ctx, websocket.MessageText, []byte(msg)); err != nil { + t.Fatal(err) + } + } + configure := func(msg, want string) { + t.Helper() + send(msg) + if reply := read(); string(reply["type"]) != `"configured"` || string(reply["includeRepeats"]) != want { + t.Fatalf("configured reply: %v", reply) + } + time.Sleep(20 * time.Millisecond) // let the hub apply it + } + // Each repeat is followed by a normal marker, so a skipped repeat shows up as the marker. + next := func() string { + t.Helper() + h.BroadcastRepeat(hub.Event{Type: hub.EventPacketObservation, Payload: json.RawMessage(`"repeat"`)}) + h.Broadcast(hub.Event{Type: hub.EventPacketObservation, Payload: json.RawMessage(`"marker"`)}) + got := string(read()["data"]) + if got == `"repeat"` { + read() // marker + } + return got + } + + read() // hello + configure(`{"v":1,"type":"configure","id":"c","includeRepeats":true}`, "true") + send(`{"v":1,"type":"subscribe","id":"s","scope":{}}`) + read() // subscribed + time.Sleep(20 * time.Millisecond) + if got := next(); got != `"repeat"` { + t.Fatalf("opted-in client got %s", got) + } + configure(`{"v":1,"type":"configure","id":"c","resolvePath":true}`, "false") + if got := next(); got != `"marker"` { + t.Fatalf("client still received a repeat after opting out: %s", got) + } +} From 1887fccb8d7f8c70612dd2fa0c71dd00edb56e2b Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:56:03 -0400 Subject: [PATCH 056/141] perf(ingest): reduce database work and buffer MQTT bursts (#179) --- cmd/beacon/main.go | 7 +- internal/ingest/capability.go | 47 ++++-- internal/ingest/capability_test.go | 72 ++++++++ internal/ingest/endpoint_matching_test.go | 1 + internal/ingest/endpoint_work_test.go | 54 ++++++ internal/ingest/ingest.go | 65 ++++++-- internal/ingest/packet.go | 46 ++--- internal/ingest/queue.go | 108 ++++++++++++ internal/ingest/queue_test.go | 194 ++++++++++++++++++++++ 9 files changed, 544 insertions(+), 50 deletions(-) create mode 100644 internal/ingest/capability_test.go create mode 100644 internal/ingest/endpoint_work_test.go create mode 100644 internal/ingest/queue.go create mode 100644 internal/ingest/queue_test.go diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 2f170439..8e322e6e 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -12,6 +12,7 @@ import ( "os" "os/signal" "strconv" + "sync" "syscall" "time" @@ -284,8 +285,9 @@ func main() { broker2.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver) } - go broker1.Start(ctx) - go broker2.Start(ctx) + var ingestWorkers sync.WaitGroup + ingestWorkers.Go(func() { broker1.Start(ctx) }) + ingestWorkers.Go(func() { broker2.Start(ctx) }) tasks := []background.Task{ background.ViewRefreshTask(store, resolved.ViewRefreshInterval), @@ -343,6 +345,7 @@ func main() { if err := srv.Shutdown(shutdownCtx); err != nil { slog.Error("server shutdown error", "component", "startup", "error", err) } + ingestWorkers.Wait() coalescer.Flush(shutdownCtx) } diff --git a/internal/ingest/capability.go b/internal/ingest/capability.go index 240c3814..22f7ce56 100644 --- a/internal/ingest/capability.go +++ b/internal/ingest/capability.go @@ -5,28 +5,47 @@ package ingest import ( "context" + "sync" "github.com/google/uuid" ) -// runCapabilityDetection checks hash sizes and flips firmware capability flags. -// Called only when the observation INSERT succeeded (no dedup conflict). -// -// Rules (from design doc): -// - hash_size == 1: do nothing (proves nothing about firmware) -// - duplicate hash prefixes within the path: skip entirely -// - non-trace + hash_size 2 or 3 → supports_multibyte_paths = TRUE -// - trace (0x09) + hash_size 2 or 4 → supports_multibyte_traces = TRUE +const capabilityCacheLimit = 4096 + +type capabilityCache struct { + mu sync.Mutex + nodes map[uuid.UUID]uint8 +} + func (w *Worker) runCapabilityDetection(ctx context.Context, payloadType uint8, hashSize uint8, resolvedNodeIDs []uuid.UUID) { - if hashSize < 2 { + var flag uint8 + switch { + case payloadType != 0x09 && (hashSize == 2 || hashSize == 3): + flag = 1 + case payloadType == 0x09 && (hashSize == 2 || hashSize == 4): + flag = 2 + default: return } for _, nodeID := range resolvedNodeIDs { - switch { - case payloadType != 0x09 && (hashSize == 2 || hashSize == 3): - _ = w.db.SetNodeCapability(ctx, nodeID, true, false) - case payloadType == 0x09 && (hashSize == 2 || hashSize == 4): - _ = w.db.SetNodeCapability(ctx, nodeID, false, true) + w.capabilities.mu.Lock() + recorded := w.capabilities.nodes[nodeID]&flag != 0 + w.capabilities.mu.Unlock() + if recorded { + continue + } + if err := w.db.SetNodeCapability(ctx, nodeID, flag == 1, flag == 2); err != nil { + continue + } + w.capabilities.mu.Lock() + if w.capabilities.nodes == nil { + w.capabilities.nodes = make(map[uuid.UUID]uint8) + } + // Eviction only costs another write; capabilities never downgrade. + if len(w.capabilities.nodes) >= capabilityCacheLimit { + clear(w.capabilities.nodes) } + w.capabilities.nodes[nodeID] |= flag + w.capabilities.mu.Unlock() } } diff --git a/internal/ingest/capability_test.go b/internal/ingest/capability_test.go new file mode 100644 index 00000000..2ebfb42d --- /dev/null +++ b/internal/ingest/capability_test.go @@ -0,0 +1,72 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "errors" + "testing" + + "github.com/google/uuid" +) + +func TestCapabilityAlreadyRecorded(t *testing.T) { + w, db := newTestWorker() + id := uuid.New() + for range 3 { + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{id, id}) + } + if got := len(db.setCapabilityCalls); got != 1 { + t.Fatalf("recorded the same capability %d times, want 1", got) + } + w.runCapabilityDetection(context.Background(), 9, 2, []uuid.UUID{id}) + w.runCapabilityDetection(context.Background(), 9, 2, []uuid.UUID{id}) + if got := len(db.setCapabilityCalls); got != 2 || !db.setCapabilityCalls[1].traces { + t.Fatalf("trace capability was not recorded separately: %+v", db.setCapabilityCalls) + } +} + +type failingCapabilityDB struct { + *stubDB + fail bool +} + +func (d *failingCapabilityDB) SetNodeCapability(ctx context.Context, id uuid.UUID, paths, traces bool) error { + _ = d.stubDB.SetNodeCapability(ctx, id, paths, traces) + if d.fail { + return errors.New("write failed") + } + return nil +} + +func TestCapabilityRetriesFailedWrite(t *testing.T) { + w, base := newTestWorker() + db := &failingCapabilityDB{stubDB: base, fail: true} + w.db = db + id := uuid.New() + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{id}) + db.fail = false + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{id}) + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{id}) + if got := len(db.setCapabilityCalls); got != 2 { + t.Fatalf("got %d writes, want the failed write and one successful retry", got) + } +} + +func TestCapabilityCacheEviction(t *testing.T) { + w, db := newTestWorker() + first := uuid.New() + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{first}) + for range capabilityCacheLimit { + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{uuid.New()}) + } + if len(w.capabilities.nodes) > capabilityCacheLimit { + t.Fatal("cache grew beyond limit") + } + before := len(db.setCapabilityCalls) + w.runCapabilityDetection(context.Background(), 5, 2, []uuid.UUID{first}) + if len(db.setCapabilityCalls) != before+1 { + t.Fatal("evicted capability was not recorded again") + } +} diff --git a/internal/ingest/endpoint_matching_test.go b/internal/ingest/endpoint_matching_test.go index 1b600628..52d12195 100644 --- a/internal/ingest/endpoint_matching_test.go +++ b/internal/ingest/endpoint_matching_test.go @@ -35,6 +35,7 @@ func (s *endpointRoutingDB) ResolvePathHashes(_ context.Context, iata string, ha func TestHandlePacketSeparatesEndpointAndRelayMatching(t *testing.T) { for _, kind := range []uint8{meshcore.PayloadTypeReq, meshcore.PayloadTypeResponse, meshcore.PayloadTypeTxtMsg, meshcore.PayloadTypePath} { w, base := newTestWorker() + base.observationInserted = true db := &endpointRoutingDB{stubDB: base} w.db = db packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, kind, 0), diff --git a/internal/ingest/endpoint_work_test.go b/internal/ingest/endpoint_work_test.go new file mode 100644 index 00000000..50302913 --- /dev/null +++ b/internal/ingest/endpoint_work_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/meshcore-go/meshcore-go" +) + +type endpointLookupDB struct { + *stubDB + calls [][][]byte +} + +func (d *endpointLookupDB) ResolveEndpointHashes(_ context.Context, _ string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { + d.calls = append(d.calls, hashes) + return nil, nil +} +func endpointTestPacket() *meshcore.Packet { + return &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeFlood, meshcore.PayloadTypeTxtMsg, 0), Payload: append([]byte{0xbb, 0xaa, 0, 0}, make([]byte, 16)...)} +} +func TestEndpointLookupsOnlyForLiveHearings(t *testing.T) { + r := newRepeatHarness(t, true) + d := &endpointLookupDB{stubDB: r.db.stubDB} + r.w.db = d + packet := endpointTestPacket() + hear := func(inserted bool, path byte) { + d.observationInserted = inserted + packet.Path = []byte{path} + packet.PathLength = 1 + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(t, packet)) + } + hear(true, 0x11) + first := len(d.calls) + if first == 0 { + t.Fatal("first hearing missing endpoint lookup") + } + hear(false, 0x11) + if len(d.calls) != first { + t.Fatal("suppressed copy resolved endpoints") + } + hear(false, 0x22) + if len(d.calls) != 2*first { + t.Fatal("new path missing endpoint lookup") + } + hear(false, 0x22) + if len(d.calls) != 2*first { + t.Fatal("suppressed repeat resolved endpoints") + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index f19e403e..998c2051 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -227,6 +227,7 @@ type Worker struct { keys ChannelKeyStore scopes ScopeStore client mqtt.Client + capabilities capabilityCache onNodeUpsert func(ctx context.Context, nodeID uuid.UUID) onObserverUpsert func(ctx context.Context, observerID uuid.UUID) } @@ -243,6 +244,28 @@ func New(cfg Config, db DB, h *hub.Hub, keys ChannelKeyStore, scopes ScopeStore) // // Intended usage: go worker.Start(ctx) func (w *Worker) Start(ctx context.Context) { + if w.cfg.URL == "" { + return + } + queue := newMessageQueue(8, 2048, 32<<20, func(ctx context.Context, msg mqtt.Message) { + w.handleMessageContext(ctx, msg) + msg.Ack() + }) + reportDrops := func() { + if n := queue.takeDropped(); n > 0 { + w.log.Error("ingest queue full, messages dropped", "count", n) + } + } + defer func() { + drainCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := queue.close(drainCtx); err != nil { + w.log.Warn("ingest shutdown timed out", "error", err) + } + reportDrops() + w.log.Info("stopped") + }() + // Isolate workers across deployments; Paho reuses this ID on reconnect. // Keep it alphanumeric and within MQTT 3.1's 23-character client ID limit. opts := mqtt.NewClientOptions(). @@ -251,6 +274,7 @@ func (w *Worker) Start(ctx context.Context) { SetUsername(w.cfg.Username). SetPassword(w.cfg.Password). SetAutoReconnect(true). + SetAutoAckDisabled(true). SetMaxReconnectInterval(30 * time.Second). SetKeepAlive(30 * time.Second). SetPingTimeout(10 * time.Second). @@ -260,21 +284,33 @@ func (w *Worker) Start(ctx context.Context) { SetConnectRetryInterval(5 * time.Second). SetOnConnectHandler(func(c mqtt.Client) { w.log.Info("connected") - w.subscribe(c) + w.subscribe(c, queue) }). SetConnectionLostHandler(func(_ mqtt.Client, err error) { w.log.Warn("connection lost, will reconnect", "error", err) }) w.client = mqtt.NewClient(opts) - if tok := w.client.Connect(); tok.Wait() && tok.Error() != nil { - w.log.Error("initial connect failed", "error", tok.Error()) - // paho will retry; we fall through and wait for ctx + + token := w.client.Connect() + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + connected := token.Done() + for { + select { + case <-connected: + if err := token.Error(); err != nil { + w.log.Error("initial connect failed", "error", err) + } + connected = nil + case <-ticker.C: + reportDrops() + case <-ctx.Done(): + w.client.Disconnect(500) + return + } } - <-ctx.Done() - w.client.Disconnect(500) - w.log.Info("stopped") } func (w *Worker) BrokerName() string { @@ -294,12 +330,15 @@ func (w *Worker) SetCacheInvalidators(onNode, onObserver func(ctx context.Contex } // subscribe registers the wildcard topic handler after (re)connect. -func (w *Worker) subscribe(client mqtt.Client) { +func (w *Worker) subscribe(client mqtt.Client, queue *messageQueue) { // meshcore/{IATA}/{pubkey}/packets // meshcore/{IATA}/{pubkey}/status // We do NOT subscribe to /internal (Role 2 access). tok := client.Subscribe("meshcore/#", 1, func(_ mqtt.Client, msg mqtt.Message) { - w.handleMessage(msg) + if !queue.enqueue(msg) { + // Counted drops must release the broker's inflight slot. + msg.Ack() + } }) if tok.Wait() && tok.Error() != nil { w.log.Error("subscribe error", "error", tok.Error()) @@ -323,8 +362,12 @@ func isValidIATA(s string) bool { // handleMessage dispatches incoming MQTT messages by subtopic. // Each message is processed with a 30s timeout to prevent slow DB calls -// from blocking the MQTT receive goroutine indefinitely. +// from holding a processing worker indefinitely. func (w *Worker) handleMessage(msg mqtt.Message) { + w.handleMessageContext(context.Background(), msg) +} + +func (w *Worker) handleMessageContext(parent context.Context, msg mqtt.Message) { // Topic shape: meshcore/{IATA}/{pubkey}/{subtopic} parts := strings.SplitN(msg.Topic(), "/", 4) if len(parts) != 4 || parts[0] != "meshcore" { @@ -349,7 +392,7 @@ func (w *Worker) handleMessage(msg mqtt.Message) { } } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(parent, 30*time.Second) defer cancel() switch subtopic { diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index ad1bfcbd..92f3da78 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -766,29 +766,6 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ if err != nil { w.log.Error(fmt.Sprintf("db: get observer radio failed for %s", pubkeyHex), "error", err) } - // Endpoints for the live event only; stored rows resolve them at read time. - var resolvedSource, resolvedDestination *api.ResolvedHop - if packet.PayloadType() == meshcore.PayloadTypeAdvert && originPubkey != nil { - // Exact match: ADVERT carries the sender's real identity pubkey, not a - // short ambiguous hash prefix like the other resolvable payload types. - if nodeID, err := w.db.GetNodeByPubkey(ctx, originPubkey); err == nil { - if nodes, err := w.db.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { - hop := api.ResolveExactNode(nodes[nodeID]) - resolvedSource = &hop - } - } - } else if len(sourceHashByte) == 1 { - if r, err := w.db.ResolveEndpointHashes(ctx, iata, [][]byte{sourceHashByte}); err == nil { - hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] - resolvedSource = &hop - } - } - if len(destHashByte) == 1 { - if r, err := w.db.ResolveEndpointHashes(ctx, iata, [][]byte{destHashByte}); err == nil { - hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] - resolvedDestination = &hop - } - } // Airtime is costed from the frame as received; zero radio columns mean the // observer never reported its settings, so there is nothing to cost. var airtimeMs *float32 @@ -885,6 +862,29 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ // A duplicate is streamed, never stored, to includeRepeats clients when its path is new. repeat := !inserted && w.hub.RepeatsWanted() && w.hub.MarkSent(packetHash[:], id[:], packet.Path) if inserted || repeat { + // Endpoints for the live event only; stored rows resolve them at read time. + var resolvedSource, resolvedDestination *api.ResolvedHop + if packet.PayloadType() == meshcore.PayloadTypeAdvert && originPubkey != nil { + // Exact match: ADVERT carries the sender's real identity pubkey, not a + // short ambiguous hash prefix like the other resolvable payload types. + if nodeID, err := w.db.GetNodeByPubkey(ctx, originPubkey); err == nil { + if nodes, err := w.db.GetNodesByIDs(ctx, []uuid.UUID{nodeID}); err == nil { + hop := api.ResolveExactNode(nodes[nodeID]) + resolvedSource = &hop + } + } + } else if len(sourceHashByte) == 1 { + if r, err := w.db.ResolveEndpointHashes(ctx, iata, [][]byte{sourceHashByte}); err == nil { + hop := api.BuildResolvedPath([][]byte{sourceHashByte}, r)[0] + resolvedSource = &hop + } + } + if len(destHashByte) == 1 { + if r, err := w.db.ResolveEndpointHashes(ctx, iata, [][]byte{destHashByte}); err == nil { + hop := api.BuildResolvedPath([][]byte{destHashByte}, r)[0] + resolvedDestination = &hop + } + } if inserted { w.handlePayloadTypeSideEffects(ctx, packet, iata, packetHash[:], radio, scopeID, matchedScope, pubkeyBytes, float32(parseNumber(envelope.SNR))) if w.hub.RepeatsWanted() { diff --git a/internal/ingest/queue.go b/internal/ingest/queue.go new file mode 100644 index 00000000..ff7e6d5e --- /dev/null +++ b/internal/ingest/queue.go @@ -0,0 +1,108 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "strings" + "sync" + + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +type messageQueue struct { + mu sync.Mutex + lanes []chan mqtt.Message + bytes, maxBytes int + dropped uint64 + closed bool + done chan struct{} + cancel context.CancelFunc +} + +func newMessageQueue(workers, capacity, maxBytes int, handle func(context.Context, mqtt.Message)) *messageQueue { + ctx, cancel := context.WithCancel(context.Background()) + q := &messageQueue{lanes: make([]chan mqtt.Message, workers), maxBytes: maxBytes, done: make(chan struct{}), cancel: cancel} + var wg sync.WaitGroup + for i := range q.lanes { + lane := make(chan mqtt.Message, capacity) + q.lanes[i] = lane + wg.Go(func() { + for m := range lane { + if ctx.Err() != nil { + return + } + handle(ctx, m) + q.mu.Lock() + q.bytes -= len(m.Topic()) + len(m.Payload()) + q.mu.Unlock() + } + }) + } + go func() { wg.Wait(); close(q.done) }() + return q +} + +func (q *messageQueue) enqueue(m mqtt.Message) bool { + // Keep status and packets from each observer in arrival order. + parts := strings.SplitN(m.Topic(), "/", 4) + key := m.Topic() + if len(parts) == 4 { + key = parts[2] + } + hash := uint32(2166136261) + for i := range len(key) { + c := key[i] + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + hash = (hash ^ uint32(c)) * 16777619 + } + size := len(m.Topic()) + len(m.Payload()) + q.mu.Lock() + defer q.mu.Unlock() + if q.closed { + return false + } + if size > q.maxBytes-q.bytes { + q.dropped++ + return false + } + select { + case q.lanes[int(hash%uint32(len(q.lanes)))] <- m: + q.bytes += size + return true + default: + q.dropped++ + return false + } +} + +func (q *messageQueue) takeDropped() uint64 { + q.mu.Lock() + defer q.mu.Unlock() + n := q.dropped + q.dropped = 0 + return n +} + +func (q *messageQueue) close(ctx context.Context) error { + q.mu.Lock() + if !q.closed { + q.closed = true + for _, lane := range q.lanes { + close(lane) + } + } + q.mu.Unlock() + select { + case <-q.done: + q.cancel() + return nil + case <-ctx.Done(): + q.cancel() + <-q.done + return ctx.Err() + } +} diff --git a/internal/ingest/queue_test.go b/internal/ingest/queue_test.go new file mode 100644 index 00000000..130a17a2 --- /dev/null +++ b/internal/ingest/queue_test.go @@ -0,0 +1,194 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "fmt" + "reflect" + "sync" + "testing" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +type queuedTestMessage struct { + malformedTopicMessage + payload []byte +} + +func (m queuedTestMessage) Payload() []byte { return m.payload } +func queueMessage(observer, kind string, n int) mqtt.Message { + return queuedTestMessage{malformedTopicMessage{"meshcore/SEA/" + observer + "/" + kind}, []byte(fmt.Sprint(n))} +} + +func TestQueueKeepsObserverOrderWhileDatabaseIsBusy(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + var mu sync.Mutex + var got []string + q := newMessageQueue(2, 8, 4096, func(ctx context.Context, m mqtt.Message) { + if string(m.Payload()) == "0" { + close(entered) + select { + case <-release: + case <-ctx.Done(): + return + } + } + mu.Lock() + got = append(got, string(m.Payload())) + mu.Unlock() + }) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = q.close(ctx) + }) + if !q.enqueue(queueMessage("aa", "status", 0)) { + t.Fatal("first message rejected") + } + <-entered + for n := 1; n <= 3; n++ { + if !q.enqueue(queueMessage("AA", "packets", n)) { + t.Fatal("callback blocked or message rejected") + } + } + close(release) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := q.close(ctx); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, []string{"0", "1", "2", "3"}) { + t.Fatalf("observer order: %v", got) + } + if q.enqueue(queueMessage("aa", "packets", 4)) { + t.Fatal("accepted after shutdown") + } +} + +func TestQueueRunsOtherObserversAndReportsOverflow(t *testing.T) { + entered, other := make(chan struct{}), make(chan struct{}) + q := newMessageQueue(2, 1, 4096, func(ctx context.Context, m mqtt.Message) { + if m.Topic() == "meshcore/SEA/a/packets" { + close(entered) + <-ctx.Done() + } else { + close(other) + } + }) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = q.close(ctx) + }) + q.enqueue(queueMessage("a", "packets", 0)) + <-entered + if !q.enqueue(queueMessage("b", "packets", 0)) { + t.Fatal("other observer rejected") + } + select { + case <-other: + case <-time.After(time.Second): + t.Fatal("other observer blocked") + } + if !q.enqueue(queueMessage("a", "packets", 1)) { + t.Fatal("buffered message rejected") + } + if q.enqueue(queueMessage("a", "packets", 2)) { + t.Fatal("accepted beyond queue capacity") + } + if q.takeDropped() != 1 { + t.Fatal("overflow was not counted") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if q.close(ctx) == nil { + t.Fatal("expected shutdown deadline") + } +} + +func TestQueueBoundsPayloadMemory(t *testing.T) { + q := newMessageQueue(1, 8, 64, func(context.Context, mqtt.Message) {}) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = q.close(ctx) + }) + m := queuedTestMessage{malformedTopicMessage{"meshcore/SEA/a/packets"}, make([]byte, 65)} + if q.enqueue(m) { + t.Fatal("accepted oversized payload") + } + if q.takeDropped() != 1 { + t.Fatal("memory overflow was not counted") + } +} + +type subscribeCapture struct { + mqtt.Client + callback mqtt.MessageHandler +} + +func (c *subscribeCapture) Subscribe(_ string, _ byte, callback mqtt.MessageHandler) mqtt.Token { + c.callback = callback + return completedSubscribe{} +} + +type completedSubscribe struct{ mqtt.Token } + +func (completedSubscribe) Wait() bool { return true } +func (completedSubscribe) Error() error { return nil } + +type ackMessage struct { + mqtt.Message + acked bool +} + +func (m *ackMessage) Ack() { m.acked = true } + +func TestSubscribeReleasesRejectedMessage(t *testing.T) { + w, _ := newTestWorker() + q := newMessageQueue(1, 1, 1, func(context.Context, mqtt.Message) { t.Error("rejected message processed") }) + defer q.close(context.Background()) + client := &subscribeCapture{} + w.subscribe(client, q) + m := &ackMessage{Message: queueMessage("aa", "packets", 0)} + client.callback(client, m) + if !m.acked { + t.Fatal("rejected message still occupies the broker inflight window") + } + if q.takeDropped() != 1 { + t.Fatal("rejected message was not counted") + } +} + +func TestQueueShutdownWaitsForCanceledHandler(t *testing.T) { + entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + q := newMessageQueue(1, 1, 4096, func(ctx context.Context, _ mqtt.Message) { + close(entered) + <-ctx.Done() + close(canceled) + <-release + }) + q.enqueue(queueMessage("aa", "packets", 0)) + <-entered + ctx, cancel := context.WithCancel(context.Background()) + cancel() + stopped := make(chan error, 1) + go func() { stopped <- q.close(ctx) }() + <-canceled + select { + case <-stopped: + close(release) + t.Fatal("shutdown returned while handler was still running") + case <-time.After(20 * time.Millisecond): + } + close(release) + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("shutdown did not finish") + } +} From db30c9b573357990c41166292e7cf42785c92cc4 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:41:48 -0400 Subject: [PATCH 057/141] fix(db): batch route reconfirmation to avoid ingest stalls (#180) --- db/queries/queries.sql | 35 +-- db/reconfirm_integration_test.go | 324 ++++++++++++++++++++++++++ db/routes.go | 7 +- db/sqlc/mock/querier.go | 13 +- db/sqlc/querier.go | 4 +- db/sqlc/queries.sql.go | 48 ++-- internal/background/reconfirm_test.go | 95 ++++++++ internal/background/tasks.go | 34 ++- 8 files changed, 509 insertions(+), 51 deletions(-) create mode 100644 db/reconfirm_integration_test.go create mode 100644 internal/background/reconfirm_test.go diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 9c467944..ccc8492f 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1481,15 +1481,17 @@ REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets; -- name: RefreshObserverActivity :exec REFRESH MATERIALIZED VIEW CONCURRENTLY mv_observer_activity_hourly; --- name: ReconfirmRoutes :exec --- Checks the $1 least-recently-reconfirmed routes: deletes those with a departed +-- name: ReconfirmRoutes :one +-- Checks one batch of least-recently-reconfirmed routes: deletes those with a departed -- hop node or a hop prefix now matching >1 node in that IATA (length-aware: -- 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. -WITH batch AS ( - SELECT iata, path_key, node_ids, hash_prefix - FROM known_routes - ORDER BY last_reconfirmed_at - LIMIT $1 +WITH batch AS MATERIALIZED ( + SELECT r.iata, r.path_key, r.node_ids, r.hash_prefix + FROM known_routes r + WHERE r.last_reconfirmed_at < @before + ORDER BY r.last_reconfirmed_at + LIMIT @batch_size + FOR UPDATE OF r SKIP LOCKED ), amb AS MATERIALIZED ( SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 @@ -1522,15 +1524,18 @@ deleted AS ( DELETE FROM known_routes kr USING dead d WHERE kr.iata = d.iata AND kr.path_key = d.path_key +), +updated AS ( + UPDATE known_routes kr + SET last_reconfirmed_at = GREATEST(NOW(), @before::timestamptz) + FROM batch b + WHERE kr.iata = b.iata AND kr.path_key = b.path_key + AND NOT EXISTS ( + SELECT 1 FROM dead d + WHERE d.iata = b.iata AND d.path_key = b.path_key + ) ) -UPDATE known_routes kr -SET last_reconfirmed_at = NOW() -FROM batch b -WHERE kr.iata = b.iata AND kr.path_key = b.path_key - AND NOT EXISTS ( - SELECT 1 FROM dead d - WHERE d.iata = b.iata AND d.path_key = b.path_key - ); +SELECT count(*) FROM batch; -- name: ReconfirmNeighbors :exec -- Delete node_neighbors where the neighbor has departed from node_short_ids diff --git a/db/reconfirm_integration_test.go b/db/reconfirm_integration_test.go new file mode 100644 index 00000000..7aeed34e --- /dev/null +++ b/db/reconfirm_integration_test.go @@ -0,0 +1,324 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "fmt" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestReconfirmReleasesEarlierBatchPostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + before := time.Now() + if n, err := store.ReconfirmRoutes(ctx, 1, before); err != nil || n != 1 { + t.Fatalf("first batch: %d, %v", n, err) + } + gate, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer gate.Rollback(context.Background()) + key := time.Now().UnixNano() + if _, err := gate.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, key); err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, fmt.Sprintf(` +CREATE FUNCTION pause_second_route() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN IF NEW.path_key=int4send(2) THEN PERFORM pg_advisory_xact_lock(%d); END IF; RETURN NEW; END $$; +CREATE TRIGGER pause_second_route BEFORE UPDATE ON known_routes FOR EACH ROW EXECUTE FUNCTION pause_second_route();`, key)) + if err != nil { + t.Fatal(err) + } + finished := make(chan error, 1) + go func() { + _, err := store.ReconfirmRoutes(ctx, 1, before) + finished <- err + }() + deadline := time.Now().Add(3 * time.Second) + for { + var waiting bool + if err := pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted AND database=(SELECT oid FROM pg_database WHERE datname=current_database()))`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting { + break + } + if time.Now().After(deadline) { + t.Fatal("second batch did not reach the test gate") + } + time.Sleep(time.Millisecond) + } + upsertCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + err = sqlc.New(pool).UpsertKnownRoute(upsertCtx, sqlc.UpsertKnownRouteParams{ + PathKey: []byte{0, 0, 0, 1}, NodeIds: []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000001")}, + HashPrefix: [][]byte{{0x11}}, Iata: "SEA", HopCount: 1, + }) + if err != nil { + t.Fatalf("completed batch still blocks ingest: %v", err) + } + if err := gate.Rollback(ctx); err != nil { + t.Fatal(err) + } + if err := <-finished; err != nil { + t.Fatal(err) + } + var count int + if err := pool.QueryRow(ctx, `SELECT observation_count FROM known_routes WHERE path_key=int4send(1)`).Scan(&count); err != nil || count != 2 { + t.Fatalf("concurrent observation increment lost: count %d, error %v", count, err) + } +} + +func TestReconfirmPreservesRouteDataPostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + before := time.Now().UTC().Truncate(time.Microsecond) + _, err := pool.Exec(ctx, ` +TRUNCATE known_routes, node_short_ids; +INSERT INTO node_short_ids (node_id,iata,prefix_4) VALUES + (md5('n1')::uuid,'SEA','\x11223344'), (md5('n2')::uuid,'SEA','\x11223355'), + (md5('n3')::uuid,'SEA','\xaabbccdd'), (md5('n4')::uuid,'SEA','\xaabbccdd'), + (md5('n5')::uuid,'SEA','\x55667788'), (md5('n6')::uuid,'SFO','\x55667799'); +INSERT INTO known_routes (id,path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen,observation_count,last_reconfirmed_at) +SELECT id,int4send(id),ARRAY[md5(node)::uuid],ARRAY[decode(prefix,'hex')],iata,1, + '2025-12-01','2026-01-01',42,'2026-01-01' +FROM (VALUES + (1,'n1','11223344','SEA'), (2,'n2','11223355','SEA'), + (3,'n5','55','SEA'), (4,'n5','5566','SEA'), (5,'n5','556677','SEA'), (6,'n5','55667788','SEA'), + (7,'n1','11','SEA'), (8,'n1','1122','SEA'), (9,'n1','112233','SEA'), (10,'n3','aabbccdd','SEA'), + (11,'missing','55','SEA'), (12,'n6','55','SEA'), + (13,'n1','11223344','SEA'), (14,'n5','55667788','SEA'), (15,'n6','55','SFO'), + (16,'n5','55','SEA'), (17,'n5','55','SEA') +) AS routes(id,node,prefix,iata); +UPDATE known_routes SET node_ids=node_ids || md5('n5')::uuid, hash_prefix=hash_prefix || '\x55667788'::bytea, hop_count=2 WHERE id=13; +UPDATE known_routes SET node_ids=node_ids || md5('missing')::uuid, hash_prefix=hash_prefix || '\x99'::bytea, hop_count=2 WHERE id=14;`) + if err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, `UPDATE known_routes SET last_reconfirmed_at=$1::timestamptz + CASE WHEN id=16 THEN interval '1 hour' ELSE interval '0' END WHERE id IN (16,17)`, before); err != nil { + t.Fatal(err) + } + snapshot := func() map[int][]byte { + t.Helper() + rows, err := pool.Query(ctx, `SELECT id, to_jsonb(r)-'last_reconfirmed_at' FROM known_routes r`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + result := map[int][]byte{} + for rows.Next() { + var id int + var data []byte + if err := rows.Scan(&id, &data); err != nil { + t.Fatal(err) + } + result[id] = data + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return result + } + original := snapshot() + var checked int64 + for range 10 { + n, err := store.ReconfirmRoutes(ctx, 3, before) + if err != nil { + t.Fatal(err) + } + checked += n + if n == 0 { + break + } + } + if checked != 15 { + t.Fatalf("checked %d routes; want 15 once each, including deleted routes", checked) + } + after := snapshot() + want := []int{1, 2, 3, 4, 5, 6, 13, 15, 16, 17} + if len(after) != len(want) { + t.Fatalf("surviving routes: %v", after) + } + for _, id := range want { + if !bytes.Equal(original[id], after[id]) { + t.Errorf("route %d lost or changed fields: before %s, after %s", id, original[id], after[id]) + } + } + var stamped int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes WHERE id NOT IN (16,17) AND last_reconfirmed_at >= $1`, before).Scan(&stamped); err != nil { + t.Fatal(err) + } + if stamped != 8 { + t.Errorf("validated survivors = %d, want 8", stamped) + } +} + +func TestReconfirmCancelledBatchRollsBackPostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + before := time.Now() + if _, err := store.ReconfirmRoutes(ctx, 1, before); err != nil { + t.Fatal(err) + } + _, err := pool.Exec(ctx, ` +CREATE FUNCTION pause_reconfirm() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN PERFORM pg_sleep(1); RETURN NEW; END $$; +CREATE TRIGGER pause_reconfirm BEFORE UPDATE ON known_routes FOR EACH ROW EXECUTE FUNCTION pause_reconfirm();`) + if err != nil { + t.Fatal(err) + } + batchCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + if _, err := store.ReconfirmRoutes(batchCtx, 1, before); err == nil { + t.Fatal("expected batch cancellation") + } + // Waiting for this DDL also waits for cancellation to release the row lock. + if _, err := pool.Exec(ctx, `DROP TRIGGER pause_reconfirm ON known_routes`); err != nil { + t.Fatal(err) + } + var total, stamped int + if err := pool.QueryRow(ctx, `SELECT count(*),count(*) FILTER (WHERE last_reconfirmed_at >= $1) FROM known_routes`, before).Scan(&total, &stamped); err != nil { + t.Fatal(err) + } + if total != 2 || stamped != 1 { + t.Fatalf("cancellation changed committed work: total %d, checked %d", total, stamped) + } + if n, err := store.ReconfirmRoutes(ctx, 1, before); err != nil || n != 1 { + t.Fatalf("cancelled route was not available to retry: %d, %v", n, err) + } +} + +func TestReconfirmFutureCutoffDoesNotRepeatPostgres(t *testing.T) { + ctx, _, store := reconfirmPool(t) + before := time.Now().Add(time.Hour) + if n, err := store.ReconfirmRoutes(ctx, 2, before); err != nil || n != 2 { + t.Fatalf("first batch: %d, %v", n, err) + } + if n, err := store.ReconfirmRoutes(ctx, 2, before); err != nil || n != 0 { + t.Fatalf("completed routes consumed the run budget twice: %d, %v", n, err) + } +} + +func TestReconfirmCancellationDoesNotPartiallyDeletePostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + _, err := pool.Exec(ctx, ` +UPDATE known_routes SET node_ids=ARRAY[md5('departed')::uuid] WHERE path_key=int4send(1); +CREATE FUNCTION pause_valid_route() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN PERFORM pg_sleep(1); RETURN NEW; END $$; +CREATE TRIGGER pause_valid_route BEFORE UPDATE ON known_routes FOR EACH ROW EXECUTE FUNCTION pause_valid_route();`) + if err != nil { + t.Fatal(err) + } + before := time.Now() + batchCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + if _, err := store.ReconfirmRoutes(batchCtx, 2, before); err == nil { + t.Fatal("expected batch cancellation") + } + if _, err := pool.Exec(ctx, `DROP TRIGGER pause_valid_route ON known_routes`); err != nil { + t.Fatal(err) + } + var count int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes`).Scan(&count); err != nil || count != 2 { + t.Fatalf("cancelled batch partially deleted routes: %d, %v", count, err) + } + if n, err := store.ReconfirmRoutes(ctx, 2, before); err != nil || n != 2 { + t.Fatalf("retry did not process both routes: %d, %v", n, err) + } + if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes WHERE path_key=int4send(2)`).Scan(&count); err != nil || count != 1 { + t.Fatalf("retry lost the valid route: %d, %v", count, err) + } +} + +// A private schema lets independent connections exercise row locks. +func reconfirmPool(t *testing.T) (context.Context, *pgxpool.Pool, *Store) { + t.Helper() + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL tests") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + admin, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { admin.Close(context.Background()) }) + schema := pgx.Identifier{"reconfirm_" + uuid.NewString()}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := admin.Exec(context.Background(), "DROP SCHEMA "+schema+" CASCADE"); err != nil { + t.Error(err) + } + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.MaxConns = 4 + cfg.ConnConfig.RuntimeParams["search_path"] = schema + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + _, err = pool.Exec(ctx, ` +CREATE TABLE known_routes (LIKE public.known_routes INCLUDING ALL); +CREATE TABLE node_short_ids (LIKE public.node_short_ids INCLUDING ALL); +INSERT INTO node_short_ids (node_id, iata, prefix_4) VALUES + ('00000000-0000-0000-0000-000000000001', 'SEA', '\x11223344'); +INSERT INTO known_routes (path_key,node_ids,hash_prefix,iata,hop_count,last_reconfirmed_at) +SELECT int4send(i), ARRAY['00000000-0000-0000-0000-000000000001'::uuid], ARRAY['\x11'::bytea], 'SEA', 1, + '2026-01-01'::timestamptz + i * interval '1 second' +FROM generate_series(1,2) i;`) + if err != nil { + t.Fatal(err) + } + return ctx, pool, New(pool, 0, 0) +} + +func TestReconfirmSkipsBusyRoutePostgres(t *testing.T) { + ctx, pool, store := reconfirmPool(t) + busy, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer busy.Rollback(context.Background()) + if _, err := busy.Exec(ctx, `SELECT 1 FROM known_routes WHERE path_key=int4send(1) FOR UPDATE`); err != nil { + t.Fatal(err) + } + batchCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + defer cancel() + if _, err := store.ReconfirmRoutes(batchCtx, 1, time.Now()); err != nil { + t.Fatalf("maintenance should skip the busy route and validate the next one: %v", err) + } + var checked, preserved int + if err := pool.QueryRow(ctx, `SELECT count(*) FILTER (WHERE last_reconfirmed_at > '2026-02-01'), count(*) FROM known_routes`).Scan(&checked, &preserved); err != nil { + t.Fatal(err) + } + if checked != 1 || preserved != 2 { + t.Fatalf("checked %d, preserved %d; want 1 and 2", checked, preserved) + } + if err := busy.Rollback(ctx); err != nil { + t.Fatal(err) + } + if _, err := store.ReconfirmRoutes(ctx, 1, time.Now()); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes WHERE last_reconfirmed_at > '2026-02-01'`).Scan(&checked); err != nil { + t.Fatal(err) + } + if checked != 2 { + t.Fatalf("deferred route was not validated after release: checked %d", checked) + } +} diff --git a/db/routes.go b/db/routes.go index a629bcbc..187e07ec 100644 --- a/db/routes.go +++ b/db/routes.go @@ -288,8 +288,11 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t // ReconfirmRoutes checks the batchSize least-recently-reconfirmed routes, // deleting stale or ambiguous ones and stamping the survivors. -func (s *Store) ReconfirmRoutes(ctx context.Context, batchSize int32) error { - return s.q.ReconfirmRoutes(ctx, batchSize) +func (s *Store) ReconfirmRoutes(ctx context.Context, batchSize int32, before time.Time) (int64, error) { + return s.q.ReconfirmRoutes(ctx, sqlc.ReconfirmRoutesParams{ + BatchSize: batchSize, + Before: pgtype.Timestamptz{Time: before, Valid: true}, + }) } const routeDeleteBatch = 10000 diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 6a665e77..aeae62b1 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -1253,17 +1253,18 @@ func (mr *MockQuerierMockRecorder) ReconfirmNeighbors(ctx any) *gomock.Call { } // ReconfirmRoutes mocks base method. -func (m *MockQuerier) ReconfirmRoutes(ctx context.Context, limit int32) error { +func (m *MockQuerier) ReconfirmRoutes(ctx context.Context, arg db.ReconfirmRoutesParams) (int64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx, limit) - ret0, _ := ret[0].(error) - return ret0 + ret := m.ctrl.Call(m, "ReconfirmRoutes", ctx, arg) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 } // ReconfirmRoutes indicates an expected call of ReconfirmRoutes. -func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx, limit any) *gomock.Call { +func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx, limit) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx, arg) } // RefreshHourlyStats mocks base method. diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 232eae3b..1547c1c9 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -211,10 +211,10 @@ type Querier interface { // Delete node_neighbors where the neighbor has departed from node_short_ids // for that IATA, or where its prefix_4 is now ambiguous. ReconfirmNeighbors(ctx context.Context) error - // Checks the $1 least-recently-reconfirmed routes: deletes those with a departed + // Checks one batch of least-recently-reconfirmed routes: deletes those with a departed // hop node or a hop prefix now matching >1 node in that IATA (length-aware: // 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. - ReconfirmRoutes(ctx context.Context, limit int32) error + ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams) (int64, error) RefreshHourlyStats(ctx context.Context) error RefreshObserverActivity(ctx context.Context) error RefreshPathStats(ctx context.Context) error diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 38d299c6..c047b44f 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -3865,12 +3865,14 @@ func (q *Queries) ReconfirmNeighbors(ctx context.Context) error { return err } -const reconfirmRoutes = `-- name: ReconfirmRoutes :exec -WITH batch AS ( - SELECT iata, path_key, node_ids, hash_prefix - FROM known_routes - ORDER BY last_reconfirmed_at - LIMIT $1 +const reconfirmRoutes = `-- name: ReconfirmRoutes :one +WITH batch AS MATERIALIZED ( + SELECT r.iata, r.path_key, r.node_ids, r.hash_prefix + FROM known_routes r + WHERE r.last_reconfirmed_at < $1 + ORDER BY r.last_reconfirmed_at + LIMIT $2 + FOR UPDATE OF r SKIP LOCKED ), amb AS MATERIALIZED ( SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 @@ -3903,23 +3905,33 @@ deleted AS ( DELETE FROM known_routes kr USING dead d WHERE kr.iata = d.iata AND kr.path_key = d.path_key +), +updated AS ( + UPDATE known_routes kr + SET last_reconfirmed_at = GREATEST(NOW(), $1::timestamptz) + FROM batch b + WHERE kr.iata = b.iata AND kr.path_key = b.path_key + AND NOT EXISTS ( + SELECT 1 FROM dead d + WHERE d.iata = b.iata AND d.path_key = b.path_key + ) ) -UPDATE known_routes kr -SET last_reconfirmed_at = NOW() -FROM batch b -WHERE kr.iata = b.iata AND kr.path_key = b.path_key - AND NOT EXISTS ( - SELECT 1 FROM dead d - WHERE d.iata = b.iata AND d.path_key = b.path_key - ) +SELECT count(*) FROM batch ` -// Checks the $1 least-recently-reconfirmed routes: deletes those with a departed +type ReconfirmRoutesParams struct { + Before pgtype.Timestamptz `json:"before"` + BatchSize int32 `json:"batch_size"` +} + +// Checks one batch of least-recently-reconfirmed routes: deletes those with a departed // hop node or a hop prefix now matching >1 node in that IATA (length-aware: // 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. -func (q *Queries) ReconfirmRoutes(ctx context.Context, limit int32) error { - _, err := q.db.Exec(ctx, reconfirmRoutes, limit) - return err +func (q *Queries) ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams) (int64, error) { + row := q.db.QueryRow(ctx, reconfirmRoutes, arg.Before, arg.BatchSize) + var count int64 + err := row.Scan(&count) + return count, err } const refreshHourlyStats = `-- name: RefreshHourlyStats :exec diff --git a/internal/background/reconfirm_test.go b/internal/background/reconfirm_test.go new file mode 100644 index 00000000..2012688f --- /dev/null +++ b/internal/background/reconfirm_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "testing" + "testing/synctest" + "time" +) + +type reconfirmStore struct { + run func(context.Context, int32, time.Time) (int64, error) + neighbors bool +} + +func (*reconfirmStore) DeleteOldRoutes(context.Context, time.Time, int64, time.Time) error { + return nil +} + +func (s *reconfirmStore) ReconfirmRoutes(ctx context.Context, n int32, before time.Time) (int64, error) { + return s.run(ctx, n, before) +} + +func (s *reconfirmStore) ReconfirmNeighbors(context.Context) error { + s.neighbors = true + return nil +} + +func TestReconfirmCoverage(t *testing.T) { + for _, tc := range []struct { + name string + available, wantLeft int64 + }{ + {"empty", 0, 0}, + {"partial last batch", 1501, 0}, + {"full run budget", 750010, 10}, + } { + t.Run(tc.name, func(t *testing.T) { + left := tc.available + var cutoff time.Time + s := &reconfirmStore{run: func(ctx context.Context, n int32, before time.Time) (int64, error) { + if n <= 0 || n > 1000 { + t.Fatalf("transaction requested %d routes", n) + } + if cutoff.IsZero() { + cutoff = before + } else if !cutoff.Equal(before) { + t.Fatal("cutoff advanced, allowing completed routes to be revisited") + } + checked := min(int64(n), left) + left -= checked + return checked, ctx.Err() + }} + if err := ReconfirmTask(s, time.Hour, time.Minute, 3, time.Hour).Run(context.Background()); err != nil { + t.Fatal(err) + } + if left != tc.wantLeft || !s.neighbors { + t.Fatalf("remaining %d, neighbor cleanup %t", left, s.neighbors) + } + }) + } +} + +func TestReconfirmStopsOnFailure(t *testing.T) { + want := errors.New("database unavailable") + calls := 0 + s := &reconfirmStore{run: func(context.Context, int32, time.Time) (int64, error) { + calls++ + if calls == 1 { + return 1000, nil + } + return 0, want + }} + err := ReconfirmTask(s, time.Hour, time.Minute, 3, time.Hour).Run(context.Background()) + if !errors.Is(err, want) || calls != 2 || s.neighbors { + t.Fatalf("failure was not propagated: error %v, calls %d, neighbors %t", err, calls, s.neighbors) + } +} + +func TestReconfirmBatchDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s := &reconfirmStore{run: func(ctx context.Context, _ int32, _ time.Time) (int64, error) { + <-ctx.Done() + return 0, ctx.Err() + }} + start := time.Now() + err := ReconfirmTask(s, time.Hour, time.Minute, 3, time.Hour).Run(context.Background()) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(start) > 5*time.Second || s.neighbors { + t.Fatalf("batch exceeded its deadline: error %v, duration %s, neighbors %t", err, time.Since(start), s.neighbors) + } + }) +} diff --git a/internal/background/tasks.go b/internal/background/tasks.go index 64c4297d..7d82fdfa 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -86,9 +86,18 @@ func CleanupTask(store *db.Store, telemetryRetention, packetRetention, nodeDelet } } -// reconfirmBatchSize bounds per-tick reconfirm work; at hourly ticks a 16M-row -// table gets fully re-checked roughly daily. -const reconfirmBatchSize = 750_000 +// Limit lock lifetime while retaining the per-run work budget. +const ( + reconfirmRunLimit = 750_000 + reconfirmBatchSize = 1_000 + reconfirmBatchTimeout = 5 * time.Second +) + +type routeMaintainer interface { + DeleteOldRoutes(context.Context, time.Time, int64, time.Time) error + ReconfirmRoutes(context.Context, int32, time.Time) (int64, error) + ReconfirmNeighbors(context.Context) error +} type observerCleaner interface { DeleteOldObservers(context.Context, time.Time) ([]uuid.UUID, error) @@ -118,9 +127,8 @@ func ObserverCleanupTask(store observerCleaner, deleteAfter, interval time.Durat } // ReconfirmTask returns a Task that prunes aged routes first, then reconfirms -// stale and ambiguous resolved paths and neighbors, so known_routes only ever -// has one writer at a time. -func ReconfirmTask(store *db.Store, routeRetention, routeGrace time.Duration, routeMinObservations int64, interval time.Duration) Task { +// stale and ambiguous resolved paths and neighbors in order. +func ReconfirmTask(store routeMaintainer, routeRetention, routeGrace time.Duration, routeMinObservations int64, interval time.Duration) Task { return Task{ Name: "reconfirm", Interval: interval, @@ -129,8 +137,18 @@ func ReconfirmTask(store *db.Store, routeRetention, routeGrace time.Duration, ro if err := store.DeleteOldRoutes(ctx, now.Add(-routeRetention), routeMinObservations, now.Add(-routeGrace)); err != nil { return fmt.Errorf("route retention: %w", err) } - if err := store.ReconfirmRoutes(ctx, reconfirmBatchSize); err != nil { - return fmt.Errorf("routes: %w", err) + for remaining := int64(reconfirmRunLimit); remaining > 0; { + limit := min(int64(reconfirmBatchSize), remaining) + batchCtx, cancel := context.WithTimeout(ctx, reconfirmBatchTimeout) + n, err := store.ReconfirmRoutes(batchCtx, int32(limit), now) + cancel() + if err != nil { + return fmt.Errorf("routes: %w", err) + } + remaining -= n + if n < limit { + break + } } if err := store.ReconfirmNeighbors(ctx); err != nil { return fmt.Errorf("neighbors: %w", err) From fd7c6b3eb7b520eeb171e8fc1378846ec1954a05 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 07:16:27 -0400 Subject: [PATCH 058/141] feat(db): retain 30-day analytics independently of packet expiry (#167) --- .github/workflows/ci.yml | 2 +- README.md | 17 + db/analytics_retention_integration_test.go | 295 ++++++++++ db/migrations/039_analytics_retention.sql | 613 +++++++++++++++++++++ db/queries/queries.sql | 15 +- db/retention_integration_test.go | 5 +- db/sqlc/models.go | 158 ++++++ db/sqlc/querier.go | 2 +- db/sqlc/queries.sql.go | 24 +- sqlc.yaml | 11 + 10 files changed, 1108 insertions(+), 34 deletions(-) create mode 100644 db/analytics_retention_integration_test.go create mode 100644 db/migrations/039_analytics_retention.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beb7284f..7d374920 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index 5418223c..823c04be 100644 --- a/README.md +++ b/README.md @@ -533,3 +533,20 @@ log: `LOG_LEVEL` and `LOG_FORMAT` override file settings; empty settings use `info` and `text`. Invalid values prevent startup. Configuration-loading failures can use the bootstrap text logger before file settings are available; failures after initialization retain error severity at every supported level. Records include a component field. Ingest workers also include their broker name, and HTTP completion records include the validated client address, route, status and duration. Query strings and protocol hello payloads are excluded. Expected ingest skips and routine WebSocket lifecycle details are debug-level. Changing the application's format does not change Caddy/Apache access logs or their fail2ban configuration. Collect/rotate stderr through Docker or systemd. + +### Analytics retention + +Hourly traffic, payload, observer activity, talker, advertiser, Signal and Paths +summaries retain 30 days independently of `packets.retention`. Cleanup saves only +aggregates before deleting each packet batch, in the same transaction. Raw +packets, observations and message bodies still expire under packet retention. +The summaries use UTC hourly buckets and appear on the normal view-refresh cycle. +`mv_hourly_iata_stats` now covers 30 days instead of migration 001's seven days; +`/stats/observations?since=` can therefore return retained summaries older than a week. + +Migration 039 starts from data still present; previously deleted history cannot +be reconstructed. Telemetry has its own retention setting. Packet drill-down, +sub-hour observer activity, exact observer comparison and current entity/scope +counts continue to describe retained raw data or current entities, rather than +claiming archived packet detail. Archived summaries expire without waiting for +new packet deletions. A failed archive leaves its entire raw batch intact. diff --git a/db/analytics_retention_integration_test.go b/db/analytics_retention_integration_test.go new file mode 100644 index 00000000..10e33339 --- /dev/null +++ b/db/analytics_retention_integration_test.go @@ -0,0 +1,295 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "os" + "strings" + "testing" + "time" +) + +var retainedViews = []string{"mv_hourly_iata_stats", "mv_payload_breakdown_by_iata", "mv_top_observers_by_iata", "mv_top_talkers_by_iata", "mv_top_advertisers_by_iata", "mv_observer_activity_hourly", "mv_signal_stats_hourly", "mv_path_stats_hourly"} + +func analyticsSnapshot(t *testing.T, ctx context.Context, tx pgx.Tx) map[string]string { + t.Helper() + result := map[string]string{} + for _, view := range retainedViews { + if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW "+view); err != nil { + t.Fatal(err) + } + var rows string + if err := tx.QueryRow(ctx, "SELECT COALESCE(jsonb_agg(r ORDER BY r::text),'[]')::text FROM (SELECT to_jsonb(v) r FROM "+view+" v) s").Scan(&rows); err != nil { + t.Fatal(err) + } + result[view] = rows + } + return result +} + +func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash,last_heard_at) VALUES ('\x01',NOW()-interval '5 days'); INSERT INTO observers(id) VALUES ('00000000-0000-0000-0000-000000000001')`); err != nil { + t.Fatal(err) + } + var schema string + if err := tx.QueryRow(ctx, "SELECT current_schema()").Scan(&schema); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + conn := tx.Conn() + quoted := pgx.Identifier{schema}.Sanitize() + t.Cleanup(func() { + if _, err := conn.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error(err) + } + }) + if _, err := conn.Exec(ctx, "SET search_path TO "+quoted); err != nil { + t.Fatal(err) + } + writer, err := pgx.Connect(ctx, os.Getenv("BEACON_TEST_POSTGRES_DSN")) + if err != nil { + t.Fatal(err) + } + defer writer.Close(context.Background()) + if _, err = writer.Exec(ctx, "SET search_path TO "+quoted); err != nil { + t.Fatal(err) + } + ingest, err := writer.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer ingest.Rollback(context.Background()) + // An in-flight FK insert holds KEY SHARE on the packet until it commits. + if _, err = ingest.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\x01','00000000-0000-0000-0000-000000000001','YVR',NOW()-interval '5 days')`); err != nil { + t.Fatal(err) + } + q := sqlc.New(conn) + params := sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: time.Now().Add(-72 * time.Hour), Valid: true}, BatchSize: 1000} + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("must skip in-flight reception: %d %v", n, err) + } + if err := ingest.Commit(ctx); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 1 { + t.Fatalf("retry after ingestion: %d %v", n, err) + } + var count int + if err := conn.QueryRow(ctx, "SELECT observation_count FROM analytics_hourly_iata_stats").Scan(&count); err != nil || count != 1 { + t.Fatalf("committed reception not archived: %d %v", count, err) + } + // A second cleanup runner must wait before touching shared aggregate keys. + first, err := writer.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer first.Rollback(context.Background()) + if _, err := sqlc.New(first).DeleteOldPackets(ctx, params); err != nil { + t.Fatal(err) + } + if _, err := conn.Exec(ctx, "SET statement_timeout='300ms'"); err != nil { + t.Fatal(err) + } + if _, err := q.DeleteOldPackets(ctx, params); err == nil { + t.Fatalf("overlapping cleanup did not wait for the first transaction: %v", err) + } + if _, err := conn.Exec(ctx, "SET statement_timeout=0"); err != nil { + t.Fatal(err) + } + if err := first.Commit(ctx); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("cleanup lock was not released: %d %v", n, err) + } +} + +// Minimal real tables keep this regression runnable against an empty CI database. +func analyticsTables(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + isolateStatsSchema(t, ctx, tx) + _, err := tx.Exec(ctx, ` +CREATE TABLE packets (packet_hash bytea PRIMARY KEY, payload_type smallint, payload_version smallint, + route_type smallint, raw_payload bytea, raw_header bytea, origin_pubkey bytea, + first_heard_at timestamptz, last_heard_at timestamptz); +CREATE INDEX ON packets(last_heard_at); +CREATE TABLE packet_observations (id bigserial PRIMARY KEY, packet_hash bytea REFERENCES packets ON DELETE CASCADE, + observer_id uuid NOT NULL, iata char(3) NOT NULL, heard_at timestamptz NOT NULL, + path_length_byte smallint, hash_size smallint, hop_count smallint, path_bytes bytea, + snr real, rssi smallint, airtime_ms real, payload_type smallint, UNIQUE(packet_hash,observer_id)); +CREATE TABLE observers (id uuid PRIMARY KEY, public_key bytea, display_name text, observer_type text); +CREATE TABLE nodes (id uuid PRIMARY KEY, public_key bytea UNIQUE, node_type smallint, name text); +CREATE TABLE channel_messages (id bigserial PRIMARY KEY, channel_id integer, packet_hash bytea UNIQUE REFERENCES packets ON DELETE CASCADE, + sender_name text, content text, sent_at timestamptz); +`) + if err != nil { + t.Fatal(err) + } +} + +func TestAnalyticsRetentionPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + _, err := tx.Exec(ctx, ` + INSERT INTO observers (id,public_key,display_name,observer_type) SELECT ('00000000-0000-0000-0000-00000000000'||i)::uuid,int4send(i),CASE WHEN i=1 THEN NULL ELSE 'Observer '||i END,CASE WHEN i=1 THEN NULL ELSE 'test' END FROM generate_series(1,3) i; + INSERT INTO nodes (id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2,'Repeater'); + INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) + SELECT int4send(i),4,0,CASE WHEN i%2=0 THEN 1 ELSE 2 END,'\x00','\x00','\x02',date_trunc('hour',NOW())-interval '5 days', + CASE WHEN i=4 THEN NOW() ELSE date_trunc('hour',NOW())-interval '5 days' END FROM generate_series(1,4) i; + INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,airtime_ms,payload_type) + SELECT packet_hash,('00000000-0000-0000-0000-00000000000'||n)::uuid,iata,first_heard_at+make_interval(secs=>n),0,1,0, + CASE WHEN n=1 THEN 0 ELSE NULL END,CASE WHEN n=1 THEN -100 ELSE NULL END,1.5,4 + FROM packets CROSS JOIN (VALUES ('YVR',1),('YVR',2),('YYZ',3)) v(iata,n); + INSERT INTO channel_messages (channel_id,packet_hash,sender_name,content,sent_at) + SELECT 1,packet_hash,'Sender','body that must expire',first_heard_at FROM packets; + CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS SELECT iata,date_trunc('hour',heard_at) AS hour,count(*) AS observation_count,count(DISTINCT packet_hash) AS unique_packets,count(DISTINCT observer_id) AS active_observers FROM packet_observations GROUP BY 1,2; + `) + if err != nil { + t.Fatal(err) + } + for _, migration := range []string{"016_mv_payload_breakdown.sql", "017_mv_top_observers.sql", "018_mv_top_talkers.sql", "019_mv_top_advertisers.sql", "021_mv_top_advertisers_route_type.sql", "032_mv_observer_activity.sql", "035_mv_signal_stats.sql", "036_mv_path_stats.sql"} { + applyStatsMigration(t, ctx, tx, migration) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + for _, name := range []string{"idx_mv_signal_stats_hourly_hour", "idx_mv_path_stats_hourly_hour"} { + var definition string + if err := tx.QueryRow(ctx, "SELECT pg_get_indexdef(to_regclass($1))", name).Scan(&definition); err != nil || !strings.Contains(definition, "(hour)") { + t.Fatalf("missing unfiltered time index %s: %s %v", name, definition, err) + } + } + before := analyticsSnapshot(t, ctx, tx) + q := sqlc.New(tx) + cutoff := time.Now().Add(-72 * time.Hour) + // Failure in one archive must roll back every archive write and the raw deletion. + if _, err := tx.Exec(ctx, `SAVEPOINT archive_failure; ALTER TABLE analytics_signal_stats_hourly ADD CONSTRAINT fail_archive CHECK(receptions < 0) NOT VALID`); err != nil { + t.Fatal(err) + } + if _, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}); err == nil { + t.Fatal("expected archive failure") + } + if _, err := tx.Exec(ctx, "ROLLBACK TO archive_failure"); err != nil { + t.Fatal(err) + } + if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 4 { + t.Fatal("archive failure deleted raw packets") + } + for i := 0; i < 3; i++ { + n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}) + if err != nil || n != 1 { + t.Fatalf("delete batch: %d %v", n, err) + } + } + if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 1 || countRows(t, ctx, tx, "SELECT count(*) FROM channel_messages") != 1 { + t.Fatal("raw packets/messages did not expire") + } + after := analyticsSnapshot(t, ctx, tx) + for _, view := range retainedViews { + if after[view] != before[view] { + t.Errorf("%s lost or double-counted history after raw expiry: before=%s after=%s", view, before[view], after[view]) + } + } + // Retry and migration journal retry preserve the exact summaries. + if n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1000}); err != nil || n != 0 { + t.Fatalf("retry: %d %v", n, err) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != before[view] { + t.Errorf("retry changed %s", view) + } + } + // Later receptions on a still-live packet must join its old bucket before expiry. + if _, err := tx.Exec(ctx, `INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000004'); INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,payload_type) + SELECT packet_hash,'00000000-0000-0000-0000-000000000004','YVR',first_heard_at,0,1,0,'NaN',-80,NULL FROM packets; + UPDATE packets SET last_heard_at=first_heard_at`); err != nil { + t.Fatal(err) + } + late := analyticsSnapshot(t, ctx, tx) + if countRows(t, ctx, tx, "SELECT COALESCE(SUM(observations),0) FROM mv_observer_activity_hourly WHERE payload_type=-1") != 1 { + t.Fatal("unknown-payload observation was omitted") + } + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != late[view] { + t.Errorf("late reception changed %s: %s != %s", view, rows, late[view]) + } + } + // Archived labels follow a current rename; no entity FK may erase retained history. + if _, err := tx.Exec(ctx, `UPDATE nodes SET name='Renamed'; UPDATE observers SET display_name=NULL,observer_type=NULL;`); err != nil { + t.Fatal(err) + } + analyticsSnapshot(t, ctx, tx) + if countRows(t, ctx, tx, "SELECT count(*) FROM mv_top_advertisers_by_iata WHERE name='Renamed'") != 2 { + t.Fatal("archive name did not follow rename") + } + if _, err := q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{Column1: pgtype.Interval{Microseconds: int64(30 * 24 * time.Hour / time.Microsecond), Valid: true}, Limit: 10}); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, "DELETE FROM nodes; DELETE FROM observers"); err != nil { + t.Fatal(err) + } + if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_top_advertisers_by_iata") != 2 { + t.Fatal("entity deletion erased archive") + } + // Independent expiry runs even when there are no packets left to delete. + for _, view := range retainedViews { + table := "analytics_" + view[3:] + at := "bucket" + if view == "mv_hourly_iata_stats" || view == "mv_signal_stats_hourly" || view == "mv_path_stats_hourly" { + at = "hour" + } + if _, err := tx.Exec(ctx, "UPDATE "+table+" SET "+at+"="+at+"-interval '31 days'"); err != nil { + t.Fatal(err) + } + } + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + for _, view := range retainedViews { + if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) != 0 { + t.Errorf("%s archive did not expire", view) + } + } + // More than one populated cohort, with shared hourly keys across the boundary. + if _, err := tx.Exec(ctx, ` +INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000001'); +INSERT INTO nodes (id,public_key,node_type) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2); +INSERT INTO packets (packet_hash,payload_type,route_type,origin_pubkey,first_heard_at,last_heard_at,raw_payload) +SELECT int4send(i),4,1,'\x02',date_trunc('hour',NOW())-interval '5 days',date_trunc('hour',NOW())-interval '5 days',decode(repeat('ab',1024),'hex') FROM generate_series(10000,11000) i; +INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) +SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YVR',first_heard_at,0,1,0,4,0,-100 FROM packets; +INSERT INTO channel_messages (packet_hash,sender_name,sent_at) SELECT packet_hash,'Sender',first_heard_at FROM packets; +`); err != nil { + t.Fatal(err) + } + populated := analyticsSnapshot(t, ctx, tx) + started := time.Now() + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + t.Logf("1001 packets with bodies and observations archived/deleted in %s", time.Since(started)) + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != populated[view] { + t.Errorf("batch boundary changed %s", view) + } + } + var archiveRows int + for _, view := range retainedViews { + archiveRows += countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) + } + if archiveRows > 12 { + t.Fatalf("expected compact rollups, got %d archive rows for 1001 packets", archiveRows) + } +} diff --git a/db/migrations/039_analytics_retention.sql b/db/migrations/039_analytics_retention.sql new file mode 100644 index 00000000..069204ad --- /dev/null +++ b/db/migrations/039_analytics_retention.sql @@ -0,0 +1,613 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Archive only aggregates of deleted packet cohorts, never packet/message bodies +-- or raw paths. Live and archived cohorts are disjoint in every refresh snapshot. +-- Thirty days matches the longest analytics window; raw retention is independent. + + +CREATE TABLE IF NOT EXISTS analytics_hourly_iata_stats ( + iata char(3), + hour timestamptz, + observation_count bigint NOT NULL DEFAULT 0, + unique_packets bigint NOT NULL DEFAULT 0, + PRIMARY KEY (iata,hour) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_hourly_iata_stats_time ON analytics_hourly_iata_stats (hour); + +CREATE TABLE IF NOT EXISTS analytics_payload_breakdown_by_iata ( + iata char(3), + payload_type smallint, + bucket timestamptz, + count bigint NOT NULL DEFAULT 0, + PRIMARY KEY (iata,payload_type,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_payload_breakdown_by_iata_time ON analytics_payload_breakdown_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_observers_by_iata ( + iata char(3), + observer_id uuid, + bucket timestamptz, + observation_count bigint NOT NULL DEFAULT 0, + display_name text, + observer_type text, + PRIMARY KEY (iata,observer_id,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_observers_by_iata_time ON analytics_top_observers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_talkers_by_iata ( + iata char(3), + sender_name text, + bucket timestamptz, + message_count bigint NOT NULL DEFAULT 0, + last_sent timestamptz, + PRIMARY KEY (iata,sender_name,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_talkers_by_iata_time ON analytics_top_talkers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_advertisers_by_iata ( + iata char(3), + node_id uuid, + bucket timestamptz, + advert_count bigint NOT NULL DEFAULT 0, + flood_advert_count bigint NOT NULL DEFAULT 0, + direct_advert_count bigint NOT NULL DEFAULT 0, + last_heard timestamptz, + name text, + node_type smallint, + PRIMARY KEY (iata,node_id,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_advertisers_by_iata_time ON analytics_top_advertisers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_observer_activity_hourly ( + observer_id uuid, + payload_type smallint, + bucket timestamptz, + observations bigint NOT NULL DEFAULT 0, + airtime_ms real, + airtime_n bigint NOT NULL DEFAULT 0, + snr_sum real, + snr_n bigint NOT NULL DEFAULT 0, + snr_min real, + rssi_sum bigint, + rssi_n bigint NOT NULL DEFAULT 0, + PRIMARY KEY (observer_id,payload_type,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_observer_activity_hourly_time ON analytics_observer_activity_hourly (bucket); + +CREATE TABLE IF NOT EXISTS analytics_signal_stats_hourly ( + iata char(3), + hour timestamptz, + kind integer, + snr_bin integer, + rssi_bin integer, + receptions bigint NOT NULL DEFAULT 0, + snr_samples bigint NOT NULL DEFAULT 0, + snr_sum double precision, + rssi_samples bigint NOT NULL DEFAULT 0, + rssi_sum double precision, + PRIMARY KEY (iata,hour,kind,snr_bin,rssi_bin) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_signal_stats_hourly_time ON analytics_signal_stats_hourly (hour); + +CREATE TABLE IF NOT EXISTS analytics_path_stats_hourly ( + iata char(3), + hour timestamptz, + category integer, + hash_bytes integer, + entries integer, + receptions bigint NOT NULL DEFAULT 0, + PRIMARY KEY (iata,hour,category,hash_bytes,entries) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_path_stats_hourly_time ON analytics_path_stats_hourly (hour); + +DROP MATERIALIZED VIEW IF EXISTS mv_hourly_iata_stats; + +CREATE OR REPLACE VIEW analytics_live_hourly_iata_stats AS +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS +WITH combined AS ( + SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats + UNION ALL + SELECT iata, hour, observation_count, unique_packets FROM analytics_hourly_iata_stats + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observers_by_hour AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, observer_id + FROM packet_observations WHERE heard_at > NOW() - INTERVAL '30 days' + UNION + SELECT iata, bucket AS hour, observer_id FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observer_counts AS (SELECT iata, hour, COUNT(*) AS n FROM observers_by_hour GROUP BY iata,hour) +SELECT combined.iata, combined.hour, SUM(observation_count)::bigint AS observation_count, SUM(unique_packets)::bigint AS unique_packets, MAX(o.n)::bigint AS active_observers +FROM combined JOIN observer_counts o USING (iata,hour) GROUP BY combined.iata,combined.hour; + +CREATE UNIQUE INDEX idx_analytics_hourly_iata_stats_view ON mv_hourly_iata_stats (iata,hour); + + +DROP MATERIALIZED VIEW IF EXISTS mv_payload_breakdown_by_iata; + +CREATE OR REPLACE VIEW analytics_live_payload_breakdown_by_iata AS +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS +WITH combined AS ( + SELECT iata, payload_type, bucket, count FROM analytics_live_payload_breakdown_by_iata + UNION ALL + SELECT iata, payload_type, bucket, count FROM analytics_payload_breakdown_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, payload_type, bucket, SUM(count)::bigint AS count +FROM combined GROUP BY iata,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_payload_breakdown_by_iata_view ON mv_payload_breakdown_by_iata (iata,payload_type,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_observers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_observers_by_iata AS +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS +WITH combined AS ( + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_live_top_observers_by_iata + UNION ALL + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, observer_id, bucket, SUM(observation_count)::bigint AS observation_count, MAX(display_name) AS display_name, MAX(observer_type) AS observer_type +FROM combined GROUP BY iata,observer_id,bucket) +SELECT t.iata, t.observer_id, COALESCE(o.display_name,t.display_name) AS display_name, + COALESCE(o.observer_type,t.observer_type) AS observer_type, t.bucket, t.observation_count +FROM totals t LEFT JOIN observers o ON o.id=t.observer_id; + +CREATE UNIQUE INDEX idx_analytics_top_observers_by_iata_view ON mv_top_observers_by_iata (iata,observer_id,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_talkers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_talkers_by_iata AS +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN packet_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS +WITH combined AS ( + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_live_top_talkers_by_iata + UNION ALL + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_top_talkers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, sender_name, bucket, SUM(message_count)::bigint AS message_count, MAX(last_sent) AS last_sent +FROM combined GROUP BY iata,sender_name,bucket; + +CREATE UNIQUE INDEX idx_analytics_top_talkers_by_iata_view ON mv_top_talkers_by_iata (iata,sender_name,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_advertisers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_advertisers_by_iata AS +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN packet_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS +WITH combined AS ( + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_live_top_advertisers_by_iata + UNION ALL + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_top_advertisers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, node_id, bucket, SUM(advert_count)::bigint AS advert_count, SUM(flood_advert_count)::bigint AS flood_advert_count, SUM(direct_advert_count)::bigint AS direct_advert_count, MAX(last_heard) AS last_heard, MAX(name) AS name, MAX(node_type) AS node_type +FROM combined GROUP BY iata,node_id,bucket) +SELECT t.iata, t.node_id, COALESCE(n.name,t.name) AS name, COALESCE(n.node_type,t.node_type) AS node_type, + t.bucket, t.advert_count, t.flood_advert_count, t.direct_advert_count, t.last_heard +FROM totals t LEFT JOIN nodes n ON n.id=t.node_id; + +CREATE UNIQUE INDEX idx_analytics_top_advertisers_by_iata_view ON mv_top_advertisers_by_iata (iata,node_id,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_observer_activity_hourly; +CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS +WITH combined AS ( + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly + UNION ALL + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n +FROM combined GROUP BY observer_id,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); + + + +DROP MATERIALIZED VIEW IF EXISTS mv_signal_stats_hourly; + +CREATE OR REPLACE VIEW analytics_live_signal_stats_hourly AS +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); + +CREATE MATERIALIZED VIEW mv_signal_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_live_signal_stats_hourly + UNION ALL + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_signal_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, kind, snr_bin, rssi_bin, SUM(receptions)::bigint AS receptions, SUM(snr_samples)::bigint AS snr_samples, SUM(snr_sum)::double precision AS snr_sum, SUM(rssi_samples)::bigint AS rssi_samples, SUM(rssi_sum)::double precision AS rssi_sum +FROM combined GROUP BY iata,hour,kind,snr_bin,rssi_bin; + +CREATE UNIQUE INDEX idx_analytics_signal_stats_hourly_view ON mv_signal_stats_hourly (iata,hour,kind,snr_bin,rssi_bin); +CREATE INDEX idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly (hour); + + +DROP MATERIALIZED VIEW IF EXISTS mv_path_stats_hourly; + +CREATE OR REPLACE VIEW analytics_live_path_stats_hourly AS +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; + +CREATE MATERIALIZED VIEW mv_path_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_live_path_stats_hourly + UNION ALL + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_path_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, category, hash_bytes, entries, SUM(receptions)::bigint AS receptions +FROM combined GROUP BY iata,hour,category,hash_bytes,entries; + +CREATE UNIQUE INDEX idx_analytics_path_stats_hourly_view ON mv_path_stats_hourly (iata,hour,category,hash_bytes,entries); +CREATE INDEX idx_mv_path_stats_hourly_hour ON mv_path_stats_hourly (hour); + + +-- VOLATILE gives the archive statement a new READ COMMITTED snapshot after +-- acquiring parent locks. FK inserts cannot race between that snapshot and the +-- cascade. SKIP LOCKED leaves packets being ingested for a later cleanup tick. +CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) +RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ +DECLARE hashes bytea[]; deleted bigint; +BEGIN + -- Serialize cleanup runners before overlapping aggregate upserts. + PERFORM pg_advisory_xact_lock(hashtext('beacon.archive_delete_packets')); + IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; + SELECT array_agg(p.packet_hash) INTO hashes FROM ( + SELECT ep.packet_hash FROM packets ep + WHERE ep.last_heard_at < cutoff + ORDER BY ep.last_heard_at, ep.packet_hash + LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED + ) p; + + WITH expired_observations AS MATERIALIZED ( + SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) + ), +archived_hourly_iata_stats AS ( + INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) + SELECT iata, hour, observation_count, unique_packets FROM ( +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,hour) DO UPDATE SET + observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, + unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets +), +archived_payload_breakdown_by_iata AS ( + INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) + SELECT iata, payload_type, bucket, count FROM ( +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET + count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count +), +archived_top_observers_by_iata AS ( + INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM expired_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET + observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, + display_name = EXCLUDED.display_name, + observer_type = EXCLUDED.observer_type +), +archived_top_talkers_by_iata AS ( + INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) + SELECT iata, sender_name, bucket, message_count, last_sent FROM ( +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN expired_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') + ) batch + ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET + message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, + last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) +), +archived_top_advertisers_by_iata AS ( + INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN expired_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,node_id,bucket) DO UPDATE SET + advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, + flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, + direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, + last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), + name = EXCLUDED.name, + node_type = EXCLUDED.node_type +), +archived_observer_activity_hourly AS ( + INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET + observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, + airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, + airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, + snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, + snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, + snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), + rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, + rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n +), +archived_signal_stats_hourly AS ( + INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) + ) batch + ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET + receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, + snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, + snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, + rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, + rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum +), +archived_path_stats_hourly AS ( + INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) + SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries + ) batch + ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET + receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions +) + DELETE FROM packets WHERE packet_hash = ANY(hashes); + GET DIAGNOSTICS deleted = ROW_COUNT; + + -- Once per completed cleanup, including when there were no raw packets to delete. + IF deleted < batch_size THEN + DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + END IF; + RETURN deleted; +END $$; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index ccc8492f..29690a16 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -661,18 +661,9 @@ ORDER BY po.id ASC LIMIT $6; --- name: DeleteOldPackets :execrows --- One batch of expired packets; observations and channel messages cascade. -WITH expired AS ( - SELECT ep.packet_hash - FROM packets ep - WHERE ep.last_heard_at < @cutoff - ORDER BY ep.last_heard_at - LIMIT @batch_size - FOR UPDATE OF ep SKIP LOCKED -) -DELETE FROM packets p USING expired e -WHERE p.packet_hash = e.packet_hash; +-- name: DeleteOldPackets :one +-- Locks, archives and cascades one bounded packet cohort in a single transaction. +SELECT archive_delete_packets(@cutoff::timestamptz, @batch_size::integer)::bigint; -- name: DeleteOldNodes :exec -- Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- diff --git a/db/retention_integration_test.go b/db/retention_integration_test.go index 5e40dbd2..d6307176 100644 --- a/db/retention_integration_test.go +++ b/db/retention_integration_test.go @@ -46,10 +46,9 @@ func countRows(t *testing.T, ctx context.Context, tx pgx.Tx, sql string) int { // More than one batch of old packets; observations go with them. func TestDeleteOldPacketsBatchesPostgres(t *testing.T) { ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") _, err := tx.Exec(ctx, ` -CREATE TEMP TABLE packets (LIKE public.packets INCLUDING ALL) ON COMMIT DROP; -CREATE TEMP TABLE packet_observations (LIKE public.packet_observations INCLUDING ALL) ON COMMIT DROP; -ALTER TABLE packet_observations ADD FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) SELECT int4send(i), 4, 0, 1, '\x00', '\x00', CASE WHEN i <= 2500 THEN '2026-01-01'::timestamptz ELSE '2026-02-01'::timestamptz END, diff --git a/db/sqlc/models.go b/db/sqlc/models.go index b5ffb054..8a24d538 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -16,6 +16,164 @@ type Account struct { DeactivatedAt pgtype.Timestamptz `json:"deactivated_at"` } +type AnalyticsHourlyIataStat struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount int64 `json:"observation_count"` + UniquePackets int64 `json:"unique_packets"` +} + +type AnalyticsLiveHourlyIataStat struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount int64 `json:"observation_count"` + UniquePackets int64 `json:"unique_packets"` +} + +type AnalyticsLiveObserverActivityHourly struct { + ObserverID uuid.UUID `json:"observer_id"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin float32 `json:"snr_min"` + RssiSum int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + +type AnalyticsLivePathStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +type AnalyticsLivePayloadBreakdownByIatum struct { + Iata string `json:"iata"` + PayloadType *int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Count int64 `json:"count"` +} + +type AnalyticsLiveSignalStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrSum float64 `json:"snr_sum"` + RssiSamples int64 `json:"rssi_samples"` + RssiSum float64 `json:"rssi_sum"` +} + +type AnalyticsLiveTopAdvertisersByIatum struct { + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + NodeType int16 `json:"node_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` + LastHeard interface{} `json:"last_heard"` +} + +type AnalyticsLiveTopObserversByIatum struct { + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + ObservationCount int64 `json:"observation_count"` +} + +type AnalyticsLiveTopTalkersByIatum struct { + Iata string `json:"iata"` + SenderName *string `json:"sender_name"` + Bucket pgtype.Timestamptz `json:"bucket"` + MessageCount int64 `json:"message_count"` + LastSent interface{} `json:"last_sent"` +} + +type AnalyticsObserverActivityHourly struct { + ObserverID uuid.UUID `json:"observer_id"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs *float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum *float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin *float32 `json:"snr_min"` + RssiSum *int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + +type AnalyticsPathStatsHourly struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +type AnalyticsPayloadBreakdownByIatum struct { + Iata string `json:"iata"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Count int64 `json:"count"` +} + +type AnalyticsSignalStatsHourly struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrSum *float64 `json:"snr_sum"` + RssiSamples int64 `json:"rssi_samples"` + RssiSum *float64 `json:"rssi_sum"` +} + +type AnalyticsTopAdvertisersByIatum struct { + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Bucket pgtype.Timestamptz `json:"bucket"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` + LastHeard pgtype.Timestamptz `json:"last_heard"` + Name *string `json:"name"` + NodeType *int16 `json:"node_type"` +} + +type AnalyticsTopObserversByIatum struct { + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + Bucket pgtype.Timestamptz `json:"bucket"` + ObservationCount int64 `json:"observation_count"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` +} + +type AnalyticsTopTalkersByIatum struct { + Iata string `json:"iata"` + SenderName string `json:"sender_name"` + Bucket pgtype.Timestamptz `json:"bucket"` + MessageCount int64 `json:"message_count"` + LastSent pgtype.Timestamptz `json:"last_sent"` +} + type Channel struct { ID int32 `json:"id"` ChannelHash []byte `json:"channel_hash"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 1547c1c9..38e6c971 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -29,7 +29,7 @@ type Querier interface { // Opt-in age-out: preserve retained history and manually recorded ownership. // Bound deletions per cleanup tick and skip observers being updated by ingest. DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) - // One batch of expired packets; observations and channel messages cascade. + // Locks, archives and cascades one bounded packet cohort in a single transaction. DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) // One batch of routes past retention, or past grace with too few observations. // GREATEST keeps the scan on idx_known_routes_last_seen. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index c047b44f..6a0eee13 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -124,17 +124,8 @@ func (q *Queries) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timest return items, nil } -const deleteOldPackets = `-- name: DeleteOldPackets :execrows -WITH expired AS ( - SELECT ep.packet_hash - FROM packets ep - WHERE ep.last_heard_at < $1 - ORDER BY ep.last_heard_at - LIMIT $2 - FOR UPDATE OF ep SKIP LOCKED -) -DELETE FROM packets p USING expired e -WHERE p.packet_hash = e.packet_hash +const deleteOldPackets = `-- name: DeleteOldPackets :one +SELECT archive_delete_packets($1::timestamptz, $2::integer)::bigint ` type DeleteOldPacketsParams struct { @@ -142,13 +133,12 @@ type DeleteOldPacketsParams struct { BatchSize int32 `json:"batch_size"` } -// One batch of expired packets; observations and channel messages cascade. +// Locks, archives and cascades one bounded packet cohort in a single transaction. func (q *Queries) DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) { - result, err := q.db.Exec(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) - if err != nil { - return 0, err - } - return result.RowsAffected(), nil + row := q.db.QueryRow(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err } const deleteOldRoutes = `-- name: DeleteOldRoutes :execrows diff --git a/sqlc.yaml b/sqlc.yaml index 79431093..b5e86b89 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -22,3 +22,14 @@ sql: go_type: "encoding/json.RawMessage" - db_type: "bpchar" go_type: "string" + + # Coalesced archive labels can still be NULL; preserve the API pointers. + - column: "mv_top_observers_by_iata.display_name" + go_type: {type: "string", pointer: true} + - column: "mv_top_observers_by_iata.observer_type" + go_type: {type: "string", pointer: true} + - column: "mv_top_advertisers_by_iata.name" + go_type: {type: "string", pointer: true} + + - column: "mv_observer_activity_hourly.payload_type" + go_type: {type: "int16", pointer: true} From 5fbba24a4a8a3d97c87c4a88b60bbeacf7bbc68f Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 21:59:48 -0400 Subject: [PATCH 059/141] fix(observers): separate packet metrics from presence updates --- .github/workflows/ci.yml | 2 +- README.md | 18 ++ .../040_observer_unknown_activity.sql | 279 ++++++++++++++++++ db/observer_metrics_integration_test.go | 87 ++++++ db/observer_retention_integration_test.go | 4 +- db/observers.go | 62 ++-- db/observers_test.go | 35 ++- db/presence.go | 11 +- db/queries/observer_activity_summary.sql | 13 + db/queries/queries.sql | 19 +- db/sqlc/mock/querier.go | 17 +- db/sqlc/observer_activity_summary.sql.go | 52 ++++ db/sqlc/querier.go | 2 + db/sqlc/queries.sql.go | 52 +++- docs/docs.go | 46 +++ docs/swagger.json | 46 +++ docs/swagger.yaml | 33 +++ internal/api/handlers/observers.go | 13 +- internal/api/handlers/observers_test.go | 20 +- internal/api/handlers/stub_reader_test.go | 6 +- internal/api/observers.go | 19 +- internal/api/reader.go | 2 +- internal/cache/cache_test.go | 2 +- internal/cache/reader.go | 6 +- internal/ingest/ingest.go | 2 +- internal/ingest/ingest_test.go | 6 +- internal/ingest/neighbors.go | 2 +- internal/ingest/packet.go | 2 +- internal/ingest/status.go | 2 +- internal/presence/broker_freshness_test.go | 50 ++++ internal/presence/coalescer.go | 27 +- 31 files changed, 844 insertions(+), 93 deletions(-) create mode 100644 db/migrations/040_observer_unknown_activity.sql create mode 100644 db/observer_metrics_integration_test.go create mode 100644 db/queries/observer_activity_summary.sql create mode 100644 db/sqlc/observer_activity_summary.sql.go create mode 100644 internal/presence/broker_freshness_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d374920..c9d71abe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index 823c04be..5f539b53 100644 --- a/README.md +++ b/README.md @@ -550,3 +550,21 @@ sub-hour observer activity, exact observer comparison and current entity/scope counts continue to describe retained raw data or current entities, rather than claiming archived packet detail. Archived summaries expire without waiting for new packet deletions. A failed archive leaves its entire raw batch intact. + +### Observer monitoring metrics + +Observer activity returns complete buckets in `[windowStart, windowEnd)`, plus +`generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the +sum of stored observations in those buckets; repeated broker delivery of the +same retained packet/observer pair counts once. Unknown payload types appear as +`-1` rather than disappearing. `lastCompleteHour` uses the previous complete UTC +hour and includes its own start/end; `latestRecordedAt` is the latest retained +reception timestamp. Missing records do not prove downtime. The optional `until` +(epoch milliseconds, within the last 30 days) aligns two observers' charts. + +The existing observer `observationCount` remains a legacy cumulative presence +counter for compatibility, including status/neighbour events. It is not a +period packet total. Broker presence and packet-arrival timestamps are now +updated separately; this cannot reconstruct previously overwritten timestamps. +Migration 040 repairs archived unknown-type counts from the all-payload observer +rollup; radio samples already discarded for those legacy rows remain unknown. diff --git a/db/migrations/040_observer_unknown_activity.sql b/db/migrations/040_observer_unknown_activity.sql new file mode 100644 index 00000000..6afab7e7 --- /dev/null +++ b/db/migrations/040_observer_unknown_activity.sql @@ -0,0 +1,279 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Unknown legacy payloads are still recorded packets. Restore their archived +-- counts from the all-payload observer rollup; their discarded radio samples +-- cannot be recovered. Subtract all existing activity (including prior repairs) +-- so migration-journal retries do not double-count or erase newer samples. +INSERT INTO analytics_observer_activity_hourly +(observer_id,payload_type,bucket,observations,airtime_n,snr_n,rssi_n) +SELECT totals.observer_id, -1, totals.bucket, totals.n-COALESCE(activity.n,0),0,0,0 +FROM (SELECT observer_id,bucket,SUM(observation_count)::bigint AS n + FROM analytics_top_observers_by_iata GROUP BY observer_id,bucket) totals +LEFT JOIN (SELECT observer_id,bucket,SUM(observations)::bigint AS n + FROM analytics_observer_activity_hourly GROUP BY observer_id,bucket) activity +USING (observer_id,bucket) +WHERE totals.n>COALESCE(activity.n,0) +ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE +SET observations=analytics_observer_activity_hourly.observations+EXCLUDED.observations; + +DROP MATERIALIZED VIEW mv_observer_activity_hourly; +CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS +WITH combined AS ( + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly + UNION ALL + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n +FROM combined GROUP BY observer_id,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); + + + +-- Preserve the bounded atomic archive, including unknown payload activity. +CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) +RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ +DECLARE hashes bytea[]; deleted bigint; +BEGIN + IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; + SELECT array_agg(p.packet_hash) INTO hashes FROM ( + SELECT ep.packet_hash FROM packets ep + WHERE ep.last_heard_at < cutoff + ORDER BY ep.last_heard_at, ep.packet_hash + LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED + ) p; + + WITH expired_observations AS MATERIALIZED ( + SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) + ), +archived_hourly_iata_stats AS ( + INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) + SELECT iata, hour, observation_count, unique_packets FROM ( +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,hour) DO UPDATE SET + observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, + unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets +), +archived_payload_breakdown_by_iata AS ( + INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) + SELECT iata, payload_type, bucket, count FROM ( +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET + count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count +), +archived_top_observers_by_iata AS ( + INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM expired_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET + observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, + display_name = EXCLUDED.display_name, + observer_type = EXCLUDED.observer_type +), +archived_top_talkers_by_iata AS ( + INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) + SELECT iata, sender_name, bucket, message_count, last_sent FROM ( +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN expired_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') + ) batch + ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET + message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, + last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) +), +archived_top_advertisers_by_iata AS ( + INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN expired_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,node_id,bucket) DO UPDATE SET + advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, + flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, + direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, + last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), + name = EXCLUDED.name, + node_type = EXCLUDED.node_type +), +archived_observer_activity_hourly AS ( + INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET + observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, + airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, + airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, + snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, + snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, + snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), + rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, + rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n +), +archived_signal_stats_hourly AS ( + INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) + ) batch + ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET + receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, + snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, + snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, + rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, + rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum +), +archived_path_stats_hourly AS ( + INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) + SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries + ) batch + ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET + receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions +) + DELETE FROM packets WHERE packet_hash = ANY(hashes); + GET DIAGNOSTICS deleted = ROW_COUNT; + + -- Once per completed cleanup, including when there were no raw packets to delete. + IF deleted < batch_size THEN + DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + END IF; + RETURN deleted; +END $$; diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go new file mode 100644 index 00000000..868ce6c0 --- /dev/null +++ b/db/observer_metrics_integration_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "strings" + "testing" + "time" +) + +func TestObserverMetricsPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + // Use the real observer DDL without depending on a preinstalled public schema. + initial, err := migrationFiles.ReadFile("migrations/001_initial_schema.sql") + if err != nil { + t.Fatal(err) + } + ddl := string(initial) + start := strings.Index(ddl, "CREATE TABLE observers (") + end := strings.Index(ddl[start:], "\n);") + start + 3 + if _, err := tx.Exec(ctx, "DROP TABLE observers; "+ddl[start:end]+" ALTER TABLE observers ADD COLUMN region_scope TEXT;"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, ` + CREATE TABLE observer_brokers (observer_id uuid,broker_name text,last_seen timestamptz,last_packet_at timestamptz,PRIMARY KEY(observer_id,broker_name)); + INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); + INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,0,-100 FROM packets; + `); err != nil { + t.Fatal(err) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + store := &Store{q: sqlc.New(tx)} + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { + t.Fatal(err) + } + // Migration repairs the two unknown counts that 039 did not archive. + applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") + applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") + until := time.Now().UTC().Truncate(time.Hour) + activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until) + if err != nil { + t.Fatal(err) + } + if activity.Summary.RecordedPackets != 3 || len(activity.PayloadTypes) != 2 || activity.Summary.LatestRecordedAt != nil || activity.Summary.LastCompleteHour != 0 { + t.Fatalf("archive summary: %+v types=%+v", activity.Summary, activity.PayloadTypes) + } + if activity.WindowEnd != until.UnixMilli() || activity.WindowStart != until.Add(-7*24*time.Hour).UnixMilli() || activity.Source != "hourly" { + t.Fatalf("window: %+v", activity) + } + if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,last_heard_at) VALUES ('\xaa',NOW()); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,0,-100); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW()) ON CONFLICT DO NOTHING;`); err != nil { + t.Fatal(err) + } + raw, err := store.GetObserverActivity(ctx, id, time.Hour, 15*time.Minute, until) + if err != nil { + t.Fatal(err) + } + if raw.Summary.RecordedPackets != 1 || raw.Summary.LastCompleteHour != 1 || raw.Summary.LatestRecordedAt == nil || raw.PayloadTypes[0].PayloadType != -1 { + t.Fatalf("raw/duplicate: %+v", raw) + } + if err := store.UpsertObserverBroker(ctx, id, "one", false); err != nil { + t.Fatal(err) + } + var packet pgtype.Timestamptz + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if packet.Valid { + t.Fatal("status created a packet timestamp") + } + if err := store.UpsertObserverBroker(ctx, id, "one", true); err != nil { + t.Fatal(err) + } + if err := store.TouchObserverBrokers(ctx, []uuid.UUID{id}, []string{"one"}, []time.Time{until.Add(time.Hour)}, []time.Time{{}}); err != nil { + t.Fatal(err) + } + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if !packet.Valid { + t.Fatal("status erased packet timestamp") + } +} diff --git a/db/observer_retention_integration_test.go b/db/observer_retention_integration_test.go index 8672ea42..8e64ea9c 100644 --- a/db/observer_retention_integration_test.go +++ b/db/observer_retention_integration_test.go @@ -100,7 +100,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil { t.Fatal(err) } - if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture"); err != nil { + if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture", true); err != nil { t.Fatal(err) } // A future cutoff simulates this cached observer having aged past retention. @@ -112,7 +112,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil || newID == oldID { t.Fatalf("returning observer reused a deleted ID: %v", err) } - if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture"); err != nil { + if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture", true); err != nil { t.Fatal(err) } var brokers int diff --git a/db/observers.go b/db/observers.go index a3448cf1..1735042a 100644 --- a/db/observers.go +++ b/db/observers.go @@ -230,12 +230,37 @@ func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uui // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. // Buckets of an hour or coarser come from the hourly rollup; anything finer reads observations directly. // Range and Interval are left empty for the handler to fill. -func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { +func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { obs, err := s.q.GetObserverByID(ctx, observerID) if err != nil { return nil, err } - activity := &api.ObserverActivity{} + now := time.Now().UTC() + if until.IsZero() { + until = now + } + until = until.UTC().Truncate(interval) + since := until.Add(-window) + if !since.Equal(since.Truncate(interval)) { + since = since.Truncate(interval).Add(interval) + } + activity := &api.ObserverActivity{WindowStart: since.UnixMilli(), WindowEnd: until.UnixMilli(), GeneratedAt: now.UnixMilli(), Source: "raw", Summary: &api.ObserverActivitySummary{}} + if interval >= time.Hour { + activity.Source = "hourly" + } + live, err := s.q.GetObserverActivityLiveSummary(ctx, sqlc.GetObserverActivityLiveSummaryParams{ + ObserverID: observerID, GeneratedAt: pgtype.Timestamptz{Time: now, Valid: true}, + HourStart: pgtype.Timestamptz{Time: now.Truncate(time.Hour).Add(-time.Hour), Valid: true}, HourEnd: pgtype.Timestamptz{Time: now.Truncate(time.Hour), Valid: true}}) + if err != nil { + return nil, err + } + activity.Summary.LastCompleteHour = live.LastCompleteHour + if live.LatestRecordedAt.Valid { + v := live.LatestRecordedAt.Time.UnixMilli() + activity.Summary.LatestRecordedAt = &v + } + activity.Summary.LastCompleteHourStart = now.Truncate(time.Hour).Add(-time.Hour).UnixMilli() + activity.Summary.LastCompleteHourEnd = now.Truncate(time.Hour).UnixMilli() // radio is non-nil only when airtime is actually costable, so radio != null implies costed buckets if obs.RadioSf != nil && obs.RadioBwKhz != nil && obs.RadioCr != nil && *obs.RadioSf >= 7 && *obs.RadioSf <= 12 && *obs.RadioBwKhz > 0 && *obs.RadioCr > 0 { @@ -247,12 +272,6 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w PreambleSymbols: lora.PreambleSymbols(int(*obs.RadioSf)), } } - // round the window start up to a bucket boundary so the first bucket is never a partial one - start := time.Now().Add(-window).UTC() - since := start.Truncate(interval) - if since.Before(start) { - since = since.Add(interval) - } sinceTS := pgtype.Timestamptz{Time: since, Valid: true} binWidth := pgtype.Interval{Microseconds: interval.Microseconds(), Valid: true} @@ -261,6 +280,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w ObserverID: observerID, Column2: sinceTS, Column3: binWidth, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err @@ -268,6 +288,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) for _, r := range rows { p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations if r.AirtimeN > 0 { airtime := r.AirtimeMs p.AirtimeMs = &airtime @@ -286,18 +307,20 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityHourlyPayloadTypes(ctx, sqlc.GetObserverActivityHourlyPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - if v.PayloadType == nil { - continue + payloadType := int16(-1) + if v.PayloadType != nil { + payloadType = *v.PayloadType } activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), Count: v.Count, }) } @@ -308,6 +331,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w ObserverID: observerID, Column2: sinceTS, Column3: binWidth, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err @@ -315,6 +339,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) for _, r := range rows { p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations if r.AirtimeN > 0 { airtime := r.AirtimeMs p.AirtimeMs = &airtime @@ -332,18 +357,20 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityRawPayloadTypes(ctx, sqlc.GetObserverActivityRawPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - if v.PayloadType == nil { - continue + payloadType := int16(-1) + if v.PayloadType != nil { + payloadType = *v.PayloadType } activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), Count: v.Count, }) } @@ -424,10 +451,11 @@ func (s *Store) GetObserverRadio(ctx context.Context, observerID uuid.UUID) (ing return settings, nil } -func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { params := sqlc.UpsertObserverBrokerParams{ ObserverID: observerID, BrokerName: brokerName, + IsPacket: isPacket, } return s.q.UpsertObserverBroker(ctx, params) } diff --git a/db/observers_test.go b/db/observers_test.go index 76ccf548..31b562e3 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -569,6 +569,7 @@ func f32(v float32) *float32 { return &v } func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -592,7 +593,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { }, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -615,8 +616,8 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { if p.RSSIAvg != nil { t.Errorf("expected nil RSSIAvg, got %v", *p.RSSIAvg) } - if len(got.PayloadTypes) != 1 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 { - t.Fatalf("expected one payload type 4 with count 9, got %+v", got.PayloadTypes) + if len(got.PayloadTypes) != 2 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 || got.PayloadTypes[1].PayloadType != -1 || got.PayloadTypes[1].Count != 3 { + t.Fatalf("expected known and unknown payloads to reconcile, got %+v", got.PayloadTypes) } if got.PayloadTypes[0].PayloadTypeName != api.PayloadTypeName(4) { t.Errorf("expected payload type name %q, got %q", api.PayloadTypeName(4), got.PayloadTypes[0].PayloadTypeName) @@ -638,6 +639,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -658,7 +660,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -689,6 +691,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { func TestGetObserverActivity_RawPath(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -709,7 +712,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{{PayloadType: i16(1), Count: 5}}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -737,6 +740,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) { func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -757,7 +761,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -779,6 +783,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { func TestGetObserverActivity_RawPathEmpty(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -789,7 +794,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -804,6 +809,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) { func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -814,7 +820,7 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -826,13 +832,14 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { func TestGetObserverActivity_UnknownObserver(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000002") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). Return(sqlc.Observer{}, pgx.ErrNoRows) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour, time.Time{}) if !errors.Is(err, pgx.ErrNoRows) { t.Fatalf("expected pgx.ErrNoRows, got %v", err) } @@ -858,6 +865,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { t.Run(tc.name, func(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -868,7 +876,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -882,6 +890,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") window, interval := 24*time.Hour, 6*time.Hour @@ -897,12 +906,12 @@ func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) - before := time.Now().Add(-window) + before := time.Now().UTC().Truncate(interval).Add(-window) store := &Store{q: mock} - if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval); err != nil { + if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval, time.Time{}); err != nil { t.Fatalf("unexpected error: %v", err) } - after := time.Now().Add(-window) + after := time.Now().UTC().Truncate(interval).Add(-window) if gotSince.UnixNano()%int64(interval) != 0 { t.Errorf("since %s is not aligned to %s", gotSince, interval) diff --git a/db/presence.go b/db/presence.go index ea2d6fea..41222669 100644 --- a/db/presence.go +++ b/db/presence.go @@ -20,11 +20,12 @@ func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time }) } -func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error { +func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error { return s.q.TouchObserverBrokers(ctx, sqlc.TouchObserverBrokersParams{ Column1: ids, Column2: brokers, Column3: toTimestamptzs(seen), + Column4: nullableTimes(packets), }) } @@ -42,3 +43,11 @@ func toTimestamptzs(ts []time.Time) []pgtype.Timestamptz { } return out } + +func nullableTimes(ts []time.Time) []pgtype.Timestamptz { + out := make([]pgtype.Timestamptz, len(ts)) + for i, t := range ts { + out[i] = pgtype.Timestamptz{Time: t, Valid: !t.IsZero()} + } + return out +} diff --git a/db/queries/observer_activity_summary.sql b/db/queries/observer_activity_summary.sql new file mode 100644 index 00000000..55b5aedd --- /dev/null +++ b/db/queries/observer_activity_summary.sql @@ -0,0 +1,13 @@ +-- name: GetObserverActivityLiveSummary :one +-- Two indexed ranges, bounded to one observer; no legacy presence counters. +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = @observer_id::uuid + AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 29690a16..aaf0be89 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -275,14 +275,14 @@ SELECT COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz GROUP BY bucket ORDER BY bucket; -- name: GetObserverActivityRawPayloadTypes :many SELECT payload_type, COUNT(*)::bigint AS count FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -299,14 +299,14 @@ SELECT COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz GROUP BY 1 ORDER BY 1; -- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -363,19 +363,20 @@ RETURNING o.id; -- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN @is_packet::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW(); + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at); -- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index aeae62b1..cf80ee9b 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -source=db/sqlc/querier.go -destination=db/sqlc/mock/querier.go -package=mockdb +// mockgen -source db/sqlc/querier.go -destination db/sqlc/mock/querier.go -package mockdb // // Package mockdb is a generated GoMock package. @@ -384,6 +384,21 @@ func (mr *MockQuerierMockRecorder) GetObserverActivityHourlyPayloadTypes(ctx, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityHourlyPayloadTypes", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityHourlyPayloadTypes), ctx, arg) } +// GetObserverActivityLiveSummary mocks base method. +func (m *MockQuerier) GetObserverActivityLiveSummary(ctx context.Context, arg db.GetObserverActivityLiveSummaryParams) (db.GetObserverActivityLiveSummaryRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityLiveSummary", ctx, arg) + ret0, _ := ret[0].(db.GetObserverActivityLiveSummaryRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityLiveSummary indicates an expected call of GetObserverActivityLiveSummary. +func (mr *MockQuerierMockRecorder) GetObserverActivityLiveSummary(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityLiveSummary", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityLiveSummary), ctx, arg) +} + // GetObserverActivityRaw mocks base method. func (m *MockQuerier) GetObserverActivityRaw(ctx context.Context, arg db.GetObserverActivityRawParams) ([]db.GetObserverActivityRawRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/observer_activity_summary.sql.go b/db/sqlc/observer_activity_summary.sql.go new file mode 100644 index 00000000..c6c97ac9 --- /dev/null +++ b/db/sqlc/observer_activity_summary.sql.go @@ -0,0 +1,52 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: observer_activity_summary.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = $1::uuid + AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly +` + +type GetObserverActivityLiveSummaryParams struct { + ObserverID uuid.UUID `json:"observer_id"` + GeneratedAt pgtype.Timestamptz `json:"generated_at"` + HourStart pgtype.Timestamptz `json:"hour_start"` + HourEnd pgtype.Timestamptz `json:"hour_end"` +} + +type GetObserverActivityLiveSummaryRow struct { + LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"` + LastCompleteHour int64 `json:"last_complete_hour"` +} + +// Two indexed ranges, bounded to one observer; no legacy presence counters. +func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) { + row := q.db.QueryRow(ctx, getObserverActivityLiveSummary, + arg.ObserverID, + arg.GeneratedAt, + arg.HourStart, + arg.HourEnd, + ) + var i GetObserverActivityLiveSummaryRow + err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour) + return i, err +} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 38e6c971..8f96cbe3 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -58,6 +58,8 @@ type Querier interface { // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) + // Two indexed ranges, bounded to one observer; no legacy presence counters. + GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) // Sub-hour activity buckets straight off idx_observations_observer; no join to packets. // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 6a0eee13..3dd6de88 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -650,7 +650,7 @@ SELECT COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $4::timestamptz GROUP BY 1 ORDER BY 1 ` @@ -659,6 +659,7 @@ type GetObserverActivityHourlyParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityHourlyRow struct { @@ -675,7 +676,12 @@ type GetObserverActivityHourlyRow struct { // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityHourly, arg.ObserverID, arg.Column2, arg.Column3) + rows, err := q.db.Query(ctx, getObserverActivityHourly, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + ) if err != nil { return nil, err } @@ -707,7 +713,7 @@ func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserver const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -715,6 +721,7 @@ ORDER BY count DESC type GetObserverActivityHourlyPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityHourlyPayloadTypesRow struct { @@ -723,7 +730,7 @@ type GetObserverActivityHourlyPayloadTypesRow struct { } func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2) + rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) if err != nil { return nil, err } @@ -754,7 +761,7 @@ SELECT COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $4::timestamptz GROUP BY bucket ORDER BY bucket ` @@ -763,6 +770,7 @@ type GetObserverActivityRawParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityRawRow struct { @@ -781,7 +789,12 @@ type GetObserverActivityRawRow struct { // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverActivityRawParams) ([]GetObserverActivityRawRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityRaw, arg.ObserverID, arg.Column2, arg.Column3) + rows, err := q.db.Query(ctx, getObserverActivityRaw, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + ) if err != nil { return nil, err } @@ -813,7 +826,7 @@ func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverAct const getObserverActivityRawPayloadTypes = `-- name: GetObserverActivityRawPayloadTypes :many SELECT payload_type, COUNT(*)::bigint AS count FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -821,6 +834,7 @@ ORDER BY count DESC type GetObserverActivityRawPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityRawPayloadTypesRow struct { @@ -829,7 +843,7 @@ type GetObserverActivityRawPayloadTypesRow struct { } func (q *Queries) GetObserverActivityRawPayloadTypes(ctx context.Context, arg GetObserverActivityRawPayloadTypesParams) ([]GetObserverActivityRawPayloadTypesRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2) + rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) if err != nil { return nil, err } @@ -4422,11 +4436,12 @@ func (q *Queries) SetPacketDecrypted(ctx context.Context, packetHash []byte) err const touchObserverBrokers = `-- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name ` @@ -4435,10 +4450,16 @@ type TouchObserverBrokersParams struct { Column1 []uuid.UUID `json:"column_1"` Column2 []string `json:"column_2"` Column3 []pgtype.Timestamptz `json:"column_3"` + Column4 []pgtype.Timestamptz `json:"column_4"` } func (q *Queries) TouchObserverBrokers(ctx context.Context, arg TouchObserverBrokersParams) error { - _, err := q.db.Exec(ctx, touchObserverBrokers, arg.Column1, arg.Column2, arg.Column3) + _, err := q.db.Exec(ctx, touchObserverBrokers, + arg.Column1, + arg.Column2, + arg.Column3, + arg.Column4, + ) return err } @@ -4943,22 +4964,23 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe const upsertObserverBroker = `-- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN $3::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW() + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at) ` type UpsertObserverBrokerParams struct { ObserverID uuid.UUID `json:"observer_id"` BrokerName string `json:"broker_name"` + IsPacket bool `json:"is_packet"` } // ============================================================ // OBSERVER BROKERS // ============================================================ func (q *Queries) UpsertObserverBroker(ctx context.Context, arg UpsertObserverBrokerParams) error { - _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName) + _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName, arg.IsPacket) return err } diff --git a/docs/docs.go b/docs/docs.go index 43ed843a..bc88fbf3 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1468,6 +1468,12 @@ const docTemplate = `{ "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", "name": "interval", "in": "query" + }, + { + "type": "integer", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "name": "until", + "in": "query" } ], "responses": { @@ -3924,6 +3930,7 @@ const docTemplate = `{ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -3979,6 +3986,10 @@ const docTemplate = `{ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { "type": "object", "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, "interval": { "type": "string" }, @@ -3999,6 +4010,21 @@ const docTemplate = `{ }, "range": { "type": "string" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive complete-bucket end, epoch ms", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" } } }, @@ -4046,6 +4072,26 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 8a83f4c3..75a8e5ac 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1466,6 +1466,12 @@ "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", "name": "interval", "in": "query" + }, + { + "type": "integer", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "name": "until", + "in": "query" } ], "responses": { @@ -3922,6 +3928,7 @@ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -3977,6 +3984,10 @@ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { "type": "object", "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, "interval": { "type": "string" }, @@ -3997,6 +4008,21 @@ }, "range": { "type": "string" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive complete-bucket end, epoch ms", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" } } }, @@ -4044,6 +4070,26 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f03b978b..cab6df77 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -521,6 +521,7 @@ definitions: description: epoch ms type: integer observationCount: + description: legacy cumulative presence counter; includes non-packet events type: integer observerType: description: e.g. "meshcoretomqtt", "meshcoreha" @@ -560,6 +561,9 @@ definitions: type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity: properties: + generatedAt: + description: response computation time, not proof of continuous coverage + type: integer interval: type: string payloadTypes: @@ -574,6 +578,17 @@ definitions: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio' range: type: string + source: + description: raw or hourly; missing records do not prove an outage + type: string + summary: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary' + windowEnd: + description: exclusive complete-bucket end, epoch ms + type: integer + windowStart: + description: inclusive complete-bucket start, epoch ms + type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint: properties: @@ -604,6 +619,19 @@ definitions: sf: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary: + properties: + lastCompleteHour: + type: integer + lastCompleteHourEnd: + type: integer + lastCompleteHourStart: + type: integer + latestRecordedAt: + type: integer + recordedPackets: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker: properties: lastPacketAt: @@ -2401,6 +2429,11 @@ paths: in: query name: interval type: string + - description: Optional exclusive end in epoch milliseconds; aligned down to + a complete bucket, at most 30 days old + in: query + name: until + type: integer produces: - application/json responses: diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 43147b5a..15d52d6a 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -256,6 +256,7 @@ var activityIntervals = map[string]time.Duration{ // @Param observerId path string true "Observer UUID" // @Param range query string false "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h" // @Param interval query string false "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)" +// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old" // @Success 200 {object} api.ObserverActivity // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError @@ -295,7 +296,17 @@ func getObserverActivity(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "range/interval exceeds 1000 buckets") return } - activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval) + until := time.Time{} + if values, ok := r.URL.Query()["until"]; ok { + n, err := strconv.ParseInt(values[0], 10, 64) + now := time.Now() + if len(values) != 1 || err != nil || n < 0 || n > now.UnixMilli() || n < now.Add(-720*time.Hour).UnixMilli() { + respondError(w, http.StatusBadRequest, "until must be one past epoch-millisecond timestamp within 30 days") + return + } + until = time.UnixMilli(n) + } + activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval, until) if err != nil { if errors.Is(err, pgx.ErrNoRows) { respondError(w, http.StatusNotFound, "observer not found") diff --git a/internal/api/handlers/observers_test.go b/internal/api/handlers/observers_test.go index 1bf865fa..cfef1c9c 100644 --- a/internal/api/handlers/observers_test.go +++ b/internal/api/handlers/observers_test.go @@ -200,7 +200,7 @@ func TestGetObserverActivity_Defaults(t *testing.T) { var gotWindow, gotInterval time.Duration r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -232,7 +232,7 @@ func TestGetObserverActivity_CustomRangeAndInterval(t *testing.T) { var gotWindow, gotInterval time.Duration r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -264,7 +264,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { t.Fatal("reader should not be called") return nil, nil }, @@ -286,7 +286,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { var gotWindow, gotInterval time.Duration ok := chi.NewRouter() ok.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -309,6 +309,10 @@ func TestGetObserverActivity_BadRequests(t *testing.T) { query string }{ {"invalid uuid", "not-a-uuid", ""}, + {"bad end", "00000000-0000-0000-0000-000000000001", "?until=banana"}, + {"negative end", "00000000-0000-0000-0000-000000000001", "?until=-1"}, + {"duplicate end", "00000000-0000-0000-0000-000000000001", "?until=1&until=2"}, + {"future end", "00000000-0000-0000-0000-000000000001", "?until=253402300799999"}, {"unparseable range", "00000000-0000-0000-0000-000000000001", "?range=banana"}, {"range too long", "00000000-0000-0000-0000-000000000001", "?range=721h"}, {"zero range", "00000000-0000-0000-0000-000000000001", "?range=0"}, @@ -319,7 +323,7 @@ func TestGetObserverActivity_BadRequests(t *testing.T) { t.Run(tt.name, func(t *testing.T) { r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { t.Fatal("reader should not be called") return nil, nil }, @@ -338,7 +342,7 @@ func TestGetObserverActivity_NotFound(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, pgx.ErrNoRows }, })) @@ -364,7 +368,7 @@ func TestGetObserverActivity_ReaderError(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, errors.New("boom") }, })) @@ -381,7 +385,7 @@ func TestGetObserverActivity_NullsSerialise(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return &api.ObserverActivity{ PayloadTypes: []api.PayloadBreakdownItem{}, Points: []api.ObserverActivityPoint{{T: 1757376000000, Observations: 3}}, diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 42200c1a..004d94cf 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -32,7 +32,7 @@ type stubReader struct { getObserver func(ctx context.Context, observerID uuid.UUID) (*api.Observer, error) getObserverTelemetry func(ctx context.Context, observerID uuid.UUID, since, until time.Time, afterID int64) (*api.ObserverTelemetry, error) getObserverTelemetryBucketed func(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]api.ObserverTelemetryPoint, error) - getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) + getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) getObserverScopes func(ctx context.Context, observerID uuid.UUID) ([]string, error) listObserverAdverts func(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) listNodes func(ctx context.Context, nodeType int16, iatas []string, supportsMultibytePaths, supportsMultibyteTraces *bool, pubkey []byte, pubkeyPrefix, name, scope string, cursor int64, limit int32, includeNeighbors bool) (api.Page[api.NodeSummary], error) @@ -186,9 +186,9 @@ func (s stubReader) GetObserverTelemetryBucketed(ctx context.Context, observerID return nil, nil } -func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { +func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { if s.getObserverActivity != nil { - return s.getObserverActivity(ctx, observerID, window, interval) + return s.getObserverActivity(ctx, observerID, window, interval, until) } return nil, nil } diff --git a/internal/api/observers.go b/internal/api/observers.go index 206387b7..d16fb131 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -44,8 +44,8 @@ type Observer struct { LastStatusAt *int64 `json:"lastStatusAt,omitempty"` // epoch ms FirstSeen int64 `json:"firstSeen"` // epoch ms LastSeen int64 `json:"lastSeen"` // epoch ms - ObservationCount int64 `json:"observationCount"` - Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on + ObservationCount int64 `json:"observationCount"` // legacy cumulative presence counter; includes non-packet events + Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on } // ObserverTelemetryPoint is a single telemetry snapshot for an observer. @@ -89,7 +89,22 @@ type ObserverActivityPoint struct { } // ObserverActivity is the per-observer heard-activity response. +// ObserverActivitySummary describes stored packet records, never MQTT presence events. +type ObserverActivitySummary struct { + RecordedPackets int64 `json:"recordedPackets"` + LastCompleteHour int64 `json:"lastCompleteHour"` + LastCompleteHourStart int64 `json:"lastCompleteHourStart"` + LastCompleteHourEnd int64 `json:"lastCompleteHourEnd"` + LatestRecordedAt *int64 `json:"latestRecordedAt"` +} + type ObserverActivity struct { + WindowStart int64 `json:"windowStart"` // inclusive complete-bucket start, epoch ms + WindowEnd int64 `json:"windowEnd"` // exclusive complete-bucket end, epoch ms + GeneratedAt int64 `json:"generatedAt"` // response computation time, not proof of continuous coverage + Source string `json:"source"` // raw or hourly; missing records do not prove an outage + Summary *ObserverActivitySummary `json:"summary,omitempty"` + Range string `json:"range"` Interval string `json:"interval"` Radio *ObserverActivityRadio `json:"radio"` diff --git a/internal/api/reader.go b/internal/api/reader.go index 8aa3f1af..c9a424d2 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -100,7 +100,7 @@ type Reader interface { // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. // interval >= 1h is served from the hourly rollup. Returns pgx.ErrNoRows for an unknown observer. // Range and Interval on the result are left empty for the handler to fill. - GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*ObserverActivity, error) + GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*ObserverActivity, error) // GetObserverScopes returns the names of all transport scopes an observer has // been seen forwarding packets for, ordered alphabetically. diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index e58f0544..21d6f3d5 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -148,7 +148,7 @@ func (s *stubReader) GetObserverTelemetryBucketed(_ context.Context, _ uuid.UUID return nil, nil } -func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { +func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, nil } diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 65551da8..2d31ba4c 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -372,10 +372,10 @@ func (cr *CachedReader) GetObserverTelemetryBucketed(ctx context.Context, observ } // GetObserverActivity implements [api.Reader]. -func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { - key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() +func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { + key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano) return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) { - return cr.inner.GetObserverActivity(ctx, observerID, window, interval) + return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until) }) } diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 998c2051..0c4d7f82 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -89,7 +89,7 @@ type DB interface { UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) // UpsertObserverBroker records that this observer was seen on brokerName. - UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error + UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error // UpsertIATA auto-creates an iata_codes row if it doesn't exist yet. UpsertIATA(ctx context.Context, iata string) error diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index ea206c5d..2090f0e5 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -201,8 +201,10 @@ func (s *stubDB) SetNodeCapability(_ context.Context, nodeID uuid.UUID, paths, t func (s *stubDB) UpsertObserver(_ context.Context, _ []byte) (uuid.UUID, string, error) { return uuid.Nil, "", nil } -func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string) error { return nil } -func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } +func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, _ bool) error { + return nil +} +func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } func (s *stubDB) UpsertPacket(_ context.Context, _ UpsertPacketParams) (bool, error) { return false, nil } diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go index f96a32a1..3def0c9a 100644 --- a/internal/ingest/neighbors.go +++ b/internal/ingest/neighbors.go @@ -48,7 +48,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra w.log.Error(fmt.Sprintf("db: upsert observer failed in neighbors from %s", pubkeyHex), "error", err) return } - if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName); err != nil { + if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName, false); err != nil { w.log.Error(fmt.Sprintf("db: upsert observer broker failed in neighbors from %s", pubkeyHex), "error", err) } diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 92f3da78..bcc9ca58 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -324,7 +324,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ w.log.Error(fmt.Sprintf("db: upsert observer failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } - err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName) + err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, true) if err != nil { w.log.Error(fmt.Sprintf("db: update observer broker failed with packet from %s/%s", iata, pubkeyHex), "error", err) return diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 31c80647..3c693db6 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -87,7 +87,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) if w.onObserverUpsert != nil { w.onObserverUpsert(ctx, id) } - if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName); err != nil { + if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, false); err != nil { w.log.Error(fmt.Sprintf("db: upsert observer broker failed in status from %s", pubkeyHex), "error", err) } params := UpdateObserverStatusParams{ diff --git a/internal/presence/broker_freshness_test.go b/internal/presence/broker_freshness_test.go new file mode 100644 index 00000000..887a6e7c --- /dev/null +++ b/internal/presence/broker_freshness_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package presence + +import ( + "context" + "github.com/google/uuid" + "testing" + "time" +) + +type brokerTestStore struct { + Store + initial []bool + seen, packets []time.Time +} + +func (s *brokerTestStore) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, packet bool) error { + s.initial = append(s.initial, packet) + return nil +} +func (s *brokerTestStore) TouchObserverBrokers(_ context.Context, _ []uuid.UUID, _ []string, seen, packets []time.Time) error { + s.seen = seen + s.packets = packets + return nil +} + +func TestBrokerPresenceDoesNotInventPacketFreshness(t *testing.T) { + ctx := context.Background() + store := &brokerTestStore{} + c := New(store, time.Second, time.Minute) + now := time.Now() + c.now = func() time.Time { return now } + id := uuid.New() + c.UpsertObserverBroker(ctx, id, "one", false) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if len(store.initial) != 1 || store.initial[0] || !store.packets[0].IsZero() { + t.Fatal("status-only activity invented a packet") + } + now = now.Add(time.Minute) + packetTime := now + c.UpsertObserverBroker(ctx, id, "one", true) + now = now.Add(time.Minute) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if !store.seen[0].Equal(now) || !store.packets[0].Equal(packetTime) { + t.Fatal("later status changed packet freshness") + } +} diff --git a/internal/presence/coalescer.go b/internal/presence/coalescer.go index 9315f433..517f1217 100644 --- a/internal/presence/coalescer.go +++ b/internal/presence/coalescer.go @@ -31,7 +31,7 @@ type Store interface { // TouchObserverBrokers applies coalesced last_seen/last_packet_at bumps // for the given (observer, broker) pairs in one statement. - TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error + TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error // TouchPackets applies coalesced last_heard_at bumps for the given packet // hashes in one statement. @@ -51,6 +51,8 @@ type observerBump struct { count int32 } +type brokerBump struct{ seen, packet time.Time } + type brokerKey struct { id uuid.UUID broker string @@ -68,7 +70,7 @@ type Coalescer struct { identities map[string]identity // pubkey -> observer row dirtyObservers map[uuid.UUID]observerBump knownBrokers map[brokerKey]struct{} - dirtyBrokers map[brokerKey]time.Time + dirtyBrokers map[brokerKey]brokerBump knownIATAs map[string]struct{} packetsSeen map[string]time.Time // hash -> last observation dirtyPackets map[string]time.Time @@ -86,7 +88,7 @@ func New(store Store, flushInterval, packetTTL time.Duration) *Coalescer { identities: make(map[string]identity), dirtyObservers: make(map[uuid.UUID]observerBump), knownBrokers: make(map[brokerKey]struct{}), - dirtyBrokers: make(map[brokerKey]time.Time), + dirtyBrokers: make(map[brokerKey]brokerBump), knownIATAs: make(map[string]struct{}), packetsSeen: make(map[string]time.Time), dirtyPackets: make(map[string]time.Time), @@ -122,17 +124,22 @@ func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUI // UpsertObserverBroker writes through the first time a pair is seen and // records a bump afterwards. -func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { key := brokerKey{id: observerID, broker: brokerName} c.mu.Lock() if _, ok := c.knownBrokers[key]; ok { - c.dirtyBrokers[key] = c.now() + bump := c.dirtyBrokers[key] + bump.seen = c.now() + if isPacket { + bump.packet = bump.seen + } + c.dirtyBrokers[key] = bump c.mu.Unlock() return nil } c.mu.Unlock() - if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName); err != nil { + if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName, isPacket); err != nil { return err } c.mu.Lock() @@ -248,7 +255,7 @@ func (c *Coalescer) Flush(ctx context.Context) { brokers := c.dirtyBrokers packets := c.dirtyPackets c.dirtyObservers = make(map[uuid.UUID]observerBump) - c.dirtyBrokers = make(map[brokerKey]time.Time) + c.dirtyBrokers = make(map[brokerKey]brokerBump) c.dirtyPackets = make(map[string]time.Time) cutoff := c.now().Add(-c.packetTTL) for key, seen := range c.packetsSeen { @@ -266,12 +273,14 @@ func (c *Coalescer) Flush(ctx context.Context) { ids := make([]uuid.UUID, 0, len(brokers)) names := make([]string, 0, len(brokers)) seen := make([]time.Time, 0, len(brokers)) + packetTimes := make([]time.Time, 0, len(brokers)) for key, ts := range brokers { ids = append(ids, key.id) names = append(names, key.broker) - seen = append(seen, ts) + seen = append(seen, ts.seen) + packetTimes = append(packetTimes, ts.packet) } - if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen); err != nil { + if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen, packetTimes); err != nil { slog.Error(fmt.Sprintf("presence: flush observer brokers failed (%d rows dropped)", len(ids)), "component", "presence", "error", err) } } From 47342abc998432e6e0b4dd74e4c86d3e09eeaaff Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 22:34:58 -0400 Subject: [PATCH 060/141] fix(observers): keep non-finite signal samples out of JSON --- db/observer_metrics_integration_test.go | 6 +++++- db/observers.go | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 868ce6c0..f4e114ea 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -3,6 +3,7 @@ package db import ( + "encoding/json" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -30,7 +31,7 @@ func TestObserverMetricsPostgres(t *testing.T) { INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) - SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,0,-100 FROM packets; + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash=int4send(2) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; `); err != nil { t.Fatal(err) } @@ -48,6 +49,9 @@ func TestObserverMetricsPostgres(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := json.Marshal(activity); err != nil { + t.Fatalf("non-finite sample leaked into JSON: %v", err) + } if activity.Summary.RecordedPackets != 3 || len(activity.PayloadTypes) != 2 || activity.Summary.LatestRecordedAt != nil || activity.Summary.LastCompleteHour != 0 { t.Fatalf("archive summary: %+v types=%+v", activity.Summary, activity.PayloadTypes) } diff --git a/db/observers.go b/db/observers.go index 1735042a..09a377a3 100644 --- a/db/observers.go +++ b/db/observers.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "fmt" "log/slog" + "math" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -296,7 +297,12 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w if r.SnrN > 0 { avg := r.SnrSum / float32(r.SnrN) min := r.SnrMin - p.SNRAvg, p.SNRMin = &avg, &min + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } } if r.RssiN > 0 { avg := float32(r.RssiSum) / float32(r.RssiN) @@ -346,7 +352,12 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w } if r.SnrN > 0 { avg, min := r.SnrAvg, r.SnrMin - p.SNRAvg, p.SNRMin = &avg, &min + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } } if r.RssiN > 0 { avg := r.RssiAvg From 5977e42c5c44185a308b82b503ee87a10c24bf7a Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 22:39:21 -0400 Subject: [PATCH 061/141] test(observers): exercise non-finite raw and archived samples --- db/observer_metrics_integration_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index f4e114ea..6266dff7 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -31,7 +31,7 @@ func TestObserverMetricsPostgres(t *testing.T) { INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) - SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash=int4send(2) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash IN (int4send(1),int4send(2)) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; `); err != nil { t.Fatal(err) } @@ -59,7 +59,7 @@ func TestObserverMetricsPostgres(t *testing.T) { t.Fatalf("window: %+v", activity) } if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,last_heard_at) VALUES ('\xaa',NOW()); - INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,0,-100); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,'NaN'::real,-100); INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW()) ON CONFLICT DO NOTHING;`); err != nil { t.Fatal(err) } @@ -67,6 +67,9 @@ func TestObserverMetricsPostgres(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := json.Marshal(raw); err != nil { + t.Fatalf("raw non-finite sample leaked into JSON: %v", err) + } if raw.Summary.RecordedPackets != 1 || raw.Summary.LastCompleteHour != 1 || raw.Summary.LatestRecordedAt == nil || raw.PayloadTypes[0].PayloadType != -1 { t.Fatalf("raw/duplicate: %+v", raw) } From d3c330e8fb9d3ebf384334ae7ab2286ea103d9eb Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 18:27:40 -0400 Subject: [PATCH 062/141] fix: address maintainer review for #169 --- README.md | 9 +- .../040_observer_unknown_activity.sql | 279 ------------------ db/observer_metrics_integration_test.go | 7 +- db/observer_window_test.go | 55 ++++ db/observers.go | 3 +- db/observers_test.go | 2 +- db/queries/observer_activity_summary.sql | 13 - db/queries/queries.sql | 14 + db/sqlc/models.go | 2 +- db/sqlc/observer_activity_summary.sql.go | 52 ---- db/sqlc/queries.sql.go | 39 +++ docs/docs.go | 3 +- docs/swagger.json | 3 +- docs/swagger.yaml | 4 +- internal/api/observers.go | 8 +- internal/cache/observer_window_test.go | 47 +++ internal/cache/reader.go | 3 + 17 files changed, 183 insertions(+), 360 deletions(-) delete mode 100644 db/migrations/040_observer_unknown_activity.sql create mode 100644 db/observer_window_test.go delete mode 100644 db/queries/observer_activity_summary.sql delete mode 100644 db/sqlc/observer_activity_summary.sql.go create mode 100644 internal/cache/observer_window_test.go diff --git a/README.md b/README.md index 5f539b53..773cfd97 100644 --- a/README.md +++ b/README.md @@ -553,11 +553,11 @@ new packet deletions. A failed archive leaves its entire raw batch intact. ### Observer monitoring metrics -Observer activity returns complete buckets in `[windowStart, windowEnd)`, plus +Observer activity returns buckets in `[windowStart, windowEnd)`, including the current partial bucket for live requests, plus `generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the sum of stored observations in those buckets; repeated broker delivery of the same retained packet/observer pair counts once. Unknown payload types appear as -`-1` rather than disappearing. `lastCompleteHour` uses the previous complete UTC +`-1` rather than disappearing. Freshness fields are measured at `generatedAt` even for historical `until` requests; `recordedPackets` alone follows the selected window. `lastCompleteHour` uses the previous complete UTC hour and includes its own start/end; `latestRecordedAt` is the latest retained reception timestamp. Missing records do not prove downtime. The optional `until` (epoch milliseconds, within the last 30 days) aligns two observers' charts. @@ -566,5 +566,6 @@ The existing observer `observationCount` remains a legacy cumulative presence counter for compatibility, including status/neighbour events. It is not a period packet total. Broker presence and packet-arrival timestamps are now updated separately; this cannot reconstruct previously overwritten timestamps. -Migration 040 repairs archived unknown-type counts from the all-payload observer -rollup; radio samples already discarded for those legacy rows remain unknown. +The initial analytics archive migration includes unknown-type activity. Existing +development previews that used the earlier draft require a separate operator repair; +radio samples already discarded for those legacy rows cannot be recovered. diff --git a/db/migrations/040_observer_unknown_activity.sql b/db/migrations/040_observer_unknown_activity.sql deleted file mode 100644 index 6afab7e7..00000000 --- a/db/migrations/040_observer_unknown_activity.sql +++ /dev/null @@ -1,279 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Unknown legacy payloads are still recorded packets. Restore their archived --- counts from the all-payload observer rollup; their discarded radio samples --- cannot be recovered. Subtract all existing activity (including prior repairs) --- so migration-journal retries do not double-count or erase newer samples. -INSERT INTO analytics_observer_activity_hourly -(observer_id,payload_type,bucket,observations,airtime_n,snr_n,rssi_n) -SELECT totals.observer_id, -1, totals.bucket, totals.n-COALESCE(activity.n,0),0,0,0 -FROM (SELECT observer_id,bucket,SUM(observation_count)::bigint AS n - FROM analytics_top_observers_by_iata GROUP BY observer_id,bucket) totals -LEFT JOIN (SELECT observer_id,bucket,SUM(observations)::bigint AS n - FROM analytics_observer_activity_hourly GROUP BY observer_id,bucket) activity -USING (observer_id,bucket) -WHERE totals.n>COALESCE(activity.n,0) -ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE -SET observations=analytics_observer_activity_hourly.observations+EXCLUDED.observations; - -DROP MATERIALIZED VIEW mv_observer_activity_hourly; -CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); - -CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS -WITH combined AS ( - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly - UNION ALL - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM combined GROUP BY observer_id,payload_type,bucket; - -CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); - - - --- Preserve the bounded atomic archive, including unknown payload activity. -CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) -RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ -DECLARE hashes bytea[]; deleted bigint; -BEGIN - IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; - SELECT array_agg(p.packet_hash) INTO hashes FROM ( - SELECT ep.packet_hash FROM packets ep - WHERE ep.last_heard_at < cutoff - ORDER BY ep.last_heard_at, ep.packet_hash - LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED - ) p; - - WITH expired_observations AS MATERIALIZED ( - SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) - ), -archived_hourly_iata_stats AS ( - INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) - SELECT iata, hour, observation_count, unique_packets FROM ( -SELECT - iata, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY iata, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,hour) DO UPDATE SET - observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, - unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets -), -archived_payload_breakdown_by_iata AS ( - INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) - SELECT iata, payload_type, bucket, count FROM ( -SELECT - iata, - payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS count -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET - count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count -), -archived_top_observers_by_iata AS ( - INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM expired_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET - observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, - display_name = EXCLUDED.display_name, - observer_type = EXCLUDED.observer_type -), -archived_top_talkers_by_iata AS ( - INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) - SELECT iata, sender_name, bucket, message_count, last_sent FROM ( -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN expired_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') - ) batch - ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET - message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, - last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) -), -archived_top_advertisers_by_iata AS ( - INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN expired_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,node_id,bucket) DO UPDATE SET - advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, - flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, - direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, - last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), - name = EXCLUDED.name, - node_type = EXCLUDED.node_type -), -archived_observer_activity_hourly AS ( - INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET - observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, - airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, - airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, - snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, - snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, - snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), - rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, - rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n -), -archived_signal_stats_hourly AS ( - INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( -WITH samples AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), binned AS ( - SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples -) -SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(sum(snr), 0)::double precision AS snr_sum, - count(rssi)::bigint AS rssi_samples, - COALESCE(sum(rssi), 0)::double precision AS rssi_sum -FROM binned -GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) - ) batch - ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET - receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, - snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, - snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, - rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, - rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum -), -archived_path_stats_hourly AS ( - INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) - SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( -WITH classified AS ( - SELECT iata, heard_at, hash_size, hop_count, - CASE WHEN payload_type = 9 THEN 2 - -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). - WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 - OR NOT (path_length_byte BETWEEN 0 AND 191 - AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 - AND hash_size = (path_length_byte >> 6) + 1 - AND hop_count = (path_length_byte & 63) - AND hash_size::integer * hop_count::integer <= 64 - AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) - THEN 3 - WHEN hop_count = 0 THEN 1 - ELSE 0 END::integer AS category - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), buckets AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, - CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, - CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries - FROM classified -) -SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions -FROM buckets GROUP BY iata, hour, category, hash_bytes, entries - ) batch - ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET - receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions -) - DELETE FROM packets WHERE packet_hash = ANY(hashes); - GET DIAGNOSTICS deleted = ROW_COUNT; - - -- Once per completed cleanup, including when there were no raw packets to delete. - IF deleted < batch_size THEN - DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - END IF; - RETURN deleted; -END $$; diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 6266dff7..91dbeca1 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -41,9 +41,10 @@ func TestObserverMetricsPostgres(t *testing.T) { if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { t.Fatal(err) } - // Migration repairs the two unknown counts that 039 did not archive. - applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") - applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") + // The initial archive migration retains unknown payloads without a second rebuild. + if err := store.RefreshObserverActivity(ctx); err != nil { + t.Fatal(err) + } until := time.Now().UTC().Truncate(time.Hour) activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until) if err != nil { diff --git a/db/observer_window_test.go b/db/observer_window_test.go new file mode 100644 index 00000000..205d1ef8 --- /dev/null +++ b/db/observer_window_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/google/uuid" + "go.uber.org/mock/gomock" +) + +func TestObserverActivityLiveAndFixedWindow(t *testing.T) { + for _, interval := range []time.Duration{15 * time.Minute, time.Hour, 6 * time.Hour} { + for _, fixed := range []bool{false, true} { + t.Run(interval.String()+"/"+map[bool]string{false: "live", true: "fixed"}[fixed], func(t *testing.T) { + mock := mockdb.NewMockQuerier(gomock.NewController(t)) + id := uuid.New() + mock.EXPECT().GetObserverByID(gomock.Any(), id).Return(sqlc.Observer{}, nil) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil) + if interval < time.Hour { + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()).Return(nil, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) + } else { + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()).Return(nil, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) + } + before := time.Now().UTC() + var until time.Time + if fixed { + until = before.Add(-24*time.Hour + 123*time.Millisecond) + } + got, err := (&Store{q: mock}).GetObserverActivity(context.Background(), id, 48*time.Hour, interval, until) + if err != nil { + t.Fatal(err) + } + if fixed { + if got.WindowEnd != until.Truncate(interval).UnixMilli() { + t.Fatalf("fixed end was not aligned: %d", got.WindowEnd) + } + } else if got.WindowEnd < before.UnixMilli() || got.WindowEnd > time.Now().UnixMilli() { + t.Fatalf("live end dropped the partial bucket: %d", got.WindowEnd) + } + // Freshness is always live, even when the activity window is historical. + if got.Summary.LastCompleteHourEnd != time.UnixMilli(got.GeneratedAt).UTC().Truncate(time.Hour).UnixMilli() { + t.Fatal("freshness was anchored to the historical activity window") + } + }) + } + } +} diff --git a/db/observers.go b/db/observers.go index 09a377a3..36316db7 100644 --- a/db/observers.go +++ b/db/observers.go @@ -239,8 +239,9 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w now := time.Now().UTC() if until.IsZero() { until = now + } else { + until = until.UTC().Truncate(interval) } - until = until.UTC().Truncate(interval) since := until.Add(-window) if !since.Equal(since.Truncate(interval)) { since = since.Truncate(interval).Add(interval) diff --git a/db/observers_test.go b/db/observers_test.go index 31b562e3..44ed23c1 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -919,7 +919,7 @@ func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { if gotSince.Before(before) { t.Errorf("since %s is before the window start %s", gotSince, before) } - if !gotSince.Before(after.Add(interval)) { + if gotSince.After(after.Add(interval)) { t.Errorf("since %s is more than one interval past the window start %s", gotSince, after) } if !gotInterval.Valid || gotInterval.Microseconds != interval.Microseconds() { diff --git a/db/queries/observer_activity_summary.sql b/db/queries/observer_activity_summary.sql deleted file mode 100644 index 55b5aedd..00000000 --- a/db/queries/observer_activity_summary.sql +++ /dev/null @@ -1,13 +0,0 @@ --- name: GetObserverActivityLiveSummary :one --- Two indexed ranges, bounded to one observer; no legacy presence counters. -WITH latest AS ( - SELECT heard_at FROM packet_observations - WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz - ORDER BY heard_at DESC LIMIT 1 -), hourly AS ( - SELECT COUNT(*)::bigint AS n FROM packet_observations - WHERE observer_id = @observer_id::uuid - AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz -) -SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, - hourly.n AS last_complete_hour FROM hourly; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index aaf0be89..05ceacde 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1572,3 +1572,17 @@ WITH target AS MATERIALIZED ( ) SELECT EXISTS(SELECT 1 FROM target) AS found, EXISTS(SELECT 1 FROM changed) AS deactivated; + +-- name: GetObserverActivityLiveSummary :one +-- Two indexed ranges, bounded to one observer; no legacy presence counters. +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = @observer_id::uuid + AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly; diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 8a24d538..378fb602 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -32,7 +32,7 @@ type AnalyticsLiveHourlyIataStat struct { type AnalyticsLiveObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` - PayloadType int16 `json:"payload_type"` + PayloadType *int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs float32 `json:"airtime_ms"` diff --git a/db/sqlc/observer_activity_summary.sql.go b/db/sqlc/observer_activity_summary.sql.go deleted file mode 100644 index c6c97ac9..00000000 --- a/db/sqlc/observer_activity_summary.sql.go +++ /dev/null @@ -1,52 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.31.1 -// source: observer_activity_summary.sql - -package db - -import ( - "context" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" -) - -const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one -WITH latest AS ( - SELECT heard_at FROM packet_observations - WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz - ORDER BY heard_at DESC LIMIT 1 -), hourly AS ( - SELECT COUNT(*)::bigint AS n FROM packet_observations - WHERE observer_id = $1::uuid - AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz -) -SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, - hourly.n AS last_complete_hour FROM hourly -` - -type GetObserverActivityLiveSummaryParams struct { - ObserverID uuid.UUID `json:"observer_id"` - GeneratedAt pgtype.Timestamptz `json:"generated_at"` - HourStart pgtype.Timestamptz `json:"hour_start"` - HourEnd pgtype.Timestamptz `json:"hour_end"` -} - -type GetObserverActivityLiveSummaryRow struct { - LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"` - LastCompleteHour int64 `json:"last_complete_hour"` -} - -// Two indexed ranges, bounded to one observer; no legacy presence counters. -func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) { - row := q.db.QueryRow(ctx, getObserverActivityLiveSummary, - arg.ObserverID, - arg.GeneratedAt, - arg.HourStart, - arg.HourEnd, - ) - var i GetObserverActivityLiveSummaryRow - err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour) - return i, err -} diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 3dd6de88..489094c4 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -749,6 +749,45 @@ func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg return items, nil } +const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = $1::uuid + AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly +` + +type GetObserverActivityLiveSummaryParams struct { + ObserverID uuid.UUID `json:"observer_id"` + GeneratedAt pgtype.Timestamptz `json:"generated_at"` + HourStart pgtype.Timestamptz `json:"hour_start"` + HourEnd pgtype.Timestamptz `json:"hour_end"` +} + +type GetObserverActivityLiveSummaryRow struct { + LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"` + LastCompleteHour int64 `json:"last_complete_hour"` +} + +// Two indexed ranges, bounded to one observer; no legacy presence counters. +func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) { + row := q.db.QueryRow(ctx, getObserverActivityLiveSummary, + arg.ObserverID, + arg.GeneratedAt, + arg.HourStart, + arg.HourEnd, + ) + var i GetObserverActivityLiveSummaryRow + err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour) + return i, err +} + const getObserverActivityRaw = `-- name: GetObserverActivityRaw :many SELECT date_bin($3::interval, heard_at, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, diff --git a/docs/docs.go b/docs/docs.go index bc88fbf3..1b271012 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4019,7 +4019,7 @@ const docTemplate = `{ "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" }, "windowEnd": { - "description": "exclusive complete-bucket end, epoch ms", + "description": "exclusive end, epoch ms; live requests include the current partial bucket", "type": "integer" }, "windowStart": { @@ -4088,6 +4088,7 @@ const docTemplate = `{ "type": "integer" }, "recordedPackets": { + "description": "stored observations within windowStart/windowEnd", "type": "integer" } } diff --git a/docs/swagger.json b/docs/swagger.json index 75a8e5ac..52df28f7 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -4017,7 +4017,7 @@ "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" }, "windowEnd": { - "description": "exclusive complete-bucket end, epoch ms", + "description": "exclusive end, epoch ms; live requests include the current partial bucket", "type": "integer" }, "windowStart": { @@ -4086,6 +4086,7 @@ "type": "integer" }, "recordedPackets": { + "description": "stored observations within windowStart/windowEnd", "type": "integer" } } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index cab6df77..4fda3a40 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -584,7 +584,8 @@ definitions: summary: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary' windowEnd: - description: exclusive complete-bucket end, epoch ms + description: exclusive end, epoch ms; live requests include the current partial + bucket type: integer windowStart: description: inclusive complete-bucket start, epoch ms @@ -630,6 +631,7 @@ definitions: latestRecordedAt: type: integer recordedPackets: + description: stored observations within windowStart/windowEnd type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker: diff --git a/internal/api/observers.go b/internal/api/observers.go index d16fb131..2da680c1 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -88,19 +88,21 @@ type ObserverActivityPoint struct { RSSIAvg *float32 `json:"rssiAvg"` } -// ObserverActivity is the per-observer heard-activity response. // ObserverActivitySummary describes stored packet records, never MQTT presence events. +// Freshness and lastCompleteHour are measured at generatedAt, even with an explicit until; +// only recordedPackets follows the selected activity window. type ObserverActivitySummary struct { - RecordedPackets int64 `json:"recordedPackets"` + RecordedPackets int64 `json:"recordedPackets"` // stored observations within windowStart/windowEnd LastCompleteHour int64 `json:"lastCompleteHour"` LastCompleteHourStart int64 `json:"lastCompleteHourStart"` LastCompleteHourEnd int64 `json:"lastCompleteHourEnd"` LatestRecordedAt *int64 `json:"latestRecordedAt"` } +// ObserverActivity is the per-observer heard-activity response. type ObserverActivity struct { WindowStart int64 `json:"windowStart"` // inclusive complete-bucket start, epoch ms - WindowEnd int64 `json:"windowEnd"` // exclusive complete-bucket end, epoch ms + WindowEnd int64 `json:"windowEnd"` // exclusive end, epoch ms; live requests include the current partial bucket GeneratedAt int64 `json:"generatedAt"` // response computation time, not proof of continuous coverage Source string `json:"source"` // raw or hourly; missing records do not prove an outage Summary *ObserverActivitySummary `json:"summary,omitempty"` diff --git a/internal/cache/observer_window_test.go b/internal/cache/observer_window_test.go new file mode 100644 index 00000000..512e5800 --- /dev/null +++ b/internal/cache/observer_window_test.go @@ -0,0 +1,47 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" +) + +type activityWindowReader struct { + stubReader + ends []time.Time +} + +func (r *activityWindowReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, until time.Time) (*api.ObserverActivity, error) { + r.ends = append(r.ends, until) + return &api.ObserverActivity{WindowEnd: until.UnixMilli()}, nil +} + +func TestObserverActivityCacheUsesEffectiveWindow(t *testing.T) { + client, redis := newTestClient(t) + inner := &activityWindowReader{} + reader := &CachedReader{inner: inner, c: client} + id := uuid.New() + anchor := time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC) + for _, offset := range []time.Duration{time.Second, time.Minute, 59 * time.Minute} { + if _, err := reader.GetObserverActivity(context.Background(), id, 24*time.Hour, time.Hour, anchor.Add(offset)); err != nil { + t.Fatal(err) + } + } + if len(inner.ends) != 1 || !inner.ends[0].Equal(anchor) || len(redis.Keys()) != 1 { + t.Fatalf("equivalent fixed windows missed the cache: %v, %v", inner.ends, redis.Keys()) + } + for _, until := range []time.Time{anchor.Add(time.Hour), {}, {}} { + if _, err := reader.GetObserverActivity(context.Background(), id, 24*time.Hour, time.Hour, until); err != nil { + t.Fatal(err) + } + } + if len(inner.ends) != 3 || !inner.ends[2].IsZero() || len(redis.Keys()) != 3 { + t.Fatalf("distinct/live windows aliased: %v", inner.ends) + } +} diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 2d31ba4c..2cec6761 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -373,6 +373,9 @@ func (cr *CachedReader) GetObserverTelemetryBucketed(ctx context.Context, observ // GetObserverActivity implements [api.Reader]. func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { + if !until.IsZero() { + until = until.UTC().Truncate(interval) + } key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano) return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) { return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until) From dea539578367dc02544d92298d263be3e8a61a3e Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:14:51 -0400 Subject: [PATCH 063/141] fix: address release review follow-ups for #169 --- db/observers.go | 5 +---- db/sqlc/models.go | 4 ++-- db/sqlc/queries.sql.go | 4 ++-- internal/api/handlers/observers.go | 2 +- sqlc.yaml | 2 -- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/db/observers.go b/db/observers.go index 36316db7..fc10efad 100644 --- a/db/observers.go +++ b/db/observers.go @@ -321,10 +321,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - payloadType := int16(-1) - if v.PayloadType != nil { - payloadType = *v.PayloadType - } + payloadType := v.PayloadType activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ PayloadType: payloadType, PayloadTypeName: api.PayloadTypeName(payloadType), diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 378fb602..47cce44e 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -32,7 +32,7 @@ type AnalyticsLiveHourlyIataStat struct { type AnalyticsLiveObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` - PayloadType *int16 `json:"payload_type"` + PayloadType int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs float32 `json:"airtime_ms"` @@ -244,7 +244,7 @@ type MvHourlyIataStat struct { type MvObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` - PayloadType *int16 `json:"payload_type"` + PayloadType int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs float32 `json:"airtime_ms"` diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 489094c4..054bc001 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -725,8 +725,8 @@ type GetObserverActivityHourlyPayloadTypesParams struct { } type GetObserverActivityHourlyPayloadTypesRow struct { - PayloadType *int16 `json:"payload_type"` - Count int64 `json:"count"` + PayloadType int16 `json:"payload_type"` + Count int64 `json:"count"` } func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) { diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 15d52d6a..66bddd8d 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -256,7 +256,7 @@ var activityIntervals = map[string]time.Duration{ // @Param observerId path string true "Observer UUID" // @Param range query string false "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h" // @Param interval query string false "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)" -// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old" +// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until." // @Success 200 {object} api.ObserverActivity // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError diff --git a/sqlc.yaml b/sqlc.yaml index b5e86b89..9210892f 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -31,5 +31,3 @@ sql: - column: "mv_top_advertisers_by_iata.name" go_type: {type: "string", pointer: true} - - column: "mv_observer_activity_hourly.payload_type" - go_type: {type: "int16", pointer: true} From 551f061f9caa938e5196a4e77f484746e87df3d6 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:23:16 -0400 Subject: [PATCH 064/141] docs(api): refresh generated contract after stack update --- docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 1b271012..15be6240 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1471,7 +1471,7 @@ const docTemplate = `{ }, { "type": "integer", - "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until.", "name": "until", "in": "query" } diff --git a/docs/swagger.json b/docs/swagger.json index 52df28f7..96ef7bdc 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1469,7 +1469,7 @@ }, { "type": "integer", - "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until.", "name": "until", "in": "query" } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 4fda3a40..144e6213 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2432,7 +2432,9 @@ paths: name: interval type: string - description: Optional exclusive end in epoch milliseconds; aligned down to - a complete bucket, at most 30 days old + a complete bucket, at most 30 days old. Activity uses this window; summary + freshness/latestRecordedAt and the last complete hour are measured at response + generation time, independently of until. in: query name: until type: integer From de42595de0513381cacacfcd17f0a77bfbfe4460 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:28:23 -0400 Subject: [PATCH 065/141] style: trim regenerated sqlc configuration --- sqlc.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlc.yaml b/sqlc.yaml index 9210892f..032c9911 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -30,4 +30,3 @@ sql: go_type: {type: "string", pointer: true} - column: "mv_top_advertisers_by_iata.name" go_type: {type: "string", pointer: true} - From b8a8188fbcee30ba456223029044fbd4e59821df Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:35:13 -0400 Subject: [PATCH 066/141] test: mirror coalesced observer payload types --- db/observers_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/observers_test.go b/db/observers_test.go index 44ed23c1..f6c67d3d 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -588,8 +588,8 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { }}, nil) mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{ - {PayloadType: i16(4), Count: 9}, - {PayloadType: nil, Count: 3}, + {PayloadType: 4, Count: 9}, + {PayloadType: -1, Count: 3}, }, nil) store := &Store{q: mock} From c448a04ea0d638ae0800f7c2b66a3baf73a0a794 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 14:40:01 -0400 Subject: [PATCH 067/141] feat(routes): add bounded retained observation evidence --- .github/workflows/ci.yml | 2 +- README.md | 10 + db/migrations/041_route_evidence_index.sql | 9 + db/queries/route_evidence.sql | 22 ++ db/route_evidence.go | 93 +++++++++ db/route_evidence_integration_test.go | 205 +++++++++++++++++++ db/route_evidence_test.go | 23 +++ db/routes.go | 6 +- db/sqlc/mock/querier.go | 30 +++ db/sqlc/querier.go | 5 + db/sqlc/route_evidence.sql.go | 125 +++++++++++ docs/docs.go | 166 +++++++++++++++ docs/swagger.json | 166 +++++++++++++++ docs/swagger.yaml | 116 +++++++++++ internal/api/handlers/route_evidence.go | 118 +++++++++++ internal/api/handlers/route_evidence_test.go | 69 +++++++ internal/api/handlers/routes.go | 1 + internal/api/handlers/stub_reader_test.go | 8 + internal/api/reader.go | 2 + internal/api/route_evidence.go | 110 ++++++++++ internal/api/route_evidence_test.go | 25 +++ internal/api/routes.go | 1 + internal/cache/cache_test.go | 6 +- internal/cache/reader.go | 5 + 24 files changed, 1319 insertions(+), 4 deletions(-) create mode 100644 db/migrations/041_route_evidence_index.sql create mode 100644 db/queries/route_evidence.sql create mode 100644 db/route_evidence.go create mode 100644 db/route_evidence_integration_test.go create mode 100644 db/route_evidence_test.go create mode 100644 db/sqlc/route_evidence.sql.go create mode 100644 internal/api/handlers/route_evidence.go create mode 100644 internal/api/handlers/route_evidence_test.go create mode 100644 internal/api/route_evidence.go create mode 100644 internal/api/route_evidence_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9d71abe..da907fe7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index 773cfd97..a8215298 100644 --- a/README.md +++ b/README.md @@ -569,3 +569,13 @@ updated separately; this cannot reconstruct previously overwritten timestamps. The initial analytics archive migration includes unknown-type activity. Existing development previews that used the earlier draft require a separate operator repair; radio samples already discarded for those legacy rows cannot be recovered. + +### Saved-route observation evidence + +Known-route responses include `pathKey`, a stable identity within the route's IATA. Use `GET /api/v1/routes/{iata}/{pathKey}/observations` to fetch the **full saved route** and retained report references. Search results can contain a subsegment while sharing the full route's key; the evidence response always describes the complete saved sequence. + +The match requires the complete saved `pathBytes`, `hashSize`, hop count and IATA. A compact digest index narrows candidates, but full bytes are still compared. Other hash widths, TRACE readings/intended routes and unclassified legacy observations are excluded. Matching short prefixes does not confirm historical node identities, forwarding or delivery. The stored route counter can include repeated processing and outlive raw reports; it is not a retained-result total. + +`range` defaults to `24h` and accepts durations up to `720h`, anchored on the server. Alternatively supply both `since` and exclusive `until` in epoch milliseconds, with a maximum 30-day span and no future end. `limit` defaults to 50 and is capped at 200. Follow `nextPageCursor` as `pageCursor`; its route, window and microsecond/ID boundary are pinned. Do not combine it with another range, or change its explicit window. Numeric legacy `cursor` is unsupported. Responses include effective window bounds, `matchAvailable`, an empty `items` array when no matching raw evidence remains, and `hasMore`; no total-count scan or packet/message body is added. Malformed saved path metadata is explicitly unavailable, and missing routes return 404. + +Migration 041 builds the compact observation index concurrently. Keep it as a single statement outside a transaction; the existing runner handles an interrupted or already-built index before recording completion. It does not alter retained rows or expiry configuration. Native PostgreSQL tests cover ties below millisecond precision, cursor scope, different widths/sites, TRACE/unknown exclusions, raw expiry, index retry and custom/generic indexed plans. diff --git a/db/migrations/041_route_evidence_index.sql b/db/migrations/041_route_evidence_index.sql new file mode 100644 index 00000000..5b614da7 --- /dev/null +++ b/db/migrations/041_route_evidence_index.sql @@ -0,0 +1,9 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- One statement outside a transaction. RunMigrations recovers interrupted builds; +-- avoid IF NOT EXISTS, which would silently accept an invalid index. +-- A compact digest narrows candidates; queries still compare the complete bytes. +CREATE INDEX CONCURRENTLY idx_observations_route_evidence +ON packet_observations (iata, hash_size, (decode(md5(path_bytes), 'hex')), heard_at DESC, id DESC) +WHERE path_bytes IS NOT NULL AND payload_type IS NOT NULL AND payload_type <> 9; diff --git a/db/queries/route_evidence.sql b/db/queries/route_evidence.sql new file mode 100644 index 00000000..960b95e7 --- /dev/null +++ b/db/queries/route_evidence.sql @@ -0,0 +1,22 @@ +-- name: GetRouteEvidenceRoute :one +SELECT * FROM known_routes +WHERE iata = @iata::bpchar AND path_key = @path_key::bytea; + +-- name: ListRouteEvidence :many +-- Leading index equalities and the time/ID boundary bound both custom and generic plans. +-- Full-byte equality is required even when the compact digest matches. TRACE path bytes +-- carry readings; unclassified legacy observations cannot be safely called ordinary paths. +SELECT po.id, po.packet_hash, po.observer_id, o.display_name AS observer_name, + po.heard_at, po.payload_type, po.rssi, po.snr +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.iata = @iata::bpchar + AND po.hash_size = @hash_size::smallint + AND decode(md5(po.path_bytes), 'hex') = @path_digest::bytea + AND po.path_bytes = @path_bytes::bytea + AND po.hop_count = @hop_count::smallint + AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 + AND po.heard_at >= @since::timestamptz AND po.heard_at < @until::timestamptz + AND (po.heard_at, po.id) < (@before_at::timestamptz, @before_id::bigint) +ORDER BY po.heard_at DESC, po.id DESC +LIMIT @page_limit::integer; diff --git a/db/route_evidence.go b/db/route_evidence.go new file mode 100644 index 00000000..fc7209e0 --- /dev/null +++ b/db/route_evidence.go @@ -0,0 +1,93 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "math" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgtype" +) + +func savedRoutePath(prefixes [][]byte, count int32) (int16, []byte, bool) { + if count < 2 || count > 63 || len(prefixes) != int(count) { + return 0, nil, false + } + width := len(prefixes[0]) + if width < 1 || width > 3 { + return 0, nil, false + } + for _, p := range prefixes { + if len(p) != width { + return 0, nil, false + } + } + return int16(width), bytes.Join(prefixes, nil), true +} + +func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + if !api.ValidRouteEvidenceKey(iata, key) || !api.ValidRouteEvidenceWindow(query.Since, query.Until) || query.Limit < 1 || query.Limit > 200 { + return nil, api.ErrRouteEvidenceInput + } + if c := query.Cursor; c != nil { + if c.IATA != iata || c.PathKey != key || !c.Since.Equal(query.Since) || !c.Until.Equal(query.Until) || c.ID <= 0 || c.HeardAt.Before(query.Since) || !c.HeardAt.Before(query.Until) { + return nil, api.ErrRouteEvidenceInput + } + } + keyBytes, _ := hex.DecodeString(key) + row, err := s.q.GetRouteEvidenceRoute(ctx, sqlc.GetRouteEvidenceRouteParams{Iata: iata, PathKey: keyBytes}) + if err != nil { + return nil, err + } + nodes, err := s.GetNodesByIDs(ctx, row.NodeIds) + if err != nil { + return nil, err + } + route := toKnownRoutes([]knownRouteRow{{ID: row.ID, NodeIds: row.NodeIds, HashPrefix: row.HashPrefix, Iata: row.Iata, HopCount: row.HopCount, FirstSeen: row.FirstSeen, LastSeen: row.LastSeen, ObservationCount: row.ObservationCount}}, nodes)[0] + route.PathKey = hex.EncodeToString(row.PathKey) + out := &api.RouteEvidence{Page: api.Page[api.RouteObservation]{Items: []api.RouteObservation{}}, Route: route, WindowStart: query.Since.UnixMilli(), WindowEnd: query.Until.UnixMilli(), GeneratedAt: time.Now().UnixMilli(), MatchType: "saved_path_prefixes"} + width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) + if !valid { + return out, nil + } + out.MatchAvailable = true + out.HashSize = width + out.PathBytes = hex.EncodeToString(path) + beforeAt, beforeID := query.Until, int64(math.MaxInt64) + if query.Cursor != nil { + beforeAt, beforeID = query.Cursor.HeardAt, query.Cursor.ID + } + ts := func(t time.Time) pgtype.Timestamptz { return pgtype.Timestamptz{Time: t, Valid: true} } + digest := md5.Sum(path) + rows, err := s.q.ListRouteEvidence(ctx, sqlc.ListRouteEvidenceParams{Iata: iata, HashSize: width, HopCount: int16(row.HopCount), PathDigest: digest[:], PathBytes: path, Since: ts(query.Since), Until: ts(query.Until), BeforeAt: ts(beforeAt), BeforeID: beforeID, PageLimit: query.Limit + 1}) + if err != nil { + return nil, err + } + out.HasMore = len(rows) > int(query.Limit) + if out.HasMore { + rows = rows[:query.Limit] + } + for _, r := range rows { + payload := int16(-1) + if r.PayloadType != nil { + payload = *r.PayloadType + } + snr := r.Snr + if snr != nil && (math.IsNaN(float64(*snr)) || math.IsInf(float64(*snr), 0)) { + snr = nil + } + out.Items = append(out.Items, api.RouteObservation{ID: r.ID, PacketHash: hex.EncodeToString(r.PacketHash), ObserverID: r.ObserverID, ObserverName: r.ObserverName, HeardAt: r.HeardAt.Time.UnixMilli(), PayloadType: payload, PayloadTypeName: api.PayloadTypeName(payload), RSSI: r.Rssi, SNR: snr}) + } + if out.HasMore { + last := rows[len(rows)-1] + cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID}).String() + out.NextPageCursor = &cursor + } + return out, nil +} diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go new file mode 100644 index 00000000..b49224ed --- /dev/null +++ b/db/route_evidence_integration_test.go @@ -0,0 +1,205 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +type evidenceCapture struct { + pgx.Tx + statement string + args []any +} + +func (c *evidenceCapture) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + if strings.Contains(sql, "-- name: ListRouteEvidence") { + c.statement = sql + c.args = args + } + return c.Tx.Query(ctx, sql, args...) +} + +func TestRouteEvidencePostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + ddl, err := migrationFiles.ReadFile("migrations/024_known_routes_pathkey.sql") + if err != nil { + t.Fatal(err) + } + text := string(ddl) + start := strings.Index(text, "CREATE TABLE known_routes_new (") + end := start + strings.Index(text[start:], "\n);") + 3 + table := strings.ReplaceAll(text[start:end], "known_routes_new", "known_routes") + _, err = tx.Exec(ctx, `CREATE TABLE iata_codes (iata char(3) PRIMARY KEY); INSERT INTO iata_codes VALUES ('YOW'),('YVR'); + ALTER TABLE nodes ADD COLUMN latitude double precision, ADD COLUMN longitude double precision;`+table+` + INSERT INTO observers(id,display_name) VALUES ('00000000-0000-0000-0000-000000000001','One'),('00000000-0000-0000-0000-000000000002','Two'); + INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa','A'),('00000000-0000-0000-0000-000000000002','\xbb','B'); + INSERT INTO packets(packet_hash) SELECT int4send(i) FROM generate_series(1,10) i;`) + if err != nil { + t.Fatal(err) + } + index, err := migrationFiles.ReadFile("migrations/041_route_evidence_index.sql") + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, strings.Replace(string(index), "CREATE INDEX CONCURRENTLY", "CREATE INDEX", 1)); err != nil { + t.Fatal(err) + } + capture := &evidenceCapture{Tx: tx} + store := &Store{q: sqlc.New(capture)} + ids := []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000001"), uuid.MustParse("00000000-0000-0000-0000-000000000002")} + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } + key := hex.EncodeToString(routePathKey(ids)) + since := time.Now().UTC().Truncate(time.Millisecond).Add(-time.Hour) + until := since.Add(time.Hour) + at := since.Add(time.Minute + 123456*time.Microsecond) + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,snr) + SELECT i,int4send(i), '00000000-0000-0000-0000-000000000001',CASE WHEN i=5 THEN 'YVR' ELSE 'YOW' END, + CASE WHEN i=9 THEN $2::timestamptz WHEN i=1 THEN $1::timestamptz-interval '2 microseconds' ELSE $1::timestamptz END, + CASE WHEN i=4 THEN 2 ELSE 1 END,CASE WHEN i=4 THEN 1 WHEN i=10 THEN 3 ELSE 2 END, + CASE WHEN i=6 THEN '\xabbb'::bytea WHEN i=10 THEN '\xaabbcc'::bytea ELSE '\xaabb'::bytea END, + CASE WHEN i=7 THEN 9 WHEN i=8 THEN NULL ELSE 4 END,CASE WHEN i=1 THEN 'NaN'::real ELSE 0 END FROM generate_series(1,10) i; + `, at, until) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) VALUES (11,int4send(1),'00000000-0000-0000-0000-000000000002','YOW',$1,1,2,'\xaabb',4)`, at.Add(-time.Microsecond)); err != nil { + t.Fatal(err) + } + query := api.RouteEvidenceQuery{Since: since, Until: until, Limit: 2} + first, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil { + t.Fatal(err) + } + if !first.HasMore || first.NextPageCursor == nil || len(first.Items) != 2 || first.Items[0].ID != 3 || first.Items[1].ID != 2 || first.Route.PathKey != key || first.HashSize != 1 || first.PathBytes != "aabb" { + t.Fatalf("first page: %+v", first) + } + query.Cursor, err = api.ParseRouteEvidenceCursor(*first.NextPageCursor) + if err != nil { + t.Fatal(err) + } + second, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil { + t.Fatal(err) + } + if second.HasMore || len(second.Items) != 2 || second.Items[0].ID != 11 || second.Items[1].ID != 1 || second.Items[1].SNR != nil { + t.Fatalf("precise second page: %+v", second) + } + if _, err = json.Marshal(second); err != nil { + t.Fatalf("non-finite JSON: %v", err) + } + query.Cursor.IATA = "YVR" + if _, err = store.GetRouteEvidence(ctx, "YOW", key, query); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatal("cross-route cursor accepted") + } + query.Cursor = nil + if _, err = store.GetRouteEvidence(ctx, "YVR", key, query); !errors.Is(err, pgx.ErrNoRows) { + t.Fatal("unknown route did not return not-found") + } + if _, err = tx.Exec(ctx, "DELETE FROM packets"); err != nil { + t.Fatal(err) + } + empty, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || !empty.MatchAvailable || len(empty.Items) != 0 || empty.HasMore { + t.Fatalf("expired evidence: %+v %v", empty, err) + } + + // Many unrelated paths must not turn this into a raw-table or route-ID scan. + started := time.Now() + _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash) SELECT int4send(i) FROM generate_series(100,200099) i;`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',$1,1,2, + CASE WHEN get_byte(packet_hash,3)=0 THEN '\xaabb'::bytea ELSE packet_hash END,4 FROM packets;`, at) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, "ANALYZE packet_observations; ANALYZE observers;"); err != nil { + t.Fatal(err) + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err = tx.Exec(ctx, "SET LOCAL plan_cache_mode = "+mode); err != nil { + t.Fatal(err) + } + result, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || len(result.Items) != 2 { + t.Fatalf("indexed read: %v", err) + } + var plan string + if err = tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+capture.statement, capture.args...).Scan(&plan); err != nil { + t.Fatal(err) + } + if !strings.Contains(plan, "idx_observations_route_evidence") { + t.Fatalf("missing evidence index: %s", plan) + } + t.Logf("%s plan: %s", mode, plan) + } + t.Logf("200000-row fixture insert/analyze/read: %s", time.Since(started)) + if _, err = tx.Exec(ctx, `UPDATE known_routes SET hash_prefix=ARRAY['\xaa'::bytea]`); err != nil { + t.Fatal(err) + } + unavailable, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || unavailable.MatchAvailable || len(unavailable.Items) != 0 { + t.Fatalf("malformed stored path: %+v %v", unavailable, err) + } +} + +func TestRouteEvidenceIndexPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + schema := fmt.Sprintf("route_evidence_%d", time.Now().UnixNano()) + ident := pgx.Identifier{schema}.Sanitize() + if _, err = conn.Exec(ctx, "CREATE SCHEMA "+ident+"; SET search_path TO "+ident); err != nil { + t.Fatal(err) + } + defer func() { _, _ = conn.Exec(context.Background(), "DROP SCHEMA "+ident+" CASCADE") }() + if _, err = conn.Exec(ctx, `CREATE TABLE packet_observations (id bigint,iata char(3),hash_size smallint,path_bytes bytea,heard_at timestamptz,payload_type smallint)`); err != nil { + t.Fatal(err) + } + migration, err := migrationFiles.ReadFile("migrations/041_route_evidence_index.sql") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err = applyMigration(ctx, conn, string(migration)); err != nil { + t.Fatalf("migration/retry: %v", err) + } + } + if _, err = conn.Exec(ctx, `UPDATE pg_index SET indisvalid=false WHERE indexrelid='idx_observations_route_evidence'::regclass`); err != nil { + t.Fatal(err) + } + if err = applyMigration(ctx, conn, string(migration)); err != nil { + t.Fatal(err) + } + var valid bool + if err = conn.QueryRow(ctx, `SELECT indisvalid FROM pg_index WHERE indexrelid='idx_observations_route_evidence'::regclass`).Scan(&valid); err != nil || !valid { + t.Fatalf("invalid index after recovery: %v", err) + } +} diff --git a/db/route_evidence_test.go b/db/route_evidence_test.go new file mode 100644 index 00000000..868b565b --- /dev/null +++ b/db/route_evidence_test.go @@ -0,0 +1,23 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "bytes" + "testing" +) + +func TestSavedRoutePath(t *testing.T) { + size, path, ok := savedRoutePath([][]byte{{0xaa}, {0xbb}}, 2) + if !ok || size != 1 || !bytes.Equal(path, []byte{0xaa, 0xbb}) { + t.Fatal("valid path lost") + } + for _, prefixes := range [][][]byte{nil, {{1}}, {{1}, {2, 3}}, {{}, {2}}, {{1, 2, 3, 4}, {5, 6, 7, 8}}} { + if _, _, ok := savedRoutePath(prefixes, int32(len(prefixes))); ok { + t.Fatalf("accepted malformed path %x", prefixes) + } + } + if _, _, ok := savedRoutePath([][]byte{{1}, {2}}, 3); ok { + t.Fatal("accepted inconsistent hop count") + } +} diff --git a/db/routes.go b/db/routes.go index 187e07ec..71820ed4 100644 --- a/db/routes.go +++ b/db/routes.go @@ -118,6 +118,7 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st hops = append(hops, hop) } items = append(items, api.KnownRoute{ + PathKey: hex.EncodeToString(routePathKey(r.NodeIds)), ID: r.ID, IATA: r.Iata, HopCount: int32(len(hops)), @@ -223,7 +224,7 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t return nil, nil } - // 5. find cross-IATA links — nodes at the boundary of source routes + // 5. find cross-IATA links — nodes at the boundary of source routes // that have neighbors in the target IATA at the start of target routes var results []api.CrossIATARoute @@ -249,7 +250,7 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t continue } if targetHops, ok := targetNodeSet[neighbor.ID]; ok { - // found a cross-IATA link — build the route + // found a cross-IATA link — build the route sourceSegment := sr.Hops[:i+1] targetSegment := extractFromNode(targetHops, neighbor.ID) @@ -348,6 +349,7 @@ func toKnownRoutes(rows []knownRouteRow, nodes map[uuid.UUID]*api.ResolvedNode) hops = append(hops, hop) } items = append(items, api.KnownRoute{ + PathKey: hex.EncodeToString(routePathKey(r.NodeIds)), ID: r.ID, IATA: r.Iata, HopCount: r.HopCount, diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index cf80ee9b..9d3fb184 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -684,6 +684,21 @@ func (mr *MockQuerierMockRecorder) GetRegionIATAs(ctx, regionID any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegionIATAs", reflect.TypeOf((*MockQuerier)(nil).GetRegionIATAs), ctx, regionID) } +// GetRouteEvidenceRoute mocks base method. +func (m *MockQuerier) GetRouteEvidenceRoute(ctx context.Context, arg db.GetRouteEvidenceRouteParams) (db.KnownRoute, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRouteEvidenceRoute", ctx, arg) + ret0, _ := ret[0].(db.KnownRoute) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRouteEvidenceRoute indicates an expected call of GetRouteEvidenceRoute. +func (mr *MockQuerierMockRecorder) GetRouteEvidenceRoute(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRouteEvidenceRoute", reflect.TypeOf((*MockQuerier)(nil).GetRouteEvidenceRoute), ctx, arg) +} + // GetScopeByName mocks base method. func (m *MockQuerier) GetScopeByName(ctx context.Context, name string) (db.GetScopeByNameRow, error) { m.ctrl.T.Helper() @@ -1223,6 +1238,21 @@ func (mr *MockQuerierMockRecorder) ListRegions(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegions", reflect.TypeOf((*MockQuerier)(nil).ListRegions), ctx) } +// ListRouteEvidence mocks base method. +func (m *MockQuerier) ListRouteEvidence(ctx context.Context, arg db.ListRouteEvidenceParams) ([]db.ListRouteEvidenceRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRouteEvidence", ctx, arg) + ret0, _ := ret[0].([]db.ListRouteEvidenceRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRouteEvidence indicates an expected call of ListRouteEvidence. +func (mr *MockQuerierMockRecorder) ListRouteEvidence(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRouteEvidence", reflect.TypeOf((*MockQuerier)(nil).ListRouteEvidence), ctx, arg) +} + // ListTraceTags mocks base method. func (m *MockQuerier) ListTraceTags(ctx context.Context, arg db.ListTraceTagsParams) ([]db.ListTraceTagsRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 8f96cbe3..a5770646 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -87,6 +87,7 @@ type Querier interface { GetRegion(ctx context.Context, id int32) (GetRegionRow, error) GetRegionBySlug(ctx context.Context, slug string) (GetRegionBySlugRow, error) GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) + GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) GetScopeNames(ctx context.Context) ([]string, error) // Aggregate matching observations once, separately from node memberships to avoid @@ -198,6 +199,10 @@ type Querier interface { // REGIONS // ============================================================ ListRegions(ctx context.Context) ([]ListRegionsRow, error) + // Leading index equalities and the time/ID boundary bound both custom and generic plans. + // Full-byte equality is required even when the compact digest matches. TRACE path bytes + // carry readings; unclassified legacy observations cannot be safely called ordinary paths. + ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) // ============================================================ // TRACES // ============================================================ diff --git a/db/sqlc/route_evidence.sql.go b/db/sqlc/route_evidence.sql.go new file mode 100644 index 00000000..ab7f1ee3 --- /dev/null +++ b/db/sqlc/route_evidence.sql.go @@ -0,0 +1,125 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: route_evidence.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const getRouteEvidenceRoute = `-- name: GetRouteEvidenceRoute :one +SELECT id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count, last_reconfirmed_at FROM known_routes +WHERE iata = $1::bpchar AND path_key = $2::bytea +` + +type GetRouteEvidenceRouteParams struct { + Iata string `json:"iata"` + PathKey []byte `json:"path_key"` +} + +func (q *Queries) GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) { + row := q.db.QueryRow(ctx, getRouteEvidenceRoute, arg.Iata, arg.PathKey) + var i KnownRoute + err := row.Scan( + &i.ID, + &i.PathKey, + &i.NodeIds, + &i.HashPrefix, + &i.Iata, + &i.HopCount, + &i.FirstSeen, + &i.LastSeen, + &i.ObservationCount, + &i.LastReconfirmedAt, + ) + return i, err +} + +const listRouteEvidence = `-- name: ListRouteEvidence :many +SELECT po.id, po.packet_hash, po.observer_id, o.display_name AS observer_name, + po.heard_at, po.payload_type, po.rssi, po.snr +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.iata = $1::bpchar + AND po.hash_size = $2::smallint + AND decode(md5(po.path_bytes), 'hex') = $3::bytea + AND po.path_bytes = $4::bytea + AND po.hop_count = $5::smallint + AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 + AND po.heard_at >= $6::timestamptz AND po.heard_at < $7::timestamptz + AND (po.heard_at, po.id) < ($8::timestamptz, $9::bigint) +ORDER BY po.heard_at DESC, po.id DESC +LIMIT $10::integer +` + +type ListRouteEvidenceParams struct { + Iata string `json:"iata"` + HashSize int16 `json:"hash_size"` + PathDigest []byte `json:"path_digest"` + PathBytes []byte `json:"path_bytes"` + HopCount int16 `json:"hop_count"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + BeforeAt pgtype.Timestamptz `json:"before_at"` + BeforeID int64 `json:"before_id"` + PageLimit int32 `json:"page_limit"` +} + +type ListRouteEvidenceRow struct { + ID int64 `json:"id"` + PacketHash []byte `json:"packet_hash"` + ObserverID uuid.UUID `json:"observer_id"` + ObserverName *string `json:"observer_name"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + PayloadType *int16 `json:"payload_type"` + Rssi *int16 `json:"rssi"` + Snr *float32 `json:"snr"` +} + +// Leading index equalities and the time/ID boundary bound both custom and generic plans. +// Full-byte equality is required even when the compact digest matches. TRACE path bytes +// carry readings; unclassified legacy observations cannot be safely called ordinary paths. +func (q *Queries) ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) { + rows, err := q.db.Query(ctx, listRouteEvidence, + arg.Iata, + arg.HashSize, + arg.PathDigest, + arg.PathBytes, + arg.HopCount, + arg.Since, + arg.Until, + arg.BeforeAt, + arg.BeforeID, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListRouteEvidenceRow{} + for rows.Next() { + var i ListRouteEvidenceRow + if err := rows.Scan( + &i.ID, + &i.PacketHash, + &i.ObserverID, + &i.ObserverName, + &i.HeardAt, + &i.PayloadType, + &i.Rssi, + &i.Snr, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/docs/docs.go b/docs/docs.go index 15be6240..93bc6f52 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2131,6 +2131,92 @@ const docTemplate = `{ } } }, + "/routes/{iata}/{pathKey}/observations": { + "get": { + "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Get retained reports matching a full saved route prefix sequence", + "parameters": [ + { + "type": "string", + "description": "Three-character IATA code", + "name": "iata", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Stable 32-hex pathKey from a known-route response", + "name": "pathKey", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Window start epoch ms; provide with until (default last 24h)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Exclusive window end epoch ms; maximum span 30d, no future end", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Default 50; positive, capped at 200", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/scopes": { "get": { "produces": [ @@ -3601,6 +3687,10 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" + }, + "pathKey": { + "description": "stable identity within this IATA; use for route evidence links", + "type": "string" } } }, @@ -4845,6 +4935,50 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence": { + "type": "object", + "properties": { + "generatedAt": { + "type": "integer" + }, + "hasMore": { + "type": "boolean" + }, + "hashSize": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation" + } + }, + "matchAvailable": { + "type": "boolean" + }, + "matchType": { + "type": "string" + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + }, + "pathBytes": { + "type": "string" + }, + "route": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + }, + "windowEnd": { + "type": "integer" + }, + "windowStart": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop": { "type": "object", "properties": { @@ -4865,6 +4999,38 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation": { + "type": "object", + "properties": { + "heardAt": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "observerId": { + "type": "string" + }, + "observerName": { + "type": "string" + }, + "packetHash": { + "type": "string" + }, + "payloadType": { + "type": "integer" + }, + "payloadTypeName": { + "type": "string" + }, + "rssi": { + "type": "integer" + }, + "snr": { + "type": "number" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 96ef7bdc..617277a7 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2129,6 +2129,92 @@ } } }, + "/routes/{iata}/{pathKey}/observations": { + "get": { + "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Get retained reports matching a full saved route prefix sequence", + "parameters": [ + { + "type": "string", + "description": "Three-character IATA code", + "name": "iata", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Stable 32-hex pathKey from a known-route response", + "name": "pathKey", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Window start epoch ms; provide with until (default last 24h)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Exclusive window end epoch ms; maximum span 30d, no future end", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Default 50; positive, capped at 200", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/scopes": { "get": { "produces": [ @@ -3599,6 +3685,10 @@ }, "observationCount": { "type": "integer" + }, + "pathKey": { + "description": "stable identity within this IATA; use for route evidence links", + "type": "string" } } }, @@ -4843,6 +4933,50 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence": { + "type": "object", + "properties": { + "generatedAt": { + "type": "integer" + }, + "hasMore": { + "type": "boolean" + }, + "hashSize": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation" + } + }, + "matchAvailable": { + "type": "boolean" + }, + "matchType": { + "type": "string" + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + }, + "pathBytes": { + "type": "string" + }, + "route": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + }, + "windowEnd": { + "type": "integer" + }, + "windowStart": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop": { "type": "object", "properties": { @@ -4863,6 +4997,38 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation": { + "type": "object", + "properties": { + "heardAt": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "observerId": { + "type": "string" + }, + "observerName": { + "type": "string" + }, + "packetHash": { + "type": "string" + }, + "payloadType": { + "type": "integer" + }, + "payloadTypeName": { + "type": "string" + }, + "rssi": { + "type": "integer" + }, + "snr": { + "type": "number" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 144e6213..3898b5de 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -272,6 +272,9 @@ definitions: type: integer observationCount: type: integer + pathKey: + description: stable identity within this IATA; use for route evidence links + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.Node: properties: @@ -1153,6 +1156,35 @@ definitions: description: hex-encoded prefix used for resolution type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence: + properties: + generatedAt: + type: integer + hasMore: + type: boolean + hashSize: + type: integer + items: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation' + type: array + matchAvailable: + type: boolean + matchType: + type: string + nextCursor: + type: integer + nextPageCursor: + type: string + pathBytes: + type: string + route: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute' + windowEnd: + type: integer + windowStart: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop: properties: hashBytes: @@ -1165,6 +1197,27 @@ definitions: nodeId: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation: + properties: + heardAt: + type: integer + id: + type: integer + observerId: + type: string + observerName: + type: string + packetHash: + type: string + payloadType: + type: integer + payloadTypeName: + type: string + rssi: + type: integer + snr: + type: number + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail: properties: iataCount: @@ -2803,6 +2856,69 @@ paths: summary: List known routes tags: - Routes + /routes/{iata}/{pathKey}/observations: + get: + description: Matches the saved IATA, complete path bytes and hash width. Excludes + TRACE and unclassified reports; other widths and search-result subsegments + are not included. Byte matches do not prove hop identities or delivery. The + route and its historical counter can outlive raw evidence. Cursors retain + full timestamp precision and pin route/window scope. + parameters: + - description: Three-character IATA code + in: path + name: iata + required: true + type: string + - description: Stable 32-hex pathKey from a known-route response + in: path + name: pathKey + required: true + type: string + - description: Server-anchored duration (default 24h, max 720h); exclusive with + since/until/pageCursor + in: query + name: range + type: string + - description: Window start epoch ms; provide with until (default last 24h) + in: query + name: since + type: integer + - description: Exclusive window end epoch ms; maximum span 30d, no future end + in: query + name: until + type: integer + - description: Opaque precise cursor from nextPageCursor; window is pinned + in: query + name: pageCursor + type: string + - description: Default 50; positive, capped at 200 + in: query + maximum: 200 + minimum: 1 + name: limit + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Get retained reports matching a full saved route prefix sequence + tags: + - Routes /routes/cross: get: parameters: diff --git a/internal/api/handlers/route_evidence.go b/internal/api/handlers/route_evidence.go new file mode 100644 index 00000000..6c156fef --- /dev/null +++ b/internal/api/handlers/route_evidence.go @@ -0,0 +1,118 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" +) + +func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.RouteEvidenceQuery, error) { + q := api.RouteEvidenceQuery{} + values := r.URL.Query() + for _, name := range []string{"pageCursor", "since", "until", "range", "limit"} { + if len(values[name]) > 1 { + return q, api.ErrRouteEvidenceInput + } + } + if values.Has("cursor") { + return q, api.ErrRouteEvidenceInput + } + limit, err := parseLimit(r, 50) + if err != nil { + return q, err + } + q.Limit = limit + if values.Has("pageCursor") { + c, err := api.ParseRouteEvidenceCursor(values.Get("pageCursor")) + if err != nil || c.IATA != iata || c.PathKey != key { + return q, api.ErrRouteEvidenceInput + } + q.Cursor = c + q.Since = c.Since + q.Until = c.Until + } else { + q.Until = now.UTC().Truncate(time.Millisecond) + q.Since = q.Until.Add(-24 * time.Hour) + } + if values.Has("range") { + if values.Has("pageCursor") || values.Has("since") || values.Has("until") { + return q, api.ErrRouteEvidenceInput + } + duration, err := time.ParseDuration(values.Get("range")) + if err != nil || duration < time.Millisecond || duration > api.MaxRouteEvidenceWindow || duration%time.Millisecond != 0 { + return q, api.ErrRouteEvidenceInput + } + q.Since = q.Until.Add(-duration) + } + if values.Has("since") || values.Has("until") { + if !values.Has("since") || !values.Has("until") { + return q, api.ErrRouteEvidenceInput + } + since, e1 := strconv.ParseInt(values.Get("since"), 10, 64) + until, e2 := strconv.ParseInt(values.Get("until"), 10, 64) + if e1 != nil || e2 != nil || since < 0 || until < 0 || since > 253402300799999 || until > 253402300799999 { + return q, api.ErrRouteEvidenceInput + } + q.Since = time.UnixMilli(since).UTC() + q.Until = time.UnixMilli(until).UTC() + if q.Cursor != nil && (!q.Since.Equal(q.Cursor.Since) || !q.Until.Equal(q.Cursor.Until)) { + return q, api.ErrRouteEvidenceInput + } + } + if !api.ValidRouteEvidenceWindow(q.Since, q.Until) || q.Until.After(now) { + return q, api.ErrRouteEvidenceInput + } + return q, nil +} + +// getRouteEvidence godoc +// +// @Summary Get retained reports matching a full saved route prefix sequence +// @Description Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope. +// @Tags Routes +// @Produce json +// @Param iata path string true "Three-character IATA code" +// @Param pathKey path string true "Stable 32-hex pathKey from a known-route response" +// @Param range query string false "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor" +// @Param since query int false "Window start epoch ms; provide with until (default last 24h)" +// @Param until query int false "Exclusive window end epoch ms; maximum span 30d, no future end" +// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; window is pinned" +// @Param limit query int false "Default 50; positive, capped at 200" minimum(1) maximum(200) +// @Success 200 {object} api.RouteEvidence +// @Failure 400 {object} handlers.APIError +// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Router /routes/{iata}/{pathKey}/observations [get] +func getRouteEvidence(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + iata, key := strings.ToUpper(chi.URLParam(r, "iata")), strings.ToLower(chi.URLParam(r, "pathKey")) + if !api.ValidRouteEvidenceKey(iata, key) { + respondError(w, 400, "invalid IATA or route pathKey") + return + } + query, err := routeEvidenceQuery(r, iata, key, time.Now()) + if err != nil { + respondError(w, 400, "invalid route evidence window, limit or page cursor") + return + } + result, err := reader.GetRouteEvidence(r.Context(), iata, key, query) + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && result == nil: + respondError(w, 404, "saved route not found") + case errors.Is(err, api.ErrRouteEvidenceInput): + respondError(w, 400, "invalid route evidence request") + case err != nil: + respondError(w, 500, "internal server error") + default: + respond(w, 200, result) + } + } +} diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go new file mode 100644 index 00000000..382ea0ff --- /dev/null +++ b/internal/api/handlers/route_evidence_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "context" + "errors" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +const evidenceKey = "f097439148601d9f3291c474f82fa64c" + +func TestRouteEvidenceQuery(t *testing.T) { + now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC) + q, err := routeEvidenceQuery(httptest.NewRequest("GET", "/", nil), "YOW", evidenceKey, now) + if err != nil || q.Limit != 50 || q.Until != now || q.Until.Sub(q.Since) != 24*time.Hour { + t.Fatalf("defaults: %+v %v", q, err) + } + c := api.RouteEvidenceCursor{IATA: "YOW", PathKey: evidenceKey, Since: q.Since, Until: q.Until, HeardAt: q.Since.Add(time.Hour + 123*time.Microsecond), ID: 3} + next, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String())+"&limit=999", nil), "YOW", evidenceKey, now.Add(time.Hour)) + if err != nil || next.Limit != 200 || !next.Until.Equal(now) || !next.Cursor.HeardAt.Equal(c.HeardAt) { + t.Fatalf("pinned cursor: %+v %v", next, err) + } + for _, query := range []string{"?limit=0", "?limit=-1", "?limit=a", "?limit=1&limit=2", "?since=0", "?until=10", "?since=2&until=1", "?since=0&until=2678400000", "?since=0&until=99999999999999999", "?pageCursor=bad", "?pageCursor=", "?cursor=123", "?pageCursor=" + url.QueryEscape(c.String()) + "&since=0&until=10"} { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String()), nil), "YVR", evidenceKey, now); err == nil { + t.Fatal("cursor crossed IATA") + } +} + +func TestRouteEvidenceHandler(t *testing.T) { + for _, tc := range []struct { + path string + err error + status int + }{ + {"/yow/" + evidenceKey + "/observations", nil, 200}, + {"/YOW/bad/observations", nil, 400}, + {"/LONG/" + evidenceKey + "/observations", nil, 400}, + {"/YOW/" + evidenceKey + "/observations", pgx.ErrNoRows, 404}, + {"/YOW/" + evidenceKey + "/observations", errors.New("database unavailable"), 500}, + } { + called := false + router := RoutesRouter(stubReader{getRouteEvidence: func(_ context.Context, iata, key string, q api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + called = true + if iata != "YOW" || key != evidenceKey || q.Limit != 50 { + t.Fatal("wrong reader scope") + } + return &api.RouteEvidence{Page: api.Page[api.RouteObservation]{Items: []api.RouteObservation{}}}, tc.err + }}) + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", tc.path, nil)) + if w.Code != tc.status { + t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String()) + } + if tc.status == 400 && called { + t.Fatal("invalid request reached database") + } + } +} diff --git a/internal/api/handlers/routes.go b/internal/api/handlers/routes.go index a5bdd76c..77f1117e 100644 --- a/internal/api/handlers/routes.go +++ b/internal/api/handlers/routes.go @@ -22,6 +22,7 @@ func RoutesRouter(reader api.Reader) http.Handler { r.Get("/", listKnownRoutes(reader)) r.Get("/cross", searchCrossIATARoutes(reader)) r.Get("/search", searchKnownRoutes(reader)) + r.Get("/{iata}/{pathKey}/observations", getRouteEvidence(reader)) return r } diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 004d94cf..f9f885a6 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -16,6 +16,7 @@ import ( // Unset fields return zero values. Use it for both validation tests // (leave all fields nil) and happy path tests (set only what you need). type stubReader struct { + getRouteEvidence func(context.Context, string, string, api.RouteEvidenceQuery) (*api.RouteEvidence, error) getObserverComparison func(context.Context, uuid.UUID, uuid.UUID, time.Time, time.Time, []string) (*api.ObserverComparison, error) listIATAs func(ctx context.Context) ([]api.IATA, error) getIATA func(ctx context.Context, iata string) (*api.IATA, error) @@ -409,3 +410,10 @@ func (s stubReader) GetNodesByIDs(ctx context.Context, ids []uuid.UUID) (map[uui } return nil, nil } + +func (s stubReader) GetRouteEvidence(ctx context.Context, iata, key string, q api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + if s.getRouteEvidence != nil { + return s.getRouteEvidence(ctx, iata, key, q) + } + return nil, nil +} diff --git a/internal/api/reader.go b/internal/api/reader.go index c9a424d2..d46ec2d9 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -23,6 +23,8 @@ type Page[T any] struct { } type Reader interface { + // GetRouteEvidence returns the full saved route and exact-prefix retained ordinary reports. + GetRouteEvidence(ctx context.Context, iata, pathKey string, query RouteEvidenceQuery) (*RouteEvidence, error) // ListIATAs returns all known IATA codes with display name and coordinates. // IATAs are auto-created on first packet arrival from that location. ListIATAs(ctx context.Context) ([]IATA, error) diff --git a/internal/api/route_evidence.go b/internal/api/route_evidence.go new file mode 100644 index 00000000..cadc056a --- /dev/null +++ b/internal/api/route_evidence.go @@ -0,0 +1,110 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import ( + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +const MaxRouteEvidenceWindow = 30 * 24 * time.Hour + +var ErrRouteEvidenceInput = errors.New("invalid route evidence request") + +// RouteObservation is a retained report reference, not a packet body or proof of hop identity. +type RouteObservation struct { + ID int64 `json:"id"` + PacketHash string `json:"packetHash"` + ObserverID uuid.UUID `json:"observerId"` + ObserverName *string `json:"observerName,omitempty"` + HeardAt int64 `json:"heardAt"` + PayloadType int16 `json:"payloadType"` + PayloadTypeName string `json:"payloadTypeName"` + RSSI *int16 `json:"rssi,omitempty"` + SNR *float32 `json:"snr,omitempty"` +} + +// RouteEvidence uses nextPageCursor rather than the lossy legacy numeric cursor. +type RouteEvidence struct { + Page[RouteObservation] + Route KnownRoute `json:"route"` + WindowStart int64 `json:"windowStart"` + WindowEnd int64 `json:"windowEnd"` + GeneratedAt int64 `json:"generatedAt"` + MatchType string `json:"matchType"` + MatchAvailable bool `json:"matchAvailable"` + HashSize int16 `json:"hashSize"` + PathBytes string `json:"pathBytes"` + NextPageCursor *string `json:"nextPageCursor,omitempty"` +} + +type RouteEvidenceQuery struct { + Since time.Time + Until time.Time + Cursor *RouteEvidenceCursor + Limit int32 +} + +// RouteEvidenceCursor binds the precise (heard_at DESC,id DESC) boundary to its route/window. +type RouteEvidenceCursor struct { + IATA string + PathKey string + Since time.Time + Until time.Time + HeardAt time.Time + ID int64 +} + +func ValidRouteEvidenceKey(iata, key string) bool { + if len(iata) != 3 || len(key) != 32 { + return false + } + for _, c := range iata { + if !(c >= 'A' && c <= 'Z') && !(c >= '0' && c <= '9') { + return false + } + } + _, err := hex.DecodeString(key) + return err == nil && key == strings.ToLower(key) +} + +func ValidRouteEvidenceWindow(since, until time.Time) bool { + return !since.Before(time.Unix(0, 0)) && until.After(since) && until.Sub(since) <= MaxRouteEvidenceWindow +} + +func (c RouteEvidenceCursor) String() string { + return fmt.Sprintf("v1:%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) +} + +func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { + if len(raw) > 200 { + return nil, ErrRouteEvidenceInput + } + p := strings.Split(raw, ":") + if len(p) != 7 || p[0] != "v1" || !ValidRouteEvidenceKey(p[1], p[2]) { + return nil, ErrRouteEvidenceInput + } + values := [4]int64{} + for i, s := range p[3:] { + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || n < 0 || strconv.FormatInt(n, 10) != s { + return nil, ErrRouteEvidenceInput + } + values[i] = n + } + // Bound before converting; this also keeps millisecond/microsecond comparisons safe. + if values[0] > 253402300799999 || values[1] > 253402300799999 || values[2] > 253402300799999999 || values[3] <= 0 { + return nil, ErrRouteEvidenceInput + } + c := &RouteEvidenceCursor{IATA: p[1], PathKey: p[2], Since: time.UnixMilli(values[0]).UTC(), Until: time.UnixMilli(values[1]).UTC(), HeardAt: time.UnixMicro(values[2]).UTC(), ID: values[3]} + if !ValidRouteEvidenceWindow(c.Since, c.Until) || c.HeardAt.Before(c.Since) || !c.HeardAt.Before(c.Until) { + return nil, ErrRouteEvidenceInput + } + return c, nil +} diff --git a/internal/api/route_evidence_test.go b/internal/api/route_evidence_test.go new file mode 100644 index 00000000..42c9a73a --- /dev/null +++ b/internal/api/route_evidence_test.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import ( + "strings" + "testing" + "time" +) + +func TestRouteEvidenceCursor(t *testing.T) { + c := RouteEvidenceCursor{IATA: "YOW", PathKey: "f097439148601d9f3291c474f82fa64c", Since: time.UnixMilli(1000).UTC(), Until: time.UnixMilli(3000).UTC(), HeardAt: time.UnixMicro(2000123).UTC(), ID: 9223372036854775806} + got, err := ParseRouteEvidenceCursor(c.String()) + if err != nil || *got != c { + t.Fatalf("cursor round-trip: %+v %v", got, err) + } + if got.HeardAt.Nanosecond() != 123000 { + t.Fatal("lost microsecond precision") + } + for _, bad := range []string{"", strings.Repeat("x", 300), "v2:YOW:x:1:2:3:4", strings.Replace(c.String(), "YOW", "TOOLONG", 1), strings.Replace(c.String(), c.PathKey, "bad", 1), strings.Replace(c.String(), "2000123", "4000000", 1), strings.TrimSuffix(c.String(), "9223372036854775806") + "0"} { + if _, err := ParseRouteEvidenceCursor(bad); err == nil { + t.Errorf("accepted %q", bad) + } + } +} diff --git a/internal/api/routes.go b/internal/api/routes.go index 3bc7eced..3f1bee1e 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -16,6 +16,7 @@ type RouteHop struct { // have been confirmed as high confidence. type KnownRoute struct { ID int64 `json:"id"` + PathKey string `json:"pathKey,omitempty"` // stable identity within this IATA; use for route evidence links IATA string `json:"iata"` HopCount int32 `json:"hopCount"` Hops []RouteHop `json:"hops"` diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 21d6f3d5..58467cee 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -366,7 +366,7 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { return &api.StatsOverview{TotalPackets: 42}, nil }) - // call CachedReader with reversed order — should hit same key + // call CachedReader with reversed order — should hit same key _ = cr result, err := getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func() (*api.StatsOverview, error) { calls++ @@ -418,3 +418,7 @@ func TestCachedReader_InvalidateObserver(t *testing.T) { t.Error("expected observer scopes key to be deleted") } } + +func (s *stubReader) GetRouteEvidence(_ context.Context, _, _ string, _ api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + return nil, nil +} diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 2cec6761..a570df9e 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -467,6 +467,11 @@ func (cr *CachedReader) SearchKnownRoutes(ctx context.Context, iata, fromHash, t return cr.inner.SearchKnownRoutes(ctx, iata, fromHash, toHash) } +// Precise cursor/window combinations are intentionally passed through, like route lists. +func (cr *CachedReader) GetRouteEvidence(ctx context.Context, iata, key string, query api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + return cr.inner.GetRouteEvidence(ctx, iata, key, query) +} + // SearchCrossIATARoutes implements [api.Reader]. func (cr *CachedReader) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, toHash, toIATA string) ([]api.CrossIATARoute, error) { return cr.inner.SearchCrossIATARoutes(ctx, fromHash, fromIATA, toHash, toIATA) From cd20bc5c34f40338db908a5bcd7a40855a7b6ac7 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 14:56:08 -0400 Subject: [PATCH 068/141] test(routes): cover server-anchored evidence windows --- internal/api/handlers/route_evidence_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go index 382ea0ff..8340d85b 100644 --- a/internal/api/handlers/route_evidence_test.go +++ b/internal/api/handlers/route_evidence_test.go @@ -22,6 +22,10 @@ func TestRouteEvidenceQuery(t *testing.T) { if err != nil || q.Limit != 50 || q.Until != now || q.Until.Sub(q.Since) != 24*time.Hour { t.Fatalf("defaults: %+v %v", q, err) } + week, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?range=168h", nil), "YOW", evidenceKey, now) + if err != nil || !week.Until.Equal(now) || week.Until.Sub(week.Since) != 7*24*time.Hour { + t.Fatalf("server anchored range: %+v %v", week, err) + } c := api.RouteEvidenceCursor{IATA: "YOW", PathKey: evidenceKey, Since: q.Since, Until: q.Until, HeardAt: q.Since.Add(time.Hour + 123*time.Microsecond), ID: 3} next, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String())+"&limit=999", nil), "YOW", evidenceKey, now.Add(time.Hour)) if err != nil || next.Limit != 200 || !next.Until.Equal(now) || !next.Cursor.HeardAt.Equal(c.HeardAt) { @@ -35,6 +39,11 @@ func TestRouteEvidenceQuery(t *testing.T) { if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String()), nil), "YVR", evidenceKey, now); err == nil { t.Fatal("cursor crossed IATA") } + for _, query := range []string{"range=0h", "range=-1h", "range=721h", "range=1ns", "range=7d", "range=24h&range=168h", "range=24h&since=0&until=10", "range=24h&pageCursor=" + url.QueryEscape(c.String())} { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } } func TestRouteEvidenceHandler(t *testing.T) { From 85c9123e9a8c8ca6af73879ccc7993c76ac75285 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 18:27:40 -0400 Subject: [PATCH 069/141] fix: address maintainer review for #172 --- db/migrations/041_route_evidence_index.sql | 2 +- db/queries/route_evidence.sql | 1 + db/route_evidence_integration_test.go | 11 ++++++++++- db/routes.go | 4 ++-- db/sqlc/route_evidence.sql.go | 1 + internal/cache/cache_test.go | 2 +- 6 files changed, 16 insertions(+), 5 deletions(-) diff --git a/db/migrations/041_route_evidence_index.sql b/db/migrations/041_route_evidence_index.sql index 5b614da7..71088cc5 100644 --- a/db/migrations/041_route_evidence_index.sql +++ b/db/migrations/041_route_evidence_index.sql @@ -6,4 +6,4 @@ -- A compact digest narrows candidates; queries still compare the complete bytes. CREATE INDEX CONCURRENTLY idx_observations_route_evidence ON packet_observations (iata, hash_size, (decode(md5(path_bytes), 'hex')), heard_at DESC, id DESC) -WHERE path_bytes IS NOT NULL AND payload_type IS NOT NULL AND payload_type <> 9; +WHERE path_bytes IS NOT NULL AND payload_type IS NOT NULL AND payload_type <> 9 AND hop_count >= 2; diff --git a/db/queries/route_evidence.sql b/db/queries/route_evidence.sql index 960b95e7..dcc056fd 100644 --- a/db/queries/route_evidence.sql +++ b/db/queries/route_evidence.sql @@ -14,6 +14,7 @@ WHERE po.iata = @iata::bpchar AND po.hash_size = @hash_size::smallint AND decode(md5(po.path_bytes), 'hex') = @path_digest::bytea AND po.path_bytes = @path_bytes::bytea + AND po.hop_count >= 2 AND po.hop_count = @hop_count::smallint AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 AND po.heard_at >= @since::timestamptz AND po.heard_at < @until::timestamptz diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index b49224ed..f4a3e177 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -180,7 +180,7 @@ func TestRouteEvidenceIndexPostgres(t *testing.T) { t.Fatal(err) } defer func() { _, _ = conn.Exec(context.Background(), "DROP SCHEMA "+ident+" CASCADE") }() - if _, err = conn.Exec(ctx, `CREATE TABLE packet_observations (id bigint,iata char(3),hash_size smallint,path_bytes bytea,heard_at timestamptz,payload_type smallint)`); err != nil { + if _, err = conn.Exec(ctx, `CREATE TABLE packet_observations (id bigint,iata char(3),hash_size smallint,path_bytes bytea,heard_at timestamptz,payload_type smallint,hop_count smallint)`); err != nil { t.Fatal(err) } migration, err := migrationFiles.ReadFile("migrations/041_route_evidence_index.sql") @@ -202,4 +202,13 @@ func TestRouteEvidenceIndexPostgres(t *testing.T) { if err = conn.QueryRow(ctx, `SELECT indisvalid FROM pg_index WHERE indexrelid='idx_observations_route_evidence'::regclass`).Scan(&valid); err != nil || !valid { t.Fatalf("invalid index after recovery: %v", err) } + if _, err = conn.Exec(ctx, `INSERT INTO packet_observations(id,iata,hash_size,path_bytes,heard_at,payload_type,hop_count) +SELECT n,'YOW',1,'\xaabb',NOW(),4,n FROM generate_series(0,2) n; +ANALYZE packet_observations`); err != nil { + t.Fatal(err) + } + var indexed float32 + if err = conn.QueryRow(ctx, `SELECT reltuples FROM pg_class WHERE oid='idx_observations_route_evidence'::regclass`).Scan(&indexed); err != nil || indexed != 1 { + t.Fatalf("0/1-hop rows entered the route index: %v %v", indexed, err) + } } diff --git a/db/routes.go b/db/routes.go index 71820ed4..090d2a00 100644 --- a/db/routes.go +++ b/db/routes.go @@ -224,7 +224,7 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t return nil, nil } - // 5. find cross-IATA links — nodes at the boundary of source routes + // 5. find cross-IATA links — nodes at the boundary of source routes // that have neighbors in the target IATA at the start of target routes var results []api.CrossIATARoute @@ -250,7 +250,7 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t continue } if targetHops, ok := targetNodeSet[neighbor.ID]; ok { - // found a cross-IATA link — build the route + // found a cross-IATA link — build the route sourceSegment := sr.Hops[:i+1] targetSegment := extractFromNode(targetHops, neighbor.ID) diff --git a/db/sqlc/route_evidence.sql.go b/db/sqlc/route_evidence.sql.go index ab7f1ee3..0099c47b 100644 --- a/db/sqlc/route_evidence.sql.go +++ b/db/sqlc/route_evidence.sql.go @@ -49,6 +49,7 @@ WHERE po.iata = $1::bpchar AND po.hash_size = $2::smallint AND decode(md5(po.path_bytes), 'hex') = $3::bytea AND po.path_bytes = $4::bytea + AND po.hop_count >= 2 AND po.hop_count = $5::smallint AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 AND po.heard_at >= $6::timestamptz AND po.heard_at < $7::timestamptz diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 58467cee..8184a17b 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -366,7 +366,7 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { return &api.StatsOverview{TotalPackets: 42}, nil }) - // call CachedReader with reversed order — should hit same key + // call CachedReader with reversed order — should hit same key _ = cr result, err := getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func() (*api.StatsOverview, error) { calls++ From 831148ff5f403fcef9c48943390e420c00b39ae8 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:14:52 -0400 Subject: [PATCH 070/141] fix: address release review follow-ups for #172 --- README.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/README.md b/README.md index a8215298..3ecf2d55 100644 --- a/README.md +++ b/README.md @@ -570,12 +570,4 @@ The initial analytics archive migration includes unknown-type activity. Existing development previews that used the earlier draft require a separate operator repair; radio samples already discarded for those legacy rows cannot be recovered. -### Saved-route observation evidence - -Known-route responses include `pathKey`, a stable identity within the route's IATA. Use `GET /api/v1/routes/{iata}/{pathKey}/observations` to fetch the **full saved route** and retained report references. Search results can contain a subsegment while sharing the full route's key; the evidence response always describes the complete saved sequence. - -The match requires the complete saved `pathBytes`, `hashSize`, hop count and IATA. A compact digest index narrows candidates, but full bytes are still compared. Other hash widths, TRACE readings/intended routes and unclassified legacy observations are excluded. Matching short prefixes does not confirm historical node identities, forwarding or delivery. The stored route counter can include repeated processing and outlive raw reports; it is not a retained-result total. - -`range` defaults to `24h` and accepts durations up to `720h`, anchored on the server. Alternatively supply both `since` and exclusive `until` in epoch milliseconds, with a maximum 30-day span and no future end. `limit` defaults to 50 and is capped at 200. Follow `nextPageCursor` as `pageCursor`; its route, window and microsecond/ID boundary are pinned. Do not combine it with another range, or change its explicit window. Numeric legacy `cursor` is unsupported. Responses include effective window bounds, `matchAvailable`, an empty `items` array when no matching raw evidence remains, and `hasMore`; no total-count scan or packet/message body is added. Malformed saved path metadata is explicitly unavailable, and missing routes return 404. - -Migration 041 builds the compact observation index concurrently. Keep it as a single statement outside a transaction; the existing runner handles an interrupted or already-built index before recording completion. It does not alter retained rows or expiry configuration. Native PostgreSQL tests cover ties below millisecond precision, cursor scope, different widths/sites, TRACE/unknown exclusions, raw expiry, index retry and custom/generic indexed plans. +Saved-route evidence: see [the operator guide](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/app_documentation/saved-route-evidence.md). From c18e059e0df736c19d44f0f1dcb25c2eb14f8e16 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 17:59:04 -0400 Subject: [PATCH 071/141] feat(scopes): import cached regional MeshMapper catalogues --- .github/workflows/ci.yml | 2 +- cmd/beacon/main.go | 17 ++ config.yaml.example | 15 ++ db/meshmapper.go | 49 ++++ db/meshmapper_integration_test.go | 139 +++++++++++ db/migrations/042_meshmapper_scopes.sql | 17 ++ db/queries/meshmapper.sql | 27 +++ db/queries/queries.sql | 5 +- db/sqlc/meshmapper.sql.go | 94 ++++++++ db/sqlc/mock/querier.go | 29 +++ db/sqlc/models.go | 12 + db/sqlc/querier.go | 7 + db/sqlc/queries.sql.go | 5 +- docs/meshmapper-scopes.md | 26 +++ internal/cache/reader.go | 5 + internal/config/config.go | 4 + internal/config/meshmapper.go | 64 ++++++ internal/config/meshmapper_test.go | 49 ++++ internal/config/seed.go | 27 +-- internal/config/seed_test.go | 21 +- internal/ingest/packet.go | 31 ++- internal/ingest/scope_import_test.go | 95 ++++++++ internal/meshmapper/scopes.go | 294 ++++++++++++++++++++++++ internal/meshmapper/scopes_test.go | 253 ++++++++++++++++++++ internal/scopestore/scopestore.go | 22 +- 25 files changed, 1260 insertions(+), 49 deletions(-) create mode 100644 db/meshmapper.go create mode 100644 db/meshmapper_integration_test.go create mode 100644 db/migrations/042_meshmapper_scopes.sql create mode 100644 db/queries/meshmapper.sql create mode 100644 db/sqlc/meshmapper.sql.go create mode 100644 docs/meshmapper-scopes.md create mode 100644 internal/config/meshmapper.go create mode 100644 internal/config/meshmapper_test.go create mode 100644 internal/ingest/scope_import_test.go create mode 100644 internal/meshmapper/scopes.go create mode 100644 internal/meshmapper/scopes_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da907fe7..6be1defb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 8e322e6e..1fa2ea38 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -29,6 +29,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/MeshCore-Beacon/beacon-server/internal/keystore" "github.com/MeshCore-Beacon/beacon-server/internal/logging" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" "github.com/MeshCore-Beacon/beacon-server/internal/presence" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" @@ -189,6 +190,16 @@ func main() { } scopes.Load(scopeEntries) slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") + var scopeImporter *meshmapper.Importer + if cfg.MeshMapper.Scopes.Enabled { + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, scopes, scopeEntries) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper scope catalogues", "component", "startup", "error", err) + os.Exit(1) + } + } // ── Build channel keystore ────────────────────────────────────────────── entries := make(map[string][]keystore.Entry) @@ -281,6 +292,9 @@ func main() { ) if cr, ok := reader.(*cache.CachedReader); ok { + if scopeImporter != nil { + scopeImporter.SetCacheInvalidator(cr.InvalidateScopeNames) + } broker1.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver) broker2.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver) } @@ -301,6 +315,9 @@ func main() { } tasks = append(tasks, background.ObserverCleanupTask(coalescer, resolved.ObserverDeleteAfter, resolved.CleanupInterval, onDelete)) } + if scopeImporter != nil { + tasks = append(tasks, background.Task{Name: "meshmapper.scopes", Interval: meshmapper.PollInterval, Run: scopeImporter.Refresh}) + } scheduler := background.New(tasks) go scheduler.Start(ctx) diff --git a/config.yaml.example b/config.yaml.example index a0a86cc9..3e2a66fc 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -214,6 +214,21 @@ cache: # allow_countries: [CA, US] # allow_continents: [NA] +# Optional public MeshMapper scope-name discovery. Manual scopes remain authoritative. +# No API key is needed. Sources must belong to a configured region's IATAs. +# Group endpoints are not supported: their names cannot be attributed to each IATA. +# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping. +# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. +# Refresh success/failure/freshness is logged under component=meshmapper.scopes. +# Removing/turning off a source deactivates its imported matching keys on restart; +# historical scope identities and recorded evidence are retained. +#meshmapper: +# scopes: +# enabled: false +# refresh_interval: 1h # 5m-24h; one source checked per 15s tick +# sources: +# YOW: https://yow.meshmapper.net/get_scopes.php + # Background task intervals. # Shorter intervals are useful during initial deployment to confirm data is # flowing. Back off to 1h or more once stable. diff --git a/db/meshmapper.go b/db/meshmapper.go new file mode 100644 index 00000000..3ddbb28b --- /dev/null +++ b/db/meshmapper.go @@ -0,0 +1,49 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetScopeCatalogue(ctx context.Context, iata, url string) (*meshmapper.Cache, error) { + row, err := s.q.GetScopeCatalogue(ctx, sqlc.GetScopeCatalogueParams{Iata: iata, Url: url}) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + cache := &meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError} + if row.Etag != nil { + cache.ETag = *row.Etag + } + return cache, nil +} + +func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, entries []scopestore.Entry) error { + params := sqlc.SaveScopeCatalogueParams{ + Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError, + CheckedAt: pgtype.Timestamptz{Time: cache.CheckedAt, Valid: !cache.CheckedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: cache.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: cache.NextAttempt, Valid: true}, + Names: []string{}, Keys: [][]byte{}, Fingerprints: [][]byte{}, + } + if !cache.CheckedAt.IsZero() { + params.Etag = &cache.ETag + } + for _, entry := range entries { + params.Names = append(params.Names, entry.Name) + params.Keys = append(params.Keys, entry.TransportKey) + params.Fingerprints = append(params.Fingerprints, entry.KeyFingerprint) + } + return s.q.SaveScopeCatalogue(ctx, params) +} diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go new file mode 100644 index 00000000..06d71096 --- /dev/null +++ b/db/meshmapper_integration_test.go @@ -0,0 +1,139 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestMeshMapperCataloguePostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"transport_scopes", "meshmapper_scope_catalogues"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + manual := scopestore.FromName("manual") + manual.TransportKey[0] ^= 1 + if err := store.UpsertTransportScope(ctx, manual.Name, "Operator label", manual.TransportKey, manual.KeyFingerprint); err != nil { + t.Fatal(err) + } + url := "https://yow.meshmapper.net/get_scopes.php" + if row, err := store.GetScopeCatalogue(ctx, "YOW", url); err != nil || row != nil { + t.Fatal(row, err) + } + now := time.Now().UTC().Truncate(time.Second) + payload := []byte(`{"generated_at":"2026-09-27T16:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":3,"scopes":[{"name":"yow","repeaters":0,"default":0,"monitored":true,"wardriving":false}]}`) + update := meshmapper.Cache{Payload: payload, ETag: `"v1"`, CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(time.Hour)} + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("manual"), scopestore.FromName("yow")}); err != nil { + t.Fatal(err) + } + manualRows, err := store.GetTransportScopes(ctx) + if err != nil || len(manualRows) != 1 || manualRows[0].TransportKey[0] != manual.TransportKey[0] { + t.Fatal("manual key changed", manualRows, err) + } + var label string + var imported bool + if err := tx.QueryRow(ctx, "SELECT display_name,imported_only FROM transport_scopes WHERE name='#manual'").Scan(&label, &imported); err != nil || label != "Operator label" || imported { + t.Fatal(label, imported, err) + } + row, err := store.GetScopeCatalogue(ctx, "YOW", url) + if err != nil || row.ETag != `"v1"` || !row.CheckedAt.Equal(now) { + t.Fatal(row, err) + } + scopes := scopestore.New() + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + if err != nil || len(scopes.Entries()) != 2 { + t.Fatal("restart did not restore imported membership", err) + } + // A conditional success preserves the document, advances checked time and clears a failure. + checked := now.Add(time.Hour) + if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{ETag: `"v1"`, CheckedAt: checked, AttemptedAt: checked, NextAttempt: checked.Add(time.Hour)}, nil); err != nil { + t.Fatal(err) + } + if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{AttemptedAt: checked.Add(time.Minute), NextAttempt: checked.Add(2 * time.Hour), LastError: "HTTP 429"}, nil); err != nil { + t.Fatal(err) + } + row, err = store.GetScopeCatalogue(ctx, "YOW", url) + if err != nil || !row.CheckedAt.Equal(checked) || row.ETag != `"v1"` || row.LastError != "HTTP 429" || len(row.Payload) == 0 { + t.Fatal("failed refresh lost durable snapshot", row, err) + } + // Empty membership deactivates matching but retains the old scope identity for historical rows. + empty := []byte(`{"generated_at":"2026-09-27T17:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":0,"scopes":[]}`) + update.Payload = empty + update.ETag = "" + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, nil); err != nil { + t.Fatal(err) + } + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + if err != nil || len(scopes.Entries()) != 1 { + t.Fatal("removed imported membership remained active", err) + } + if _, err := store.GetTransportScopeByName(ctx, "#yow"); err != nil { + t.Fatal("deleted historical identity", err) + } + row, _ = store.GetScopeCatalogue(ctx, "YOW", url) + if row.ETag != "" { + t.Fatal("200 without ETag retained obsolete validator") + } + // Reconfiguring a known imported name as manual promotes it without losing its identity. + yow := scopestore.FromName("yow") + if err := store.UpsertTransportScope(ctx, yow.Name, "Local", yow.TransportKey, yow.KeyFingerprint); err != nil { + t.Fatal(err) + } + manualRows, err = store.GetTransportScopes(ctx) + if err != nil || len(manualRows) != 2 { + t.Fatal("manual promotion failed", err) + } + // A failed cache write also rolls back the identity insertion in the same statement. + if _, err := tx.Exec(ctx, "ALTER TABLE meshmapper_scope_catalogues ADD CHECK(last_error <> 'force-failure'); SAVEPOINT rejected"); err != nil { + t.Fatal(err) + } + update.LastError = "force-failure" + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("uncommitted")}); err == nil { + t.Fatal("expected constraint failure") + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT rejected"); err != nil { + t.Fatal(err) + } + var count int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM transport_scopes WHERE name='#uncommitted'").Scan(&count); err != nil || count != 0 { + t.Fatal("partially committed catalogue", count, err) + } +} diff --git a/db/migrations/042_meshmapper_scopes.sql b/db/migrations/042_meshmapper_scopes.sql new file mode 100644 index 00000000..43b48b6c --- /dev/null +++ b/db/migrations/042_meshmapper_scopes.sql @@ -0,0 +1,17 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Preserve historical identities while distinguishing import-only matching keys. +ALTER TABLE transport_scopes ADD COLUMN IF NOT EXISTS imported_only BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE TABLE IF NOT EXISTS meshmapper_scope_catalogues ( + iata TEXT NOT NULL, + url TEXT NOT NULL, + payload JSONB, + etag TEXT, + checked_at TIMESTAMPTZ, + attempted_at TIMESTAMPTZ NOT NULL, + next_attempt TIMESTAMPTZ NOT NULL, + last_error TEXT NOT NULL DEFAULT '', + PRIMARY KEY (iata, url) +); diff --git a/db/queries/meshmapper.sql b/db/queries/meshmapper.sql new file mode 100644 index 00000000..16e82433 --- /dev/null +++ b/db/queries/meshmapper.sql @@ -0,0 +1,27 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: GetScopeCatalogue :one +SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; + +-- name: SaveScopeCatalogue :exec +-- One statement commits the validated snapshot and its lookup identities together. +-- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +-- after an error or 304. Imported names never replace existing manual metadata. +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest(@names::text[]) AS name, unnest(@keys::bytea[]) AS key, + unnest(@fingerprints::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 05ceacde..37cd8fa2 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -49,10 +49,11 @@ VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO UPDATE SET display_name = EXCLUDED.display_name, transport_key = EXCLUDED.transport_key, - key_fingerprint = EXCLUDED.key_fingerprint; + key_fingerprint = EXCLUDED.key_fingerprint, + imported_only = FALSE; -- name: GetTransportScopes :many -SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name; +SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name; -- name: GetTransportScopeByName :one SELECT id FROM transport_scopes WHERE name = $1; diff --git a/db/sqlc/meshmapper.sql.go b/db/sqlc/meshmapper.sql.go new file mode 100644 index 00000000..b8ba12d9 --- /dev/null +++ b/db/sqlc/meshmapper.sql.go @@ -0,0 +1,94 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: meshmapper.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getScopeCatalogue = `-- name: GetScopeCatalogue :one + +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2 +` + +type GetScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` +} + +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +func (q *Queries) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) { + row := q.db.QueryRow(ctx, getScopeCatalogue, arg.Iata, arg.Url) + var i MeshmapperScopeCatalogue + err := row.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ) + return i, err +} + +const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest($9::text[]) AS name, unnest($10::bytea[]) AS key, + unnest($11::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` + Names []string `json:"names"` + Keys [][]byte `json:"keys"` + Fingerprints [][]byte `json:"fingerprints"` +} + +// One statement commits the validated snapshot and its lookup identities together. +// Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +// after an error or 304. Imported names never replace existing manual metadata. +func (q *Queries) SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error { + _, err := q.db.Exec(ctx, saveScopeCatalogue, + arg.Iata, + arg.Url, + arg.Payload, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + arg.Names, + arg.Keys, + arg.Fingerprints, + ) + return err +} diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 9d3fb184..16ea0736 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -714,6 +714,21 @@ func (mr *MockQuerierMockRecorder) GetScopeByName(ctx, name any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeByName", reflect.TypeOf((*MockQuerier)(nil).GetScopeByName), ctx, name) } +// GetScopeCatalogue mocks base method. +func (m *MockQuerier) GetScopeCatalogue(ctx context.Context, arg db.GetScopeCatalogueParams) (db.MeshmapperScopeCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetScopeCatalogue", ctx, arg) + ret0, _ := ret[0].(db.MeshmapperScopeCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetScopeCatalogue indicates an expected call of GetScopeCatalogue. +func (mr *MockQuerierMockRecorder) GetScopeCatalogue(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).GetScopeCatalogue), ctx, arg) +} + // GetScopeNames mocks base method. func (m *MockQuerier) GetScopeNames(ctx context.Context) ([]string, error) { m.ctrl.T.Helper() @@ -1542,6 +1557,20 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } +// SaveScopeCatalogue mocks base method. +func (m *MockQuerier) SaveScopeCatalogue(ctx context.Context, arg db.SaveScopeCatalogueParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveScopeCatalogue", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveScopeCatalogue indicates an expected call of SaveScopeCatalogue. +func (mr *MockQuerierMockRecorder) SaveScopeCatalogue(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveScopeCatalogue), ctx, arg) +} + // SearchKnownRoutes mocks base method. func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 47cce44e..86fe98ea 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -234,6 +234,17 @@ type KnownRoute struct { LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"` } +type MeshmapperScopeCatalogue struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + type MvHourlyIataStat struct { Iata string `json:"iata"` Hour pgtype.Timestamptz `json:"hour"` @@ -528,4 +539,5 @@ type TransportScope struct { TransportKey []byte `json:"transport_key"` KeyFingerprint []byte `json:"key_fingerprint"` CreatedAt pgtype.Timestamptz `json:"created_at"` + ImportedOnly bool `json:"imported_only"` } diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index a5770646..f96b9a36 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -89,6 +89,9 @@ type Querier interface { GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) + // Copyright 2026 Beacon Contributors + // SPDX-License-Identifier: AGPL-3.0-or-later + GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) GetScopeNames(ctx context.Context) ([]string, error) // Aggregate matching observations once, separately from node memberships to avoid // a cross-join. Empty IATAs keep the original global counts, including associations @@ -250,6 +253,10 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + // One statement commits the validated snapshot and its lookup identities together. + // Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data + // after an error or 304. Imported names never replace existing manual metadata. + SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 054bc001..9195a629 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2051,7 +2051,7 @@ func (q *Queries) GetTransportScopeByName(ctx context.Context, name string) (int } const getTransportScopes = `-- name: GetTransportScopes :many -SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name +SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name ` type GetTransportScopesRow struct { @@ -5228,7 +5228,8 @@ VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO UPDATE SET display_name = EXCLUDED.display_name, transport_key = EXCLUDED.transport_key, - key_fingerprint = EXCLUDED.key_fingerprint + key_fingerprint = EXCLUDED.key_fingerprint, + imported_only = FALSE ` type UpsertTransportScopeParams struct { diff --git a/docs/meshmapper-scopes.md b/docs/meshmapper-scopes.md new file mode 100644 index 00000000..7c9a3916 --- /dev/null +++ b/docs/meshmapper-scopes.md @@ -0,0 +1,26 @@ +# Optional MeshMapper scope discovery + +Beacon can periodically read the [public Scopes API](https://wiki.meshmapper.net/scopes-api/) to discover additional transport scope names for configured regions. Manual `scopes` remain authoritative. This does not supply channel decryption keys or per-repeater records. + +```yaml +meshmapper: + scopes: + enabled: true + refresh_interval: 1h + sources: + YOW: https://yow.meshmapper.net/get_scopes.php +``` + +The IATA must already occur in `regions[].iatas`. Set the exact published regional HTTPS URL; credentials, query strings, other hosts and redirects are rejected. The response must identify the same region and exactly one zone. Group catalogues are rejected because they cannot assign each scope/count to individual member regions. No API key is required. + +The default is disabled. Refresh defaults to one hour and accepts 5 minutes through 24 hours. One source is considered every 15 seconds, keeping requests below four per minute per process. Multiple deployments share MeshMapper's per-IP allowance. ETags reduce transfer, HTTP 304 advances the last successful check, and 429 pauses all sources until the later of the normal retry time and `Retry-After`. The pause survives restarts. HTTP 503 also honors that source's `Retry-After`. + +Responses are limited to 64 KiB and 64 names per source, with at most 16 sources. Names are case-sensitive and at most 128 bytes before Beacon's usual `#` normalization; existing `#`/`$` prefixes are retained. Zero-count monitored names are kept. Imported matching candidates are limited to their source IATA; manual keys keep their existing global applicability. A short transport code that matches multiple candidate names remains unresolved. + +Migration 042 stores the last validated response, validator and check/retry timestamps in PostgreSQL. Saving a response and its scope identities is atomic. Timeouts, invalid responses and HTTP errors retain the prior catalogue; startup restores it without HTTP. A successful empty response removes that source's active imported membership. Removing or disabling a source takes effect on restart. Scope identities referenced by historical records are retained. Adding an imported name to the manual list promotes it to manual ownership. + +Operator logs under `component=meshmapper.scopes` show the IATA, source, active name count, source generation time, last successful check, next attempt and last error. An empty last error means the latest request succeeded; a zero checked time means no successful response has been stored. A 304 retains the original source generation time because MeshMapper excludes that timestamp from its ETag. Cached names may be old during an outage; use the check/error fields to assess freshness. + +Catalogue counts remain source metadata. They never create Beacon packet counts, observer associations or node defaults. Actual packet-code matching supplies those associations through the existing ingest path. New names affect subsequent packets; no historical scan is started. Channel message tags and import-status UI are separate follow-ups. Source corruption or an unavailable catalogue database at startup is reported as a startup failure; disabling the optional importer restores manual-only behavior without deleting saved data. + +Validate and back up PostgreSQL before applying the migration. Rollback to an older server must restore its matching pre-import database and configuration as well as its binary; an older matcher does not understand imported-only ownership or regional candidate limits. diff --git a/internal/cache/reader.go b/internal/cache/reader.go index a570df9e..515adc93 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -80,6 +80,11 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { } } +// InvalidateScopeNames makes newly committed catalogue names available to filters. +func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { + cr.c.del(ctx, keyScopeNames) +} + // InvalidateNode removes the cached entries for a node by UUID. // Should be called from the ingest path after a node upsert. func (cr *CachedReader) InvalidateNode(ctx context.Context, nodeID uuid.UUID) { diff --git a/internal/config/config.go b/internal/config/config.go index b3c6cd3f..b8b8fff8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -33,6 +33,7 @@ type Config struct { Routes RoutesConfig `yaml:"routes"` Ingest IngestFilterConfig `yaml:"ingest"` Scopes []ScopeConfig `yaml:"scopes"` + MeshMapper MeshMapperConfig `yaml:"meshmapper"` Cache CacheConfig `yaml:"cache"` CORS CORSConfig `yaml:"cors"` RateLimit RateLimitConfig `yaml:"ratelimit"` @@ -408,6 +409,9 @@ func Load(path string) (*Config, error) { } } configDir := filepath.Dir(path) + if err := cfg.validateMeshMapper(); err != nil { + return nil, err + } for iata, details := range cfg.IATAs { if details.BorderFile != "" && !filepath.IsAbs(details.BorderFile) { details.BorderFile = filepath.Join(configDir, details.BorderFile) diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go new file mode 100644 index 00000000..5f367747 --- /dev/null +++ b/internal/config/meshmapper.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "fmt" + "net/url" + "regexp" + "time" +) + +// MaxScopeSources bounds both network refresh work and imported matching keys. +const MaxScopeSources = 16 + +type MeshMapperConfig struct { + Scopes MeshMapperScopesConfig `yaml:"scopes"` +} + +// MeshMapperScopesConfig augments, but never replaces, the manual scopes list. +type MeshMapperScopesConfig struct { + Enabled bool `yaml:"enabled"` + Sources map[string]string `yaml:"sources"` // configured IATA -> published regional endpoint + RefreshInterval duration `yaml:"refresh_interval"` +} + +func (c MeshMapperScopesConfig) Interval() time.Duration { + if c.RefreshInterval.Duration == 0 { + return time.Hour + } + return c.RefreshInterval.Duration +} + +var meshMapperHost = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) + +func (c *Config) validateMeshMapper() error { + s := c.MeshMapper.Scopes + if !s.Enabled { + return nil + } + if len(s.Sources) == 0 || len(s.Sources) > MaxScopeSources { + return fmt.Errorf("meshmapper.scopes.sources must contain 1-%d regional endpoints", MaxScopeSources) + } + if s.Interval() < 5*time.Minute || s.Interval() > 24*time.Hour { + return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 5m and 24h") + } + configured := map[string]bool{} + for _, region := range c.Regions { + for _, iata := range region.IATAs { + configured[iata] = true + } + } + for iata, endpoint := range s.Sources { + if !configured[iata] { + return fmt.Errorf("meshmapper.scopes source %q must belong to a configured region", iata) + } + u, err := url.Parse(endpoint) + if err != nil || u.Scheme != "https" || u.User != nil || !meshMapperHost.MatchString(u.Host) || + u.Path != "/get_scopes.php" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return fmt.Errorf("meshmapper.scopes source %q must be a published https://.meshmapper.net/get_scopes.php endpoint without credentials or parameters", iata) + } + } + return nil +} diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go new file mode 100644 index 00000000..56a56345 --- /dev/null +++ b/internal/config/meshmapper_test.go @@ -0,0 +1,49 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestMeshMapperConfig(t *testing.T) { + valid := "regions:\n - slug: ottawa\n iatas: [YOW]\nmeshmapper:\n scopes:\n enabled: true\n sources:\n YOW: https://yow.meshmapper.net/get_scopes.php\n" + for _, tc := range []struct { + name, text string + bad bool + }{ + {"valid", valid, false}, + {"unknown region", strings.Replace(valid, "iatas: [YOW]", "iatas: [YVR]", 1), true}, + {"http", strings.Replace(valid, "https:", "http:", 1), true}, + {"private host", strings.Replace(valid, "yow.meshmapper.net", "127.0.0.1", 1), true}, + {"credentials", strings.Replace(valid, "https://", "https://secret@", 1), true}, + {"query", strings.Replace(valid, "get_scopes.php", "get_scopes.php?key=secret", 1), true}, + {"non API", strings.Replace(valid, "get_scopes.php", "index.php", 1), true}, + {"fast", valid + " refresh_interval: 1s\n", true}, + {"negative", valid + " refresh_interval: -1h\n", true}, + {"disabled", strings.Replace(valid, "enabled: true", "enabled: false", 1), false}, + } { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(p) + if (err != nil) != tc.bad { + t.Fatalf("error=%v", err) + } + if err == nil && cfg.MeshMapper.Scopes.Interval() != time.Hour { + t.Fatal("wrong default") + } + }) + } + cfg := Config{MeshMapper: MeshMapperConfig{Scopes: MeshMapperScopesConfig{Enabled: true}}} + if err := cfg.validateMeshMapper(); err == nil { + t.Fatal("empty sources accepted") + } +} diff --git a/internal/config/seed.go b/internal/config/seed.go index 53ea3716..a4195730 100644 --- a/internal/config/seed.go +++ b/internal/config/seed.go @@ -5,12 +5,12 @@ package config import ( "context" - "crypto/sha256" "encoding/json" "fmt" "log/slog" "os" - "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) // Seeder is the database interface required to seed config data on startup. @@ -63,29 +63,10 @@ func Seed(ctx context.Context, cfg *Config, db Seeder) error { } // Transport Codes for _, s := range cfg.Scopes { - name := normalizeScopeName(s.Name) - key := deriveScopeKey(name) - h := sha256.Sum256(key) - fingerprint := h[:8] - if err := db.UpsertTransportScope(ctx, name, "", key, fingerprint); err != nil { + entry := scopestore.FromName(s.Name) + if err := db.UpsertTransportScope(ctx, entry.Name, "", entry.TransportKey, entry.KeyFingerprint); err != nil { return err } } return nil } - -// normalizeScopeName ensures the scope name has a # or $ prefix. -// Plain names get # prepended: "bc" → "#bc". -func normalizeScopeName(name string) string { - if strings.HasPrefix(name, "#") || strings.HasPrefix(name, "$") { - return name - } - return "#" + name -} - -// deriveScopeKey derives the 16-byte transport key from a normalized scope name. -// key = SHA256(name)[:16] -func deriveScopeKey(name string) []byte { - h := sha256.Sum256([]byte(name)) - return h[:16] -} diff --git a/internal/config/seed_test.go b/internal/config/seed_test.go index 4f2fa0c0..069bbe38 100644 --- a/internal/config/seed_test.go +++ b/internal/config/seed_test.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" "os" "testing" ) @@ -198,39 +199,39 @@ func TestSeed_DBError(t *testing.T) { } func TestNormalizeScopeName_WithHash(t *testing.T) { - if normalizeScopeName("#bc") != "#bc" { + if scopestore.FromName("#bc").Name != "#bc" { t.Error("expected #bc unchanged") } } func TestNormalizeScopeName_WithDollar(t *testing.T) { - if normalizeScopeName("$bc") != "$bc" { + if scopestore.FromName("$bc").Name != "$bc" { t.Error("expected $bc unchanged") } } func TestNormalizeScopeName_WithoutPrefix(t *testing.T) { - if normalizeScopeName("bc") != "#bc" { + if scopestore.FromName("bc").Name != "#bc" { t.Error("expected bc to become #bc") } } func TestNormalizeScopeName_Empty(t *testing.T) { - if normalizeScopeName("") != "#" { + if scopestore.FromName("").Name != "#" { t.Error("expected empty string to become #") } } func TestDeriveScopeKey_Length(t *testing.T) { - key := deriveScopeKey("#bc") + key := scopestore.FromName("#bc").TransportKey if len(key) != 16 { t.Errorf("expected 16 bytes, got %d", len(key)) } } func TestDeriveScopeKey_Deterministic(t *testing.T) { - a := deriveScopeKey("#bc") - b := deriveScopeKey("#bc") + a := scopestore.FromName("#bc").TransportKey + b := scopestore.FromName("#bc").TransportKey if hex.EncodeToString(a) != hex.EncodeToString(b) { t.Error("expected same key for same input") } @@ -238,7 +239,7 @@ func TestDeriveScopeKey_Deterministic(t *testing.T) { func TestDeriveScopeKey_KnownValue(t *testing.T) { // SHA256("#bc")[:16] — pin the exact derivation so changes are caught - key := deriveScopeKey("#bc") + key := scopestore.FromName("#bc").TransportKey got := hex.EncodeToString(key) // generate this once: echo -n "#bc" | sha256sum | cut -c1-32 const want = "84509cfe73d94f7f6a8299e6bcdb8a3c" @@ -248,8 +249,8 @@ func TestDeriveScopeKey_KnownValue(t *testing.T) { } func TestDeriveScopeKey_DifferentInputs(t *testing.T) { - a := deriveScopeKey("#bc") - b := deriveScopeKey("#other") + a := scopestore.FromName("#bc").TransportKey + b := scopestore.FromName("#other").TransportKey if hex.EncodeToString(a) == hex.EncodeToString(b) { t.Error("expected different keys for different inputs") } diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index bcc9ca58..5dfe7d7c 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -11,11 +11,13 @@ import ( "encoding/hex" "encoding/json" "fmt" + "slices" "strings" "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/lora" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" "github.com/google/uuid" "github.com/meshcore-go/meshcore-go" ) @@ -694,14 +696,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ var matchedScope *string if packet.RouteType() == meshcore.RouteTypeTransportFlood || packet.RouteType() == meshcore.RouteTypeTransportDirect { - for _, entry := range w.scopes.Entries() { - code := computeTransportCode(entry.TransportKey, packet.PayloadType(), packet.Payload) - if code == packet.TransportCode1 { - s := entry.Name - matchedScope = &s - break - } - } + matchedScope = matchTransportScope(w.scopes.Entries(), iata, packet.PayloadType(), packet.Payload, packet.TransportCode1) } var scopeID *int32 if matchedScope != nil { @@ -930,6 +925,26 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } } +// matchTransportScope never treats catalogue membership as forwarding evidence. +// Multiple candidate names matching the short code are ambiguous, not first-wins. +func matchTransportScope(entries []scopestore.Entry, iata string, payloadType uint8, payload []byte, code uint16) *string { + var matched *string + for _, entry := range entries { + if entry.IATAs != nil && !slices.Contains(entry.IATAs, iata) { + continue + } + if computeTransportCode(entry.TransportKey, payloadType, payload) != code { + continue + } + if matched != nil && *matched != entry.Name { + return nil + } + name := entry.Name + matched = &name + } + return matched +} + // computeTransportCode derives transport_code_1 from a transport key and packet payload. // code = HMAC-SHA256(key, payload_type_byte || payload)[0:2] as little-endian uint16. // Coerces reserved values 0x0000 → 0x0001 and 0xFFFF → 0xFFFE per §2.4. diff --git a/internal/ingest/scope_import_test.go b/internal/ingest/scope_import_test.go new file mode 100644 index 00000000..3c085e41 --- /dev/null +++ b/internal/ingest/scope_import_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "fmt" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/meshcore-go/meshcore-go" +) + +func TestTransportScopeRegionalAndAmbiguousMatches(t *testing.T) { + entry := scopestore.FromName("yow") + entry.IATAs = []string{"YOW"} + payload := []byte{0xde, 0xad, 0xbe, 0xef} + // Fixed HMAC-SHA256 fixture: SHA256("#yow")[:16], payload type 4. + if got := matchTransportScope([]scopestore.Entry{entry}, "YOW", 4, payload, 21698); got == nil || *got != "#yow" { + t.Fatal(got) + } + if got := matchTransportScope([]scopestore.Entry{entry}, "YVR", 4, payload, 21698); got != nil { + t.Fatal("imported key escaped source region") + } + entry.IATAs = nil + if got := matchTransportScope([]scopestore.Entry{entry}, "YVR", 4, payload, 21698); got == nil { + t.Fatal("manual key was region-restricted") + } + // Distinct keys with the same short code for this payload must remain unknown. + a, b := scopestore.FromName("collision-320"), scopestore.FromName("collision-322") + for _, entries := range [][]scopestore.Entry{{a, b}, {b, a}} { + if computeTransportCode(entries[0].TransportKey, 4, payload) != 37018 { + t.Fatal("collision fixture changed") + } + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got != nil { + t.Fatal("ambiguous short code resolved", *got) + } + } +} + +type importCaptureDB struct{ *frameCaptureDB } + +func (s *importCaptureDB) GetTransportScopeByName(context.Context, string) (int32, error) { + return 123, nil +} + +func TestImportedScopeFlowsThroughPacketIngest(t *testing.T) { + for _, iata := range []string{"YOW", "YVR"} { + t.Run(iata, func(t *testing.T) { + w, base := newTestWorker() + capture := &importCaptureDB{&frameCaptureDB{stubDB: base}} + w.db = capture + key := scopestore.FromName("yow") + key.IATAs = []string{"YOW"} + store := scopestore.New() + store.Load([]scopestore.Entry{key}) + w.scopes = store + packet := buildGrpTxtPacket(t, 0x1a, make([]byte, 16)) + packet.Header = meshcore.MakeHeader(meshcore.RouteTypeTransportFlood, meshcore.PayloadTypeGrpTxt, 0) + packet.TransportCode1 = computeTransportCode(key.TransportKey, packet.PayloadType(), packet.Payload) + w.handlePacket(context.Background(), iata, "0102", packetEnvelope(t, packet)) + if len(capture.packets) != 1 { + t.Fatal("packet not stored") + } + got := capture.packets[0].ScopeID + if iata == "YOW" && (got == nil || *got != 123) { + t.Fatal("scope not persisted", got) + } + if iata == "YVR" && got != nil { + t.Fatal("scope assigned outside source region", got) + } + }) + } +} + +func BenchmarkTransportScopeCatalogue(b *testing.B) { + for _, count := range []int{0, 64, 1024} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + entries := []scopestore.Entry{scopestore.FromName("manual")} + for n := 0; n < count; n++ { + entry := scopestore.FromName(fmt.Sprintf("scope-%d", n)) + entry.IATAs = []string{fmt.Sprintf("R%02d", n/64)} + entries = append(entries, entry) + } + store := scopestore.New() + store.Load(entries) + payload := make([]byte, 128) + b.ReportAllocs() + for b.Loop() { + matchTransportScope(store.Entries(), "R00", 4, payload, 1234) + } + }) + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go new file mode 100644 index 00000000..a8b64cee --- /dev/null +++ b/internal/meshmapper/scopes.go @@ -0,0 +1,294 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package meshmapper imports the published regional scope catalogue, off the ingest path. +package meshmapper + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "slices" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +const ( + MaxScopes = 64 // per source; at most 16 configured sources (1024 imported identities) + MaxBody = 64 << 10 + PollInterval = 15 * time.Second // one source per tick, at most four requests/minute +) + +// Cache stores source provenance and freshness separately from packet evidence. +type Cache struct { + Payload json.RawMessage + ETag string + CheckedAt, AttemptedAt, NextAttempt time.Time + LastError string +} + +type Store interface { + GetScopeCatalogue(context.Context, string, string) (*Cache, error) + SaveScopeCatalogue(context.Context, string, string, Cache, []scopestore.Entry) error +} + +type source struct { + iata, url string + cache Cache + entries []scopestore.Entry + generated time.Time +} + +// Importer is owned by one background task; ScopeStore synchronizes its consumers. +type Importer struct { + store Store + scopes *scopestore.ScopeStore + manual []scopestore.Entry + sources []source + interval time.Duration + client *http.Client + retryAfter time.Time + onChange func(context.Context) +} + +// SetCacheInvalidator is wired once at startup, before the background task starts. +func (i *Importer) SetCacheInvalidator(fn func(context.Context)) { i.onChange = fn } + +// New restores validated snapshots before ingestion, without making HTTP requests. +func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) { + i := &Importer{store: store, scopes: scopes, manual: manual, interval: cfg.Interval(), client: &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} + if !cfg.Enabled { + return i, nil + } + keys := make([]string, 0, len(cfg.Sources)) + for key := range cfg.Sources { + keys = append(keys, key) + } + slices.Sort(keys) + for _, key := range keys { + s := source{iata: key, url: cfg.Sources[key]} + cached, err := store.GetScopeCatalogue(ctx, s.iata, s.url) + if err != nil { + return nil, fmt.Errorf("restore scope catalogue %s: %w", key, err) + } + if cached != nil { + s.cache = *cached + if len(cached.Payload) > 0 { + s.entries, s.generated, err = decode(cached.Payload, key) + if err != nil { + return nil, fmt.Errorf("invalid saved scope catalogue %s: %w", key, err) + } + } + if cached.LastError == "HTTP 429" && cached.NextAttempt.After(i.retryAfter) { + i.retryAfter = cached.NextAttempt + } + } + i.sources = append(i.sources, s) + i.log(s, "restored") + } + i.publish() + return i, nil +} + +// Refresh checks only one due source. The scheduler serializes calls every 15s. +func (i *Importer) Refresh(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + now := time.Now().UTC() + if now.Before(i.retryAfter) { + return nil + } + for n := range i.sources { + s := &i.sources[n] + if now.Before(s.cache.NextAttempt) { + continue + } + return i.refresh(ctx, s, now) + } + return nil +} + +func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error { + update := Cache{AttemptedAt: now, NextAttempt: now.Add(i.interval)} + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil) + if err != nil { + return err + } + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", "Beacon-MeshMapper-Scopes/1") + if len(s.cache.Payload) > 0 && s.cache.ETag != "" { + request.Header.Set("If-None-Match", s.cache.ETag) + } + response, err := i.client.Do(request) + var entries []scopestore.Entry + var generated time.Time + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + update.LastError = "request failed" // do not persist untrusted response bodies or URLs from errors + } else { + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + body, readErr := io.ReadAll(io.LimitReader(response.Body, MaxBody+1)) + if readErr == nil { + entries, generated, readErr = decode(body, s.iata) + } + if readErr != nil { + update.LastError = "invalid response" + } else { + update.Payload = body + } + case http.StatusNotModified: + if len(s.cache.Payload) == 0 || s.cache.ETag == "" { + update.LastError = "304 without cached catalogue" + } + default: + update.LastError = fmt.Sprintf("HTTP %d", response.StatusCode) + if response.StatusCode == http.StatusTooManyRequests || response.StatusCode == http.StatusServiceUnavailable { + // The published contract uses seconds. Ignore invalid/overflowing headers. + if seconds, parseErr := strconv.ParseInt(response.Header.Get("Retry-After"), 10, 64); parseErr == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) { + until := now.Add(time.Duration(seconds) * time.Second) + if until.After(update.NextAttempt) { + update.NextAttempt = until + } + } + } + } + if update.LastError == "" { + etag := response.Header.Get("ETag") + if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") { + update.LastError = "invalid ETag" + update.Payload = nil + entries = nil + } else { + update.ETag = etag + if response.StatusCode == http.StatusNotModified && etag == "" { + update.ETag = s.cache.ETag + } + update.CheckedAt = now + } + } + } + if update.LastError == "HTTP 429" { + i.retryAfter = update.NextAttempt + } + if err := i.store.SaveScopeCatalogue(ctx, s.iata, s.url, update, entries); err != nil { + // Avoid a retry every scheduler tick during a DB outage; never publish uncommitted keys. + s.cache.NextAttempt = update.NextAttempt + return fmt.Errorf("persist scope catalogue %s: %w", s.iata, err) + } + if len(update.Payload) > 0 { + s.cache.Payload = update.Payload + s.entries = entries + s.generated = generated + } + if !update.CheckedAt.IsZero() { + s.cache.CheckedAt = update.CheckedAt + s.cache.ETag = update.ETag + } + s.cache.AttemptedAt = now + s.cache.NextAttempt = update.NextAttempt + s.cache.LastError = update.LastError + if len(update.Payload) > 0 { + i.publish() + if i.onChange != nil { + i.onChange(ctx) + } + } + i.log(*s, "checked") + return nil +} + +func (i *Importer) publish() { + byName := make(map[string]scopestore.Entry, len(i.manual)) + for _, e := range i.manual { + byName[e.Name] = e + } + for _, s := range i.sources { + for _, candidate := range s.entries { + entry, exists := byName[candidate.Name] + if exists && entry.IATAs == nil { + continue + } // manual metadata/keys win + if !exists { + entry = candidate + } + entry.IATAs = append(entry.IATAs, s.iata) + byName[entry.Name] = entry + } + } + names := make([]string, 0, len(byName)) + for name := range byName { + names = append(names, name) + } + slices.Sort(names) + entries := make([]scopestore.Entry, 0, len(names)) + for _, name := range names { + entries = append(entries, byName[name]) + } + i.scopes.Load(entries) +} + +func (i *Importer) log(s source, action string) { + level := slog.LevelInfo + if s.cache.LastError != "" { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, "MeshMapper scopes "+action, "component", "meshmapper.scopes", "iata", s.iata, "source", s.url, + "names", len(s.entries), "generated_at", s.generated, "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError) +} + +func decode(body []byte, iata string) ([]scopestore.Entry, time.Time, error) { + var document struct { + GeneratedAt time.Time `json:"generated_at"` + Region string `json:"region"` + Zones []string `json:"zones"` + Repeaters, Scoped *int + Scopes *[]struct { + Name string + Repeaters, Default *int + Monitored, Wardriving *bool + } + } + if len(body) > MaxBody || !utf8.Valid(body) { + return nil, time.Time{}, fmt.Errorf("invalid catalogue size/encoding") + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, time.Time{}, err + } + if document.GeneratedAt.IsZero() || document.Region != iata || len(document.Zones) != 1 || document.Zones[0] != iata || + document.Repeaters == nil || document.Scoped == nil || *document.Scoped < 0 || *document.Repeaters < *document.Scoped || + document.Scopes == nil || len(*document.Scopes) > MaxScopes { + return nil, time.Time{}, fmt.Errorf("invalid regional catalogue") + } + entries := make([]scopestore.Entry, 0, len(*document.Scopes)) + seen := map[string]bool{} + for _, s := range *document.Scopes { + if s.Name == "" || len(s.Name) > 128 || strings.TrimSpace(s.Name) != s.Name || strings.ContainsFunc(s.Name, unicode.IsControl) || + s.Repeaters == nil || s.Default == nil || s.Monitored == nil || s.Wardriving == nil || + *s.Default < 0 || *s.Repeaters < *s.Default || *s.Repeaters > *document.Scoped { + return nil, time.Time{}, fmt.Errorf("invalid scope entry") + } + entry := scopestore.FromName(s.Name) + if len(entry.Name) < 2 || seen[entry.Name] { + return nil, time.Time{}, fmt.Errorf("empty or duplicate scope name") + } + seen[entry.Name] = true + entries = append(entries, entry) + } + return entries, document.GeneratedAt, nil +} diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go new file mode 100644 index 00000000..66d854a2 --- /dev/null +++ b/internal/meshmapper/scopes_test.go @@ -0,0 +1,253 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func catalogue(names ...string) []byte { + rows := make([]map[string]any, 0, len(names)) + for _, name := range names { + rows = append(rows, map[string]any{"name": name, "repeaters": 0, "default": 0, "monitored": true, "wardriving": false}) + } + b, _ := json.Marshal(map[string]any{"generated_at": "2026-09-27T16:58:49Z", "region": "YOW", "zones": []string{"YOW"}, "repeaters": 5, "scoped": 3, "scopes": rows}) + return b +} + +type memoryStore struct { + rows map[string]Cache + fail bool + saved []scopestore.Entry +} + +func (s *memoryStore) GetScopeCatalogue(_ context.Context, iata, url string) (*Cache, error) { + row, ok := s.rows[iata+url] + if !ok { + return nil, nil + } + return &row, nil +} +func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, next Cache, entries []scopestore.Entry) error { + if s.fail { + return errors.New("offline") + } + old := s.rows[iata+url] + if next.Payload == nil { + next.Payload = old.Payload + } + if next.CheckedAt.IsZero() { + next.CheckedAt = old.CheckedAt + next.ETag = old.ETag + } + s.rows[iata+url] = next + s.saved = append(s.saved, entries...) + return nil +} + +func names(store *scopestore.ScopeStore) []string { + out := []string{} + for _, e := range store.Entries() { + out = append(out, e.Name) + } + return out +} + +func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { + ctx := context.Background() + status, body, etag, after := 200, catalogue("yow", "manual"), `"v1"`, "" + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls > 1 && r.Header.Get("If-None-Match") != `"v1"` { + t.Errorf("missing conditional refresh: %v", r.Header) + } + w.Header().Set("ETag", etag) + w.Header().Set("Retry-After", after) + w.WriteHeader(status) + if status != 304 { + _, _ = w.Write(body) + } + })) + defer server.Close() + cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} + store := &memoryStore{rows: map[string]Cache{}} + scopes := scopestore.New() + manual := scopestore.FromName("manual") + manual.TransportKey = []byte("manual wins") + imp, err := New(ctx, cfg, store, scopes, []scopestore.Entry{manual}) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + check := func() { + t.Helper() + now = now.Add(time.Minute) + if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { + t.Fatal(err) + } + } + check() + if got := names(scopes); !reflect.DeepEqual(got, []string{"#manual", "#yow"}) { + t.Fatal(got) + } + if string(scopes.Entries()[0].TransportKey) != "manual wins" || scopes.Entries()[0].IATAs != nil { + t.Fatal("manual key replaced/restricted") + } + if !reflect.DeepEqual(scopes.Entries()[1].IATAs, []string{"YOW"}) { + t.Fatal("import not bound to region") + } + generated := imp.sources[0].generated + status = 304 + check() + checked := imp.sources[0].cache.CheckedAt + if checked != now || imp.sources[0].generated != generated { + t.Fatal("304 lost freshness or changed generation time") + } + for _, failure := range []int{200, 404, 503, 302} { + status, body = failure, []byte(`{"scopes":null}`) + check() + if imp.sources[0].cache.CheckedAt != checked || len(scopes.Entries()) != 2 { + t.Fatal("failed refresh replaced known catalogue") + } + } + status, after = 429, "7200" + check() + if !imp.retryAfter.Equal(now.Add(2 * time.Hour)) { + t.Fatal("Retry-After ignored") + } + restored := scopestore.New() + restart, err := New(ctx, cfg, store, restored, []scopestore.Entry{manual}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(names(restored), names(scopes)) { + t.Fatal("outage restart lost cached names") + } + before := calls + if err := restart.Refresh(ctx); err != nil { + t.Fatal(err) + } + if calls != before { + t.Fatal("restart ignored shared rate-limit cooldown") + } + status, after, body = 200, "", catalogue() + check() + if got := names(scopes); !reflect.DeepEqual(got, []string{"#manual"}) { + t.Fatalf("empty snapshot: %v", got) + } + store.fail = true + body = catalogue("new") + now = now.Add(time.Hour) + if err := imp.refresh(ctx, &imp.sources[0], now); err == nil || len(scopes.Entries()) != 1 { + t.Fatal("uncommitted names became active") + } +} + +func TestDecodeCatalogueBoundaries(t *testing.T) { + valid := string(catalogue("yow", "YOW", "$private")) + bad := []string{ + `null`, `{}`, `[]`, valid + `{}`, + strings.Replace(valid, `"region":"YOW"`, `"region":"PNW"`, 1), + strings.Replace(valid, `"zones":["YOW"]`, `"zones":["YOW","YVR"]`, 1), + strings.Replace(valid, `"scoped":3`, `"scoped":6`, 1), + strings.Replace(valid, `"repeaters":0`, `"repeaters":-1`, 1), + strings.Replace(valid, `"monitored":true,`, ``, 1), + strings.Replace(valid, `"generated_at":"2026-09-27T16:58:49Z"`, `"generated_at":null`, 1), + string(catalogue("yow", "#yow")), string(catalogue("#")), string(catalogue(" yow")), string(catalogue("a\nb")), + strings.Repeat(" ", MaxBody+1), + } + tooMany := []string{} + for n := 0; n <= MaxScopes; n++ { + tooMany = append(tooMany, fmt.Sprint(n)) + } + bad = append(bad, string(catalogue(tooMany...))) + for n, raw := range bad { + if _, _, err := decode([]byte(raw), "YOW"); err == nil { + t.Errorf("invalid case %d accepted", n) + } + } + entries, _, err := decode([]byte(valid), "YOW") + if err != nil || len(entries) != 3 { + t.Fatal(entries, err) + } + if entries[0].Name != "#yow" || entries[1].Name != "#YOW" || entries[2].Name != "$private" { + t.Fatal("case/prefix changed", entries) + } +} + +func TestNoRedirectAndResponseBounds(t *testing.T) { + targetCalls := 0 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { targetCalls++ })) + defer target.Close() + mode := "redirect" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch mode { + case "redirect": + http.Redirect(w, r, target.URL, 302) + case "oversize": + _, _ = w.Write([]byte(strings.Repeat("x", MaxBody+1))) + case "timeout": + select { + case <-r.Context().Done(): + case <-time.After(time.Second): + } + case "304": + w.WriteHeader(304) + } + })) + defer server.Close() + store := &memoryStore{rows: map[string]Cache{}} + imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + imp.client.Timeout = 20 * time.Millisecond + for _, value := range []string{"redirect", "oversize", "timeout", "304"} { + mode = value + if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil { + t.Fatal(err) + } + if imp.sources[0].cache.LastError == "" || len(imp.sources[0].entries) != 0 { + t.Fatal("invalid response accepted", value) + } + } + if targetCalls != 0 { + t.Fatal("followed redirect") + } +} + +func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { + ctx := context.Background() + store := &memoryStore{rows: map[string]Cache{}} + for _, iata := range []string{"YOW", "YVR"} { + store.rows[iata+iata] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata))} + } + scopes := scopestore.New() + _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "YOW", "YVR": "YVR"}}, store, scopes, nil) + if err != nil || len(scopes.Entries()) != 1 || !reflect.DeepEqual(scopes.Entries()[0].IATAs, []string{"YOW", "YVR"}) { + t.Fatal(scopes.Entries(), err) + } + _, err = New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "changed-source"}}, store, scopes, nil) + if err != nil || len(scopes.Entries()) != 0 { + t.Fatal("removed/changed source remained active", err) + } + // Disabled mode never calls the store, even if URLs remain configured. + _, err = New(ctx, config.MeshMapperScopesConfig{Sources: map[string]string{"YOW": "YOW"}}, nil, scopes, nil) + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/scopestore/scopestore.go b/internal/scopestore/scopestore.go index 8abbac3b..b95bad8d 100644 --- a/internal/scopestore/scopestore.go +++ b/internal/scopestore/scopestore.go @@ -5,13 +5,29 @@ // loaded from the database at startup. package scopestore -import "sync" +import ( + "crypto/sha256" + "strings" + "sync" +) // Entry holds a single transport scope key and its metadata. type Entry struct { Name string - TransportKey []byte // 16 bytes - KeyFingerprint []byte // 8 bytes + TransportKey []byte // 16 bytes + KeyFingerprint []byte // 8 bytes + IATAs []string // nil for manual keys; imported candidates are regional +} + +// FromName uses the same case-sensitive derivation for manual and imported scopes. +func FromName(name string) Entry { + if !strings.HasPrefix(name, "#") && !strings.HasPrefix(name, "$") { + name = "#" + name + } + h := sha256.Sum256([]byte(name)) + key := h[:16] + fingerprint := sha256.Sum256(key) + return Entry{Name: name, TransportKey: key, KeyFingerprint: fingerprint[:8]} } // ScopeStore holds all known transport scope keys in memory. From f6e628c5014970068ecf983ddaf1da9e67096d0c Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 18:07:07 -0400 Subject: [PATCH 072/141] test(scopes): run race checks on the Linux CI runner --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6be1defb..66902345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,9 @@ jobs: - name: Test run: go test ./... + - name: Verify scope refresh concurrency + run: go test -race ./internal/meshmapper ./internal/scopestore + - name: Verify backup startup against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable From 986a6c0788a607110a569e88ffd896e315c74ed4 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 18:10:20 -0400 Subject: [PATCH 073/141] test(scopes): synchronize the timeout fixture state --- internal/meshmapper/scopes_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go index 66d854a2..f84fcc1c 100644 --- a/internal/meshmapper/scopes_test.go +++ b/internal/meshmapper/scopes_test.go @@ -12,6 +12,7 @@ import ( "net/http/httptest" "reflect" "strings" + "sync/atomic" "testing" "time" @@ -193,9 +194,10 @@ func TestNoRedirectAndResponseBounds(t *testing.T) { targetCalls := 0 target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { targetCalls++ })) defer target.Close() - mode := "redirect" + var mode atomic.Value + mode.Store("redirect") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch mode { + switch mode.Load().(string) { case "redirect": http.Redirect(w, r, target.URL, 302) case "oversize": @@ -217,7 +219,7 @@ func TestNoRedirectAndResponseBounds(t *testing.T) { } imp.client.Timeout = 20 * time.Millisecond for _, value := range []string{"redirect", "oversize", "timeout", "304"} { - mode = value + mode.Store(value) if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil { t.Fatal(err) } From 574bba4e13a6f305f1aa0af10349134b0c310c83 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 18:27:41 -0400 Subject: [PATCH 074/141] fix: address maintainer review for #174 --- db/queries/meshmapper.sql | 27 -------- db/queries/queries.sql | 28 ++++++++ db/sqlc/meshmapper.sql.go | 94 -------------------------- db/sqlc/queries.sql.go | 82 ++++++++++++++++++++++ docs/meshmapper-scopes.md | 26 ------- internal/config/seed_test.go | 3 +- internal/ingest/packet.go | 16 +++-- internal/ingest/scope_import_test.go | 18 +++++ internal/meshmapper/scopes.go | 12 +++- internal/meshmapper/scopes_test.go | 25 +++++++ internal/scopestore/scopestore.go | 8 +-- internal/scopestore/scopestore_test.go | 24 ++++--- 12 files changed, 192 insertions(+), 171 deletions(-) delete mode 100644 db/queries/meshmapper.sql delete mode 100644 db/sqlc/meshmapper.sql.go delete mode 100644 docs/meshmapper-scopes.md diff --git a/db/queries/meshmapper.sql b/db/queries/meshmapper.sql deleted file mode 100644 index 16e82433..00000000 --- a/db/queries/meshmapper.sql +++ /dev/null @@ -1,27 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- name: GetScopeCatalogue :one -SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; - --- name: SaveScopeCatalogue :exec --- One statement commits the validated snapshot and its lookup identities together. --- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data --- after an error or 304. Imported names never replace existing manual metadata. -WITH inserted AS ( - INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) - SELECT entry.name, entry.key, entry.fingerprint, TRUE - FROM (SELECT unnest(@names::text[]) AS name, unnest(@keys::bytea[]) AS key, - unnest(@fingerprints::bytea[]) AS fingerprint) AS entry - ON CONFLICT (name) DO NOTHING -) -INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) -VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, - sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) -ON CONFLICT (iata, url) DO UPDATE SET - payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), - etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), - checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), - attempted_at = EXCLUDED.attempted_at, - next_attempt = EXCLUDED.next_attempt, - last_error = EXCLUDED.last_error; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 37cd8fa2..5019850a 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1587,3 +1587,31 @@ WITH latest AS ( ) SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, hourly.n AS last_complete_hour FROM hourly; + +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: GetScopeCatalogue :one +SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; + +-- name: SaveScopeCatalogue :exec +-- One statement commits the validated snapshot and its lookup identities together. +-- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +-- after an error or 304. Imported names never replace existing manual metadata. +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest(@names::text[]) AS name, unnest(@keys::bytea[]) AS key, + unnest(@fingerprints::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; diff --git a/db/sqlc/meshmapper.sql.go b/db/sqlc/meshmapper.sql.go deleted file mode 100644 index b8ba12d9..00000000 --- a/db/sqlc/meshmapper.sql.go +++ /dev/null @@ -1,94 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.31.1 -// source: meshmapper.sql - -package db - -import ( - "context" - - "github.com/jackc/pgx/v5/pgtype" -) - -const getScopeCatalogue = `-- name: GetScopeCatalogue :one - -SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2 -` - -type GetScopeCatalogueParams struct { - Iata string `json:"iata"` - Url string `json:"url"` -} - -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later -func (q *Queries) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) { - row := q.db.QueryRow(ctx, getScopeCatalogue, arg.Iata, arg.Url) - var i MeshmapperScopeCatalogue - err := row.Scan( - &i.Iata, - &i.Url, - &i.Payload, - &i.Etag, - &i.CheckedAt, - &i.AttemptedAt, - &i.NextAttempt, - &i.LastError, - ) - return i, err -} - -const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec -WITH inserted AS ( - INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) - SELECT entry.name, entry.key, entry.fingerprint, TRUE - FROM (SELECT unnest($9::text[]) AS name, unnest($10::bytea[]) AS key, - unnest($11::bytea[]) AS fingerprint) AS entry - ON CONFLICT (name) DO NOTHING -) -INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) -VALUES ($1, $2, $3::jsonb, $4::text, - $5::timestamptz, $6, $7, $8) -ON CONFLICT (iata, url) DO UPDATE SET - payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), - etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), - checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), - attempted_at = EXCLUDED.attempted_at, - next_attempt = EXCLUDED.next_attempt, - last_error = EXCLUDED.last_error -` - -type SaveScopeCatalogueParams struct { - Iata string `json:"iata"` - Url string `json:"url"` - Payload []byte `json:"payload"` - Etag *string `json:"etag"` - CheckedAt pgtype.Timestamptz `json:"checked_at"` - AttemptedAt pgtype.Timestamptz `json:"attempted_at"` - NextAttempt pgtype.Timestamptz `json:"next_attempt"` - LastError string `json:"last_error"` - Names []string `json:"names"` - Keys [][]byte `json:"keys"` - Fingerprints [][]byte `json:"fingerprints"` -} - -// One statement commits the validated snapshot and its lookup identities together. -// Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data -// after an error or 304. Imported names never replace existing manual metadata. -func (q *Queries) SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error { - _, err := q.db.Exec(ctx, saveScopeCatalogue, - arg.Iata, - arg.Url, - arg.Payload, - arg.Etag, - arg.CheckedAt, - arg.AttemptedAt, - arg.NextAttempt, - arg.LastError, - arg.Names, - arg.Keys, - arg.Fingerprints, - ) - return err -} diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 9195a629..f33aa66e 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1509,6 +1509,34 @@ func (q *Queries) GetScopeByName(ctx context.Context, name string) (GetScopeByNa return i, err } +const getScopeCatalogue = `-- name: GetScopeCatalogue :one + +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2 +` + +type GetScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` +} + +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +func (q *Queries) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) { + row := q.db.QueryRow(ctx, getScopeCatalogue, arg.Iata, arg.Url) + var i MeshmapperScopeCatalogue + err := row.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ) + return i, err +} + const getScopeNames = `-- name: GetScopeNames :many SELECT name FROM transport_scopes ORDER BY name ` @@ -4371,6 +4399,60 @@ func (q *Queries) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashes return items, nil } +const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest($9::text[]) AS name, unnest($10::bytea[]) AS key, + unnest($11::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` + Names []string `json:"names"` + Keys [][]byte `json:"keys"` + Fingerprints [][]byte `json:"fingerprints"` +} + +// One statement commits the validated snapshot and its lookup identities together. +// Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +// after an error or 304. Imported names never replace existing manual metadata. +func (q *Queries) SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error { + _, err := q.db.Exec(ctx, saveScopeCatalogue, + arg.Iata, + arg.Url, + arg.Payload, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + arg.Names, + arg.Keys, + arg.Fingerprints, + ) + return err +} + const searchKnownRoutes = `-- name: SearchKnownRoutes :many SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes diff --git a/docs/meshmapper-scopes.md b/docs/meshmapper-scopes.md deleted file mode 100644 index 7c9a3916..00000000 --- a/docs/meshmapper-scopes.md +++ /dev/null @@ -1,26 +0,0 @@ -# Optional MeshMapper scope discovery - -Beacon can periodically read the [public Scopes API](https://wiki.meshmapper.net/scopes-api/) to discover additional transport scope names for configured regions. Manual `scopes` remain authoritative. This does not supply channel decryption keys or per-repeater records. - -```yaml -meshmapper: - scopes: - enabled: true - refresh_interval: 1h - sources: - YOW: https://yow.meshmapper.net/get_scopes.php -``` - -The IATA must already occur in `regions[].iatas`. Set the exact published regional HTTPS URL; credentials, query strings, other hosts and redirects are rejected. The response must identify the same region and exactly one zone. Group catalogues are rejected because they cannot assign each scope/count to individual member regions. No API key is required. - -The default is disabled. Refresh defaults to one hour and accepts 5 minutes through 24 hours. One source is considered every 15 seconds, keeping requests below four per minute per process. Multiple deployments share MeshMapper's per-IP allowance. ETags reduce transfer, HTTP 304 advances the last successful check, and 429 pauses all sources until the later of the normal retry time and `Retry-After`. The pause survives restarts. HTTP 503 also honors that source's `Retry-After`. - -Responses are limited to 64 KiB and 64 names per source, with at most 16 sources. Names are case-sensitive and at most 128 bytes before Beacon's usual `#` normalization; existing `#`/`$` prefixes are retained. Zero-count monitored names are kept. Imported matching candidates are limited to their source IATA; manual keys keep their existing global applicability. A short transport code that matches multiple candidate names remains unresolved. - -Migration 042 stores the last validated response, validator and check/retry timestamps in PostgreSQL. Saving a response and its scope identities is atomic. Timeouts, invalid responses and HTTP errors retain the prior catalogue; startup restores it without HTTP. A successful empty response removes that source's active imported membership. Removing or disabling a source takes effect on restart. Scope identities referenced by historical records are retained. Adding an imported name to the manual list promotes it to manual ownership. - -Operator logs under `component=meshmapper.scopes` show the IATA, source, active name count, source generation time, last successful check, next attempt and last error. An empty last error means the latest request succeeded; a zero checked time means no successful response has been stored. A 304 retains the original source generation time because MeshMapper excludes that timestamp from its ETag. Cached names may be old during an outage; use the check/error fields to assess freshness. - -Catalogue counts remain source metadata. They never create Beacon packet counts, observer associations or node defaults. Actual packet-code matching supplies those associations through the existing ingest path. New names affect subsequent packets; no historical scan is started. Channel message tags and import-status UI are separate follow-ups. Source corruption or an unavailable catalogue database at startup is reported as a startup failure; disabling the optional importer restores manual-only behavior without deleting saved data. - -Validate and back up PostgreSQL before applying the migration. Rollback to an older server must restore its matching pre-import database and configuration as well as its binary; an older matcher does not understand imported-only ownership or regional candidate limits. diff --git a/internal/config/seed_test.go b/internal/config/seed_test.go index 069bbe38..04c8e63a 100644 --- a/internal/config/seed_test.go +++ b/internal/config/seed_test.go @@ -8,9 +8,10 @@ import ( "encoding/hex" "encoding/json" "errors" - "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" "os" "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) type stubSeeder struct { diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 5dfe7d7c..1a99b5e6 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -925,10 +925,10 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } } -// matchTransportScope never treats catalogue membership as forwarding evidence. -// Multiple candidate names matching the short code are ambiguous, not first-wins. +// matchTransportScope preserves manual precedence; imported collisions remain unresolved. func matchTransportScope(entries []scopestore.Entry, iata string, payloadType uint8, payload []byte, code uint16) *string { var matched *string + ambiguous := false for _, entry := range entries { if entry.IATAs != nil && !slices.Contains(entry.IATAs, iata) { continue @@ -936,12 +936,18 @@ func matchTransportScope(entries []scopestore.Entry, iata string, payloadType ui if computeTransportCode(entry.TransportKey, payloadType, payload) != code { continue } - if matched != nil && *matched != entry.Name { - return nil - } name := entry.Name + if entry.IATAs == nil { + return &name + } + if matched != nil && *matched != name { + ambiguous = true + } matched = &name } + if ambiguous { + return nil + } return matched } diff --git a/internal/ingest/scope_import_test.go b/internal/ingest/scope_import_test.go index 3c085e41..4474f135 100644 --- a/internal/ingest/scope_import_test.go +++ b/internal/ingest/scope_import_test.go @@ -29,6 +29,7 @@ func TestTransportScopeRegionalAndAmbiguousMatches(t *testing.T) { } // Distinct keys with the same short code for this payload must remain unknown. a, b := scopestore.FromName("collision-320"), scopestore.FromName("collision-322") + a.IATAs, b.IATAs = []string{"YOW"}, []string{"YOW"} for _, entries := range [][]scopestore.Entry{{a, b}, {b, a}} { if computeTransportCode(entries[0].TransportKey, 4, payload) != 37018 { t.Fatal("collision fixture changed") @@ -93,3 +94,20 @@ func BenchmarkTransportScopeCatalogue(b *testing.B) { }) } } + +func TestManualScopeWinsTransportCodeCollision(t *testing.T) { + payload := []byte{0xde, 0xad, 0xbe, 0xef} + manual, imported := scopestore.FromName("collision-320"), scopestore.FromName("collision-322") + imported.IATAs = []string{"YOW"} + for _, entries := range [][]scopestore.Entry{{manual, imported}, {imported, manual}, {imported, scopestore.Entry{Name: "ambiguous", TransportKey: imported.TransportKey, IATAs: []string{"YOW"}}, manual}} { + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got == nil || *got != manual.Name { + t.Fatalf("manual match lost to imported collision: %v", got) + } + } + imported.IATAs = nil + for _, entries := range [][]scopestore.Entry{{manual, imported}, {imported, manual}} { + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got == nil || *got != entries[0].Name { + t.Fatalf("manual-only first-match behavior changed: %v", got) + } + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index a8b64cee..23363e69 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -88,7 +88,9 @@ func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, sc if len(cached.Payload) > 0 { s.entries, s.generated, err = decode(cached.Payload, key) if err != nil { - return nil, fmt.Errorf("invalid saved scope catalogue %s: %w", key, err) + s.entries = nil + s.cache.Payload, s.cache.ETag = nil, "" + s.cache.LastError = "invalid saved catalogue; awaiting refresh" } } if cached.LastError == "HTTP 429" && cached.NextAttempt.After(i.retryAfter) { @@ -103,7 +105,13 @@ func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, sc } // Refresh checks only one due source. The scheduler serializes calls every 15s. -func (i *Importer) Refresh(ctx context.Context) error { +func (i *Importer) Refresh(ctx context.Context) (err error) { + parent := ctx + defer func() { + if parent.Err() != nil { + err = nil + } + }() ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() now := time.Now().UTC() diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go index f84fcc1c..858ea582 100644 --- a/internal/meshmapper/scopes_test.go +++ b/internal/meshmapper/scopes_test.go @@ -253,3 +253,28 @@ func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { t.Fatal(err) } } + +func TestInvalidSavedCatalogueDoesNotBlockStartupOrRefresh(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-None-Match") != "" { + t.Error("invalid saved catalogue must not accept a 304") + } + _, _ = w.Write(catalogue("yow")) + })) + defer server.Close() + store := &memoryStore{rows: map[string]Cache{"YOW" + server.URL: {Payload: []byte(`{"invalid":true}`), ETag: `"old"`}}} + scopes := scopestore.New() + imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopes, []scopestore.Entry{scopestore.FromName("manual")}) + if err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual"}) { + t.Fatal("invalid optional source blocked manual startup", err) + } + if err = imp.Refresh(context.Background()); err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual", "#yow"}) { + t.Fatal("source did not recover on refresh", err, names(scopes)) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + imp.sources[0].cache.NextAttempt = time.Time{} + if err = imp.Refresh(ctx); err != nil { + t.Fatalf("normal shutdown is a task failure: %v", err) + } +} diff --git a/internal/scopestore/scopestore.go b/internal/scopestore/scopestore.go index b95bad8d..da5c80f4 100644 --- a/internal/scopestore/scopestore.go +++ b/internal/scopestore/scopestore.go @@ -41,18 +41,16 @@ func New() *ScopeStore { return &ScopeStore{} } -// Load replaces all entries — call on startup after DB seeding. +// Load publishes a new immutable snapshot. The caller must not mutate it after publication. func (s *ScopeStore) Load(entries []Entry) { s.mu.Lock() defer s.mu.Unlock() s.entries = entries } -// Entries returns a copy of all loaded entries. +// Entries returns the current immutable snapshot. Callers must not modify it. func (s *ScopeStore) Entries() []Entry { s.mu.RLock() defer s.mu.RUnlock() - result := make([]Entry, len(s.entries)) - copy(result, s.entries) - return result + return s.entries } diff --git a/internal/scopestore/scopestore_test.go b/internal/scopestore/scopestore_test.go index 8df39001..3e2f3592 100644 --- a/internal/scopestore/scopestore_test.go +++ b/internal/scopestore/scopestore_test.go @@ -38,17 +38,19 @@ func TestLoad_ReplacesEntries(t *testing.T) { } } -func TestEntries_ReturnsCopy(t *testing.T) { +func TestEntriesRetainsImmutableSnapshotWithoutAllocation(t *testing.T) { s := New() - s.Load([]Entry{ - {Name: "#bc", TransportKey: []byte{0x01}, KeyFingerprint: []byte{0x02}}, - }) - entries := s.Entries() - entries[0].Name = "mutated" - - // original should be unchanged - original := s.Entries() - if original[0].Name != "#bc" { - t.Errorf("expected #bc, got %s after mutation of copy", original[0].Name) + s.Load([]Entry{{Name: "#bc"}}) + previous := s.Entries() + if allocations := testing.AllocsPerRun(100, func() { + if len(s.Entries()) != 1 { + t.Fatal("missing entries") + } + }); allocations != 0 { + t.Fatalf("snapshot read allocates: %v", allocations) + } + s.Load([]Entry{{Name: "#on"}}) + if previous[0].Name != "#bc" || s.Entries()[0].Name != "#on" { + t.Fatal("replacement mutated the previous reader snapshot") } } From 83cbaff1cf3e0f78a36e1536382de4b933c2b88b Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:14:52 -0400 Subject: [PATCH 075/141] fix: address release review follow-ups for #174 --- db/queries/queries.sql | 2 -- internal/api/handlers/scopes.go | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 5019850a..366bf42f 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1588,8 +1588,6 @@ WITH latest AS ( SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, hourly.n AS last_complete_hour FROM hourly; --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later -- name: GetScopeCatalogue :one SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; diff --git a/internal/api/handlers/scopes.go b/internal/api/handlers/scopes.go index 5267ed1a..c86de6fa 100644 --- a/internal/api/handlers/scopes.go +++ b/internal/api/handlers/scopes.go @@ -24,6 +24,7 @@ func ScopesRouter(reader api.Reader) http.Handler { // listScopes godoc // // @Summary List transport scopes +// @Description The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership. // @Tags Scopes // @Produce json // @Param iatas query string false "Filter by IATA code(s), comma-separated" From 7e01abf5a08156d161257974bb6115189925fbd2 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:47:32 -0400 Subject: [PATCH 076/141] docs(api): refresh generated contract after stack update --- docs/docs.go | 1 + docs/swagger.json | 1 + docs/swagger.yaml | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/docs/docs.go b/docs/docs.go index 93bc6f52..862fa8ad 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2219,6 +2219,7 @@ const docTemplate = `{ }, "/scopes": { "get": { + "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", "produces": [ "application/json" ], diff --git a/docs/swagger.json b/docs/swagger.json index 617277a7..5af6a539 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2217,6 +2217,7 @@ }, "/scopes": { "get": { + "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", "produces": [ "application/json" ], diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 3898b5de..2229cfd7 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -3002,6 +3002,10 @@ paths: - Routes /scopes: get: + description: The unfiltered list includes stored imported names after an importer + is disabled or a source is removed. Those historical identities are retained + indefinitely; listing them does not establish observed traffic or current + source membership. parameters: - description: Filter by IATA code(s), comma-separated in: query From 2abed6c1952e86353ad0f14acd27cc43e498b190 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 19:16:49 -0400 Subject: [PATCH 077/141] feat(channels): expose recorded packet scope in message delivery --- .github/workflows/ci.yml | 2 +- db/channel_scopes_integration_test.go | 156 ++++++++++++++++++++++++++ db/channels.go | 18 +-- db/queries/queries.sql | 24 ++-- db/sqlc/mock/querier.go | 4 +- db/sqlc/querier.go | 4 +- db/sqlc/queries.sql.go | 132 +++++++++++++--------- db/store.go | 4 +- db/store_test.go | 4 +- docs/channel-scopes.md | 16 +++ docs/docs.go | 13 +++ docs/swagger.json | 13 +++ docs/swagger.yaml | 11 ++ internal/api/channel_scope_test.go | 27 +++++ internal/api/channels.go | 33 ++++-- internal/ingest/backfill.go | 6 +- internal/ingest/channel_scope_test.go | 75 +++++++++++++ internal/ingest/ingest.go | 2 +- internal/ingest/ingest_test.go | 7 +- internal/ingest/side_effects.go | 25 ++++- 20 files changed, 484 insertions(+), 92 deletions(-) create mode 100644 db/channel_scopes_integration_test.go create mode 100644 docs/channel-scopes.md create mode 100644 internal/api/channel_scope_test.go create mode 100644 internal/ingest/channel_scope_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66902345..328b4528 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/db/channel_scopes_integration_test.go b/db/channel_scopes_integration_test.go new file mode 100644 index 00000000..69591aab --- /dev/null +++ b/db/channel_scopes_integration_test.go @@ -0,0 +1,156 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestChannelMessageScopesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"transport_scopes", "packets", "packet_observations", "channels", "channel_messages"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO transport_scopes(id,name,transport_key,key_fingerprint) VALUES +(1,'#yow',decode(repeat('00',16),'hex'),decode(repeat('01',8),'hex')), +(2,'#can',decode(repeat('02',16),'hex'),decode(repeat('03',8),'hex')); +INSERT INTO channels(id,channel_hash,key_fingerprint,name,is_hashtag,key_known) VALUES (123,'\x11','\x01020304','#fixture',true,true); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,transport_codes_present,scope_id,raw_payload,raw_header) +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),5,0,1,transport,scope_id,'\x00','\x15' +FROM (VALUES(1,true,1),(2,true,NULL),(3,false,NULL),(4,NULL,NULL),(5,true,2),(6,false,NULL)) v(i,transport,scope_id); +INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT packet_hash,md5(n::text)::uuid,'YOW',NOW(),0,1,0 FROM packets CROSS JOIN generate_series(1,2) n; +`) + if err != nil { + t.Fatal(err) + } + s := &Store{q: sqlc.New(tx)} + // Later matching reception must not relabel the immutable first stored packet. + scopeID := int32(1) + _, err = s.UpsertPacket(ctx, ingest.UpsertPacketParams{PacketHash: []byte{6}, PayloadType: 5, ScopeID: &scopeID, TransportCodes: []byte{1, 2, 3, 4}, RawPayload: []byte{0}, RawHeader: []byte{0}}) + if err != nil { + t.Fatal(err) + } + statuses := []api.ChannelScopeStatus{"matched", "unknown", "unscoped", "unavailable", "matched", "unscoped"} + wants := map[int64]*ingest.InsertedChannelMessage{} + for n, status := range statuses { + m := ingest.InsertChannelMessageParams{ChannelID: 123, PacketHash: []byte{byte(n + 1)}, SenderName: "Fixture", Content: "Synthetic message", SentAt: time.Now()} + inserted, err := s.InsertChannelMessage(ctx, m) + if err != nil || inserted == nil || inserted.ScopeStatus != status { + t.Fatalf("insert %d: %+v %v", n, inserted, err) + } + wants[inserted.ID] = inserted + duplicate, err := s.InsertChannelMessage(ctx, m) + if err != nil || duplicate != nil { + t.Fatal("duplicate changed live delivery", duplicate, err) + } + } + check := func(items []api.ChannelMessage) { + t.Helper() + if len(items) != 6 { + t.Fatalf("messages=%d", len(items)) + } + for _, m := range items { + want := wants[m.ID] + if want == nil || m.ScopeStatus != want.ScopeStatus { + t.Fatal("history differs from insertion", m) + } + if (m.Scope == nil) != (want.Scope == nil) || (m.Scope != nil && *m.Scope != *want.Scope) { + t.Fatal("scope differs", m) + } + if m.ObservationCount != 2 { + t.Fatal("duplicate observations changed message grain", m) + } + b, _ := json.Marshal(m) + var wire map[string]any + _ = json.Unmarshal(b, &wire) + if _, ok := wire["scope"]; !ok || wire["scopeStatus"] != string(m.ScopeStatus) { + t.Fatal("wire metadata absent", string(b)) + } + } + } + ch := int32(123) + for _, id := range []*int32{nil, &ch} { + p, err := s.ListChannelMessages(ctx, id, time.Time{}, 20, []string{"YOW"}, "", 0) + if err != nil { + t.Fatal(err) + } + check(p.Items) + } + hash, err := s.ListChannelMessagesByHash(ctx, []byte{0x11}, time.Time{}, 20, []string{"YOW"}, "", 0) + if err != nil { + t.Fatal(err) + } + check(hash.Items) + catchup, err := s.ListMessagesAfterID(ctx, 0, []string{"YOW"}, "", 20) + if err != nil { + t.Fatal(err) + } + check(catchup) + filtered, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, []string{"YOW"}, "#yow", 0) + if err != nil || len(filtered.Items) != 1 || filtered.Items[0].Scope == nil || *filtered.Items[0].Scope != "#yow" { + t.Fatal(filtered, err) + } + catchup, err = s.ListMessagesAfterID(ctx, 0, []string{"YOW"}, "#yow", 20) + if err != nil || len(catchup) != 1 { + t.Fatal(catchup, err) + } + other, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, []string{"YVR"}, "", 0) + if err != nil || len(other.Items) != 0 { + t.Fatal("region escaped", other, err) + } + first, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 2, nil, "", 0) + if err != nil || !first.HasMore || first.NextCursor == nil { + t.Fatal(first, err) + } + second, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 2, nil, "", *first.NextCursor) + if err != nil || len(second.Items) != 2 || second.Items[0].ID >= *first.NextCursor { + t.Fatal("cursor drift", second, err) + } + // Even without the production cascade, expired packet evidence must not leak orphan messages. + if _, err := tx.Exec(ctx, "DELETE FROM packets"); err != nil { + t.Fatal(err) + } + expired, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, nil, "", 0) + if err != nil || len(expired.Items) != 0 { + t.Fatal(expired, err) + } +} diff --git a/db/channels.go b/db/channels.go index 599230a3..9082e5ab 100644 --- a/db/channels.go +++ b/db/channels.go @@ -154,16 +154,16 @@ func (s *Store) GetChannel(ctx context.Context, channelID int32) (*api.Channel, return &channel, nil } -func (s *Store) InsertChannelMessage(ctx context.Context, m ingest.InsertChannelMessageParams) (bool, error) { +func (s *Store) InsertChannelMessage(ctx context.Context, m ingest.InsertChannelMessageParams) (*ingest.InsertedChannelMessage, error) { params := sqlc.InsertChannelMessageParams{ChannelID: int32(m.ChannelID), PacketHash: m.PacketHash, SenderName: &m.SenderName, Content: &m.Content, SentAt: pgtype.Timestamptz{Time: m.SentAt, Valid: true}} - _, err := s.q.InsertChannelMessage(ctx, params) + row, err := s.q.InsertChannelMessage(ctx, params) if errors.Is(err, pgx.ErrNoRows) { - return false, nil // duplicate + return nil, nil // duplicate } if err != nil { - return false, err + return nil, err } - return true, nil + return &ingest.InsertedChannelMessage{ID: row.ID, Scope: row.ScopeName, ScopeStatus: api.RecordedChannelScopeStatus(row.ScopeName, row.TransportCodesPresent)}, nil } func (s *Store) ListChannelMessages(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) { @@ -187,7 +187,7 @@ func (s *Store) ListChannelMessages(ctx context.Context, channelID *int32, since } messages = make([]api.ChannelMessage, 0, len(rows)) for _, v := range rows { - messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount)) + messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount, v.ScopeName, v.TransportCodesPresent)) } } else { rows, err := s.q.ListChannelMessages(ctx, sqlc.ListChannelMessagesParams{ @@ -207,7 +207,7 @@ func (s *Store) ListChannelMessages(ctx context.Context, channelID *int32, since } messages = make([]api.ChannelMessage, 0, len(rows)) for _, v := range rows { - messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount)) + messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount, v.ScopeName, v.TransportCodesPresent)) } } @@ -241,7 +241,7 @@ func (s *Store) ListChannelMessagesByHash(ctx context.Context, hash []byte, sinc } messages := make([]api.ChannelMessage, 0, len(rows)) for _, v := range rows { - messages = append(messages, toChannelMessage(v.ID, hex.EncodeToString(v.PacketHash), v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount)) + messages = append(messages, toChannelMessage(v.ID, hex.EncodeToString(v.PacketHash), v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount, v.ScopeName, v.TransportCodesPresent)) } var nextCursor *int64 if hasMore && len(messages) > 0 { @@ -275,6 +275,8 @@ func (s *Store) ListMessagesAfterID(ctx context.Context, afterID int64, iatas [] v.Content, v.SentAt, v.ObservationCount, + v.ScopeName, + v.TransportCodesPresent, )) } return items, nil diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 366bf42f..328960e5 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -944,17 +944,25 @@ SELECT * FROM channels WHERE id = $1; -- ============================================================ -- name: InsertChannelMessage :one -INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) -VALUES ($1, $2, $3, $4, $5) -ON CONFLICT (packet_hash) DO NOTHING -RETURNING id; +-- Read the immutable first-packet scope in the same statement as insertion. +-- A later reception's transport code must not give live and historical messages different tags. +WITH inserted AS ( + INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (packet_hash) DO NOTHING + RETURNING id, packet_hash +) +SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present +FROM inserted +JOIN packets p ON p.packet_hash = inserted.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id; -- name: ListChannelMessages :many -- Returns messages for a channel identified by integer ID. -- Pass a zero/null timestamp for since to return all messages up to limit. -- Pass empty string for iata to skip IATA filtering. -- Pass cursor=0 to start from the beginning. -SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -973,7 +981,7 @@ LIMIT $6; -- Returns all messages across all channels with optional time, IATA, scope and cursor filters. -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. -SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -992,7 +1000,7 @@ LIMIT $5; -- May return messages from multiple channels if the hash collides across different keys. -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. -SELECT DISTINCT ON (cm.id) cm.*, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.*, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -1010,7 +1018,7 @@ LIMIT $6; -- name: ListMessagesAfterID :many -- Returns messages after the given message ID, ordered oldest first. -- Used for WS reconnect backfill. -SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 16ea0736..8a9a7efe 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -940,10 +940,10 @@ func (mr *MockQuerierMockRecorder) GetTransportScopes(ctx any) *gomock.Call { } // InsertChannelMessage mocks base method. -func (m *MockQuerier) InsertChannelMessage(ctx context.Context, arg db.InsertChannelMessageParams) (int64, error) { +func (m *MockQuerier) InsertChannelMessage(ctx context.Context, arg db.InsertChannelMessageParams) (db.InsertChannelMessageRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InsertChannelMessage", ctx, arg) - ret0, _ := ret[0].(int64) + ret0, _ := ret[0].(db.InsertChannelMessageRow) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index f96b9a36..1f80ab61 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -128,7 +128,9 @@ type Querier interface { // ============================================================ // CHANNEL MESSAGES // ============================================================ - InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (int64, error) + // Read the immutable first-packet scope in the same statement as insertion. + // A later reception's transport code must not give live and historical messages different tags. + InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) // ============================================================ // PACKET OBSERVATIONS // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index f33aa66e..e6fb1edc 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2110,10 +2110,16 @@ func (q *Queries) GetTransportScopes(ctx context.Context) ([]GetTransportScopesR const insertChannelMessage = `-- name: InsertChannelMessage :one -INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) -VALUES ($1, $2, $3, $4, $5) -ON CONFLICT (packet_hash) DO NOTHING -RETURNING id +WITH inserted AS ( + INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (packet_hash) DO NOTHING + RETURNING id, packet_hash +) +SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present +FROM inserted +JOIN packets p ON p.packet_hash = inserted.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id ` type InsertChannelMessageParams struct { @@ -2124,10 +2130,18 @@ type InsertChannelMessageParams struct { SentAt pgtype.Timestamptz `json:"sent_at"` } +type InsertChannelMessageRow struct { + ID int64 `json:"id"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` +} + // ============================================================ // CHANNEL MESSAGES // ============================================================ -func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (int64, error) { +// Read the immutable first-packet scope in the same statement as insertion. +// A later reception's transport code must not give live and historical messages different tags. +func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) { row := q.db.QueryRow(ctx, insertChannelMessage, arg.ChannelID, arg.PacketHash, @@ -2135,9 +2149,9 @@ func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMes arg.Content, arg.SentAt, ) - var id int64 - err := row.Scan(&id) - return id, err + var i InsertChannelMessageRow + err := row.Scan(&i.ID, &i.ScopeName, &i.TransportCodesPresent) + return i, err } const insertObservation = `-- name: InsertObservation :one @@ -2309,7 +2323,7 @@ func (q *Queries) ListAccounts(ctx context.Context) ([]Account, error) { } const listAllChannelMessages = `-- name: ListAllChannelMessages :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -2333,16 +2347,18 @@ type ListAllChannelMessagesParams struct { } type ListAllChannelMessagesRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - PacketHashHex string `json:"packet_hash_hex"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + PacketHashHex string `json:"packet_hash_hex"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns all messages across all channels with optional time, IATA, scope and cursor filters. @@ -2373,6 +2389,8 @@ func (q *Queries) ListAllChannelMessages(ctx context.Context, arg ListAllChannel &i.SentAt, &i.PacketHashHex, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -2386,7 +2404,7 @@ func (q *Queries) ListAllChannelMessages(ctx context.Context, arg ListAllChannel } const listChannelMessages = `-- name: ListChannelMessages :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -2412,16 +2430,18 @@ type ListChannelMessagesParams struct { } type ListChannelMessagesRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - PacketHashHex string `json:"packet_hash_hex"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + PacketHashHex string `json:"packet_hash_hex"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns messages for a channel identified by integer ID. @@ -2454,6 +2474,8 @@ func (q *Queries) ListChannelMessages(ctx context.Context, arg ListChannelMessag &i.SentAt, &i.PacketHashHex, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -2467,7 +2489,7 @@ func (q *Queries) ListChannelMessages(ctx context.Context, arg ListChannelMessag } const listChannelMessagesByHash = `-- name: ListChannelMessagesByHash :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -2493,15 +2515,17 @@ type ListChannelMessagesByHashParams struct { } type ListChannelMessagesByHashRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns messages for all channels matching a hash byte. @@ -2533,6 +2557,8 @@ func (q *Queries) ListChannelMessagesByHash(ctx context.Context, arg ListChannel &i.Content, &i.SentAt, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -2775,7 +2801,7 @@ func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams } const listMessagesAfterID = `-- name: ListMessagesAfterID :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -2797,16 +2823,18 @@ type ListMessagesAfterIDParams struct { } type ListMessagesAfterIDRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - PacketHashHex string `json:"packet_hash_hex"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + PacketHashHex string `json:"packet_hash_hex"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns messages after the given message ID, ordered oldest first. @@ -2835,6 +2863,8 @@ func (q *Queries) ListMessagesAfterID(ctx context.Context, arg ListMessagesAfter &i.SentAt, &i.PacketHashHex, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err diff --git a/db/store.go b/db/store.go index 20866d76..2f9f35aa 100644 --- a/db/store.go +++ b/db/store.go @@ -130,7 +130,7 @@ func tristate(b *bool) string { } // toChannelMessage maps raw sqlc row fields to an api.ChannelMessage. -func toChannelMessage(id int64, packetHashHex string, channelHash []byte, senderName *string, content *string, sentAt pgtype.Timestamptz, observationCount int64) api.ChannelMessage { +func toChannelMessage(id int64, packetHashHex string, channelHash []byte, senderName *string, content *string, sentAt pgtype.Timestamptz, observationCount int64, scope *string, transport *bool) api.ChannelMessage { sn := "" if senderName != nil { sn = *senderName @@ -147,6 +147,8 @@ func toChannelMessage(id int64, packetHashHex string, channelHash []byte, sender Content: ct, SentAt: sentAt.Time.UnixMilli(), ObservationCount: observationCount, + Scope: scope, + ScopeStatus: api.RecordedChannelScopeStatus(scope, transport), } } diff --git a/db/store_test.go b/db/store_test.go index 4d0521be..41f43938 100644 --- a/db/store_test.go +++ b/db/store_test.go @@ -59,7 +59,7 @@ func TestToChannelMessage(t *testing.T) { channelHash := []byte{0xab} sentAt := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} - msg := toChannelMessage(42, "deadbeef", channelHash, &senderName, &content, sentAt, 7) + msg := toChannelMessage(42, "deadbeef", channelHash, &senderName, &content, sentAt, 7, nil, nil) if msg.ID != 42 { t.Errorf("expected ID 42, got %d", msg.ID) @@ -86,7 +86,7 @@ func TestToChannelMessage(t *testing.T) { func TestToChannelMessage_NilFields(t *testing.T) { sentAt := pgtype.Timestamptz{Time: time.UnixMilli(0), Valid: true} - msg := toChannelMessage(1, "abc", []byte{0x01}, nil, nil, sentAt, 0) + msg := toChannelMessage(1, "abc", []byte{0x01}, nil, nil, sentAt, 0, nil, nil) if msg.SenderName != "" { t.Errorf("expected empty SenderName, got %s", msg.SenderName) } diff --git a/docs/channel-scopes.md b/docs/channel-scopes.md new file mode 100644 index 00000000..c88aaaea --- /dev/null +++ b/docs/channel-scopes.md @@ -0,0 +1,16 @@ +# Channel-message transport scope + +Channel history, the global/hash message APIs, reconnect backfill and `channelMessage` events now include `scope` and `scopeStatus`. Live events also include their stored message `id`; existing fields, endpoints and cursors are unchanged. + +| Status | Meaning | +|---|---| +| `matched` | `scope` names the scope recorded on the first stored packet. | +| `unscoped` | The first stored packet has no transport codes; `scope` is null. | +| `unknown` | Transport codes were recorded but no scope name was resolved, including ambiguous matches. | +| `unavailable` | Scope/capture metadata is missing. | + +These labels belong to the first stored packet, not every later reception of the same payload. Later paths may differ. Live delivery reads that same evidence while inserting the channel message, so a later reception cannot relabel it differently from history. Existing scope query filters retain their meaning. Imported catalogue membership and channel names are never used as message evidence. + +The insertion query returns metadata only for a new message. Duplicate deliveries still produce no extra live message. Adding a decryption key later uses the existing backfill path, retains the stored packet evidence and does not broadcast old messages as new traffic. This change adds no keys, migration or historical scan. Expired packet/message detail remains unavailable. + +Deploy this API before its web consumer. Older clients may ignore the added fields; clients reading older servers must treat absent metadata as unavailable, not assume a message is unscoped. Scope filtering of live events requires the new fields. The channel encryption key and the packet transport scope are separate concepts. diff --git a/docs/docs.go b/docs/docs.go index 862fa8ad..288a70b6 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -3481,6 +3481,19 @@ const docTemplate = `{ "description": "hex-encoded packet hash for correlation with packet events", "type": "string" }, + "scope": { + "description": "matched scope on the first stored packet; null when none was recorded", + "type": "string" + }, + "scopeStatus": { + "type": "string", + "enum": [ + "matched", + "unscoped", + "unknown", + "unavailable" + ] + }, "senderName": { "description": "display name from the decrypted payload", "type": "string" diff --git a/docs/swagger.json b/docs/swagger.json index 5af6a539..56794446 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -3479,6 +3479,19 @@ "description": "hex-encoded packet hash for correlation with packet events", "type": "string" }, + "scope": { + "description": "matched scope on the first stored packet; null when none was recorded", + "type": "string" + }, + "scopeStatus": { + "type": "string", + "enum": [ + "matched", + "unscoped", + "unknown", + "unavailable" + ] + }, "senderName": { "description": "display name from the decrypted payload", "type": "string" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 2229cfd7..a58626f2 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -135,6 +135,17 @@ definitions: packetHash: description: hex-encoded packet hash for correlation with packet events type: string + scope: + description: matched scope on the first stored packet; null when none was + recorded + type: string + scopeStatus: + enum: + - matched + - unscoped + - unknown + - unavailable + type: string senderName: description: display name from the decrypted payload type: string diff --git a/internal/api/channel_scope_test.go b/internal/api/channel_scope_test.go new file mode 100644 index 00000000..f553a628 --- /dev/null +++ b/internal/api/channel_scope_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import "testing" + +func TestRecordedChannelScopeStatus(t *testing.T) { + name := "#yow" + for _, tc := range []struct { + name *string + transport, valid bool + want string + }{ + {&name, true, true, "matched"}, {nil, true, true, "unknown"}, + {nil, false, true, "unscoped"}, {nil, false, false, "unavailable"}, + {&name, false, false, "matched"}, + } { + var transport *bool + if tc.valid { + transport = &tc.transport + } + if got := RecordedChannelScopeStatus(tc.name, transport); string(got) != tc.want { + t.Fatalf("got %s, want %s", got, tc.want) + } + } +} diff --git a/internal/api/channels.go b/internal/api/channels.go index c609a23a..00e9d85b 100644 --- a/internal/api/channels.go +++ b/internal/api/channels.go @@ -56,13 +56,32 @@ func ParseChannelCursor(raw string) (*ChannelCursor, error) { // ChannelMessage represents a single decrypted channel message. // Only messages for channels with a known key are stored and returned. type ChannelMessage struct { - ID int64 `json:"id"` - PacketHash string `json:"packetHash"` // hex-encoded packet hash for correlation with packet events - ChannelHash string `json:"channelHash"` // hex-encoded single-byte channel hash - SenderName string `json:"senderName"` // display name from the decrypted payload - Content string `json:"content"` // decrypted message text - SentAt int64 `json:"sentAt"` // epoch ms, from the sender's embedded timestamp - ObservationCount int64 `json:"observationCount"` // number of packet_observations rows for this message's packet hash + ID int64 `json:"id"` + PacketHash string `json:"packetHash"` // hex-encoded packet hash for correlation with packet events + ChannelHash string `json:"channelHash"` // hex-encoded single-byte channel hash + SenderName string `json:"senderName"` // display name from the decrypted payload + Content string `json:"content"` // decrypted message text + SentAt int64 `json:"sentAt"` // epoch ms, from the sender's embedded timestamp + ObservationCount int64 `json:"observationCount"` // number of packet_observations rows for this message's packet hash + Scope *string `json:"scope"` // matched scope on the first stored packet; null when none was recorded + ScopeStatus ChannelScopeStatus `json:"scopeStatus" enums:"matched,unscoped,unknown,unavailable"` +} + +// ChannelScopeStatus describes stored packet evidence, not a channel key or all reception paths. +type ChannelScopeStatus string + +// RecordedChannelScopeStatus preserves missing capture metadata as unavailable. +func RecordedChannelScopeStatus(name *string, transport *bool) ChannelScopeStatus { + if name != nil { + return "matched" + } + if transport == nil { + return "unavailable" + } + if *transport { + return "unknown" + } + return "unscoped" } // ChannelSummary is the minimal channel representation used in list responses. diff --git a/internal/ingest/backfill.go b/internal/ingest/backfill.go index f402a4de..619941a3 100644 --- a/internal/ingest/backfill.go +++ b/internal/ingest/backfill.go @@ -32,6 +32,7 @@ type DecryptGroupTextResult struct { // NewMessage is false if InsertChannelMessage found the message already existed // (packet_hash is unique per message) -- e.g. re-running the backfill twice. NewMessage bool + Message *InsertedChannelMessage } // DecryptGroupText attempts to decrypt a GRP_TXT payload against keys and, on success, @@ -77,7 +78,7 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa if err != nil { return nil, fmt.Errorf("insert channel message: %w", err) } - if newMsg { + if newMsg != nil { // Non-fatal: the message is stored either way, so just log and continue -- matches // the live ingest path's existing behavior of not treating this as a hard failure. if err := db.SetPacketDecrypted(ctx, packetHash); err != nil { @@ -89,7 +90,8 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa ChannelID: channelID, ChannelHash: channelHashBytes, Entry: usedEntry, - NewMessage: newMsg, + NewMessage: newMsg != nil, + Message: newMsg, }, nil } diff --git a/internal/ingest/channel_scope_test.go b/internal/ingest/channel_scope_test.go new file mode 100644 index 00000000..3ee69334 --- /dev/null +++ b/internal/ingest/channel_scope_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "github.com/meshcore-go/meshcore-go" +) + +type scopeMessageDB struct { + *stubDB + next *InsertedChannelMessage +} + +func (s *scopeMessageDB) InsertChannelMessage(context.Context, InsertChannelMessageParams) (*InsertedChannelMessage, error) { + value := s.next + s.next = nil + return value, nil +} + +func TestChannelMessageScopeLive(t *testing.T) { + for _, status := range []string{"matched", "unscoped", "unknown", "unavailable"} { + t.Run(status, func(t *testing.T) { + w, base := newTestWorker() + stored := &InsertedChannelMessage{ID: 42, ScopeStatus: api.ChannelScopeStatus(status)} + name := "#stored" + if status == "matched" { + stored.Scope = &name + } + w.db = &scopeMessageDB{stubDB: base, next: stored} + psk := make([]byte, 16) + w.keys = &mapKeys{entries: map[byte][]keystore.Entry{0x11: {{Key: psk, Fingerprint: []byte{1}, Name: "fixture"}}}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "scope", hub.Scope{Events: []hub.EventType{hub.EventChannelMessage, hub.EventObserverStatus}}) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeTransportFlood, meshcore.PayloadTypeGrpTxt, 0), Payload: encryptedGroupText(t, 0x11, psk, "Fixture", "test")} + latest := "#different-later-reception" + w.handlePayloadTypeSideEffects(ctx, packet, "YOW", []byte{1}, RadioSettings{}, nil, &latest, nil, 0) + select { + case event := <-client.Send: + var got channelMessageEvent + if err := json.Unmarshal(event.Payload, &got); err != nil { + t.Fatal(err) + } + if got.ID != 42 || got.ScopeStatus != stored.ScopeStatus || (got.Scope == nil) != (stored.Scope == nil) || (got.Scope != nil && *got.Scope != name) { + t.Fatal("live scope did not use stored evidence", got) + } + case <-ctx.Done(): + t.Fatal("no live message") + } + w.handlePayloadTypeSideEffects(ctx, packet, "YOW", []byte{1}, RadioSettings{}, nil, &latest, nil, 0) + w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + select { + case event := <-client.Send: + if event.Type == hub.EventChannelMessage { + t.Fatal("duplicate broadcast") + } + case <-ctx.Done(): + t.Fatal("no barrier") + } + }) + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 0c4d7f82..fa505324 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -132,7 +132,7 @@ type DB interface { UpsertNodeShortID(ctx context.Context, nodeID uuid.UUID, iata string, prefix4 []byte) error // InsertChannelMessage stores a decrypted group text message. Returns insert success and an error. - InsertChannelMessage(ctx context.Context, m InsertChannelMessageParams) (bool, error) + InsertChannelMessage(ctx context.Context, m InsertChannelMessageParams) (*InsertedChannelMessage, error) // UpdateObserverStatus updates the observer row from a /status message. Returns the OberserID // and any error. diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 2090f0e5..248d3a4b 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -231,8 +231,11 @@ func (s *stubDB) GetNodesByIDs(_ context.Context, _ []uuid.UUID) (map[uuid.UUID] return nil, nil } -func (s *stubDB) InsertChannelMessage(_ context.Context, _ InsertChannelMessageParams) (bool, error) { - return s.insertChannelMessageResult, nil +func (s *stubDB) InsertChannelMessage(_ context.Context, _ InsertChannelMessageParams) (*InsertedChannelMessage, error) { + if !s.insertChannelMessageResult { + return nil, nil + } + return &InsertedChannelMessage{ID: 1, ScopeStatus: "unavailable"}, nil } func (s *stubDB) UpdateObserverStatus(_ context.Context, _ UpdateObserverStatusParams) (uuid.UUID, error) { diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 5d5c92c3..6f809fef 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -37,14 +37,24 @@ type InsertChannelMessageParams struct { SentAt time.Time } +// InsertedChannelMessage is returned only for a new message, using its stored packet evidence. +type InsertedChannelMessage struct { + ID int64 + Scope *string + ScopeStatus api.ChannelScopeStatus +} + // channelMessageEvent is the JSON payload for a channelMessage WS event. type channelMessageEvent struct { - ChannelID int `json:"channelId"` - ChannelHash string `json:"channelHash"` // hex-encoded single byte - PacketHash string `json:"packetHash"` // hex-encoded - SenderName string `json:"senderName"` - Content string `json:"content"` - SentAt int64 `json:"sentAt"` // epoch ms + ID int64 `json:"id"` + ChannelID int `json:"channelId"` + ChannelHash string `json:"channelHash"` // hex-encoded single byte + PacketHash string `json:"packetHash"` // hex-encoded + SenderName string `json:"senderName"` + Content string `json:"content"` + SentAt int64 `json:"sentAt"` // epoch ms + Scope *string `json:"scope"` + ScopeStatus api.ChannelScopeStatus `json:"scopeStatus"` } // nodeUpdateEvent is the JSON payload for a nodeUpdate WS event. @@ -207,6 +217,9 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc if result.NewMessage { evt := channelMessageEvent{ + ID: result.Message.ID, + Scope: result.Message.Scope, + ScopeStatus: result.Message.ScopeStatus, ChannelID: result.ChannelID, ChannelHash: hex.EncodeToString(channelHashBytes), PacketHash: hex.EncodeToString(packetHash), From abb6a3b9cfa4e0f2055994a6e677d898629634e1 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 19:22:37 -0400 Subject: [PATCH 078/141] test(channels): give stored packet fixtures explicit receive times --- db/channel_scopes_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/channel_scopes_integration_test.go b/db/channel_scopes_integration_test.go index 69591aab..b11e9e94 100644 --- a/db/channel_scopes_integration_test.go +++ b/db/channel_scopes_integration_test.go @@ -53,8 +53,8 @@ INSERT INTO transport_scopes(id,name,transport_key,key_fingerprint) VALUES (1,'#yow',decode(repeat('00',16),'hex'),decode(repeat('01',8),'hex')), (2,'#can',decode(repeat('02',16),'hex'),decode(repeat('03',8),'hex')); INSERT INTO channels(id,channel_hash,key_fingerprint,name,is_hashtag,key_known) VALUES (123,'\x11','\x01020304','#fixture',true,true); -INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,transport_codes_present,scope_id,raw_payload,raw_header) -SELECT decode(lpad(to_hex(i),2,'0'),'hex'),5,0,1,transport,scope_id,'\x00','\x15' +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,transport_codes_present,scope_id,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),5,0,1,transport,scope_id,'\x00','\x15',NOW(),NOW() FROM (VALUES(1,true,1),(2,true,NULL),(3,false,NULL),(4,NULL,NULL),(5,true,2),(6,false,NULL)) v(i,transport,scope_id); INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) SELECT packet_hash,md5(n::text)::uuid,'YOW',NOW(),0,1,0 FROM packets CROSS JOIN generate_series(1,2) n; From 7d3a3e289ea749f4d3fecabd210792e9f316cc3e Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 19:24:19 -0400 Subject: [PATCH 079/141] test(channels): verify scope metadata through all HTTP message routes --- db/channel_scopes_integration_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/db/channel_scopes_integration_test.go b/db/channel_scopes_integration_test.go index b11e9e94..dc3ab9b8 100644 --- a/db/channel_scopes_integration_test.go +++ b/db/channel_scopes_integration_test.go @@ -6,13 +6,16 @@ package db import ( "context" "encoding/json" + "net/http/httptest" "os" "testing" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/go-chi/chi/v5" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) @@ -125,6 +128,23 @@ SELECT packet_hash,md5(n::text)::uuid,'YOW',NOW(),0,1,0 FROM packets CROSS JOIN t.Fatal(err) } check(catchup) + router := chi.NewRouter() + router.Mount("/channels", handlers.ChannelsRouter(s)) + router.Mount("/messages", handlers.MessagesRouter(s)) + for _, path := range []string{"/channels/123/messages", "/messages", "/messages?channelHash=11", "/messages/backfill?after=0"} { + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest("GET", path, nil).WithContext(ctx)) + if response.Code != 200 { + t.Fatalf("%s: %d %s", path, response.Code, response.Body.String()) + } + var body struct { + Items []api.ChannelMessage `json:"items"` + } + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + check(body.Items) + } filtered, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, []string{"YOW"}, "#yow", 0) if err != nil || len(filtered.Items) != 1 || filtered.Items[0].Scope == nil || *filtered.Items[0].Scope != "#yow" { t.Fatal(filtered, err) From 07335e6625f2f815b93794c23e945ba64fe9da03 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 19:24:52 -0400 Subject: [PATCH 080/141] test(channels): use the documented catch-up cursor parameter --- db/channel_scopes_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/channel_scopes_integration_test.go b/db/channel_scopes_integration_test.go index dc3ab9b8..dc74951c 100644 --- a/db/channel_scopes_integration_test.go +++ b/db/channel_scopes_integration_test.go @@ -131,7 +131,7 @@ SELECT packet_hash,md5(n::text)::uuid,'YOW',NOW(),0,1,0 FROM packets CROSS JOIN router := chi.NewRouter() router.Mount("/channels", handlers.ChannelsRouter(s)) router.Mount("/messages", handlers.MessagesRouter(s)) - for _, path := range []string{"/channels/123/messages", "/messages", "/messages?channelHash=11", "/messages/backfill?after=0"} { + for _, path := range []string{"/channels/123/messages", "/messages", "/messages?channelHash=11", "/messages/backfill?afterId=0"} { response := httptest.NewRecorder() router.ServeHTTP(response, httptest.NewRequest("GET", path, nil).WithContext(ctx)) if response.Code != 200 { From 8f6032877b3411aab4444cd24a5bbe134c265809 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 19:27:41 -0400 Subject: [PATCH 081/141] test(channels): preserve the existing catch-up response shape --- db/channel_scopes_integration_test.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/db/channel_scopes_integration_test.go b/db/channel_scopes_integration_test.go index dc74951c..e02f5766 100644 --- a/db/channel_scopes_integration_test.go +++ b/db/channel_scopes_integration_test.go @@ -137,13 +137,21 @@ SELECT packet_hash,md5(n::text)::uuid,'YOW',NOW(),0,1,0 FROM packets CROSS JOIN if response.Code != 200 { t.Fatalf("%s: %d %s", path, response.Code, response.Body.String()) } - var body struct { - Items []api.ChannelMessage `json:"items"` - } - if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { - t.Fatal(err) + if path == "/messages/backfill?afterId=0" { + var items []api.ChannelMessage + if err := json.Unmarshal(response.Body.Bytes(), &items); err != nil { + t.Fatal(err) + } + check(items) + } else { + var body struct { + Items []api.ChannelMessage `json:"items"` + } + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + check(body.Items) } - check(body.Items) } filtered, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, []string{"YOW"}, "#yow", 0) if err != nil || len(filtered.Items) != 1 || filtered.Items[0].Scope == nil || *filtered.Items[0].Scope != "#yow" { From 6bf58acb805ffc34bee1a3791e2b1cb76ab24e6a Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 18:27:42 -0400 Subject: [PATCH 082/141] fix: address maintainer review for #176 --- docs/channel-scopes.md | 16 ---------------- internal/ingest/backfill.go | 9 +++------ internal/ingest/backfill_test.go | 4 ++-- internal/ingest/ingest.go | 2 +- internal/ingest/side_effects.go | 4 ++-- 5 files changed, 8 insertions(+), 27 deletions(-) delete mode 100644 docs/channel-scopes.md diff --git a/docs/channel-scopes.md b/docs/channel-scopes.md deleted file mode 100644 index c88aaaea..00000000 --- a/docs/channel-scopes.md +++ /dev/null @@ -1,16 +0,0 @@ -# Channel-message transport scope - -Channel history, the global/hash message APIs, reconnect backfill and `channelMessage` events now include `scope` and `scopeStatus`. Live events also include their stored message `id`; existing fields, endpoints and cursors are unchanged. - -| Status | Meaning | -|---|---| -| `matched` | `scope` names the scope recorded on the first stored packet. | -| `unscoped` | The first stored packet has no transport codes; `scope` is null. | -| `unknown` | Transport codes were recorded but no scope name was resolved, including ambiguous matches. | -| `unavailable` | Scope/capture metadata is missing. | - -These labels belong to the first stored packet, not every later reception of the same payload. Later paths may differ. Live delivery reads that same evidence while inserting the channel message, so a later reception cannot relabel it differently from history. Existing scope query filters retain their meaning. Imported catalogue membership and channel names are never used as message evidence. - -The insertion query returns metadata only for a new message. Duplicate deliveries still produce no extra live message. Adding a decryption key later uses the existing backfill path, retains the stored packet evidence and does not broadcast old messages as new traffic. This change adds no keys, migration or historical scan. Expired packet/message detail remains unavailable. - -Deploy this API before its web consumer. Older clients may ignore the added fields; clients reading older servers must treat absent metadata as unavailable, not assume a message is unscoped. Scope filtering of live events requires the new fields. The channel encryption key and the packet transport scope are separate concepts. diff --git a/internal/ingest/backfill.go b/internal/ingest/backfill.go index 619941a3..ecb7ebb8 100644 --- a/internal/ingest/backfill.go +++ b/internal/ingest/backfill.go @@ -29,10 +29,8 @@ type DecryptGroupTextResult struct { ChannelID int ChannelHash []byte Entry keystore.Entry - // NewMessage is false if InsertChannelMessage found the message already existed - // (packet_hash is unique per message) -- e.g. re-running the backfill twice. - NewMessage bool - Message *InsertedChannelMessage + // Message is nil when the packet already has a stored channel message. + Message *InsertedChannelMessage } // DecryptGroupText attempts to decrypt a GRP_TXT payload against keys and, on success, @@ -90,7 +88,6 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa ChannelID: channelID, ChannelHash: channelHashBytes, Entry: usedEntry, - NewMessage: newMsg != nil, Message: newMsg, }, nil } @@ -117,7 +114,7 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) ( slog.Error(fmt.Sprintf("ingest: backfill: decrypt failed for packet %s", hex.EncodeToString(p.PacketHash)), "component", "ingest", "error", err) continue } - if result != nil && result.NewMessage { + if result != nil && result.Message != nil { decrypted++ } } diff --git a/internal/ingest/backfill_test.go b/internal/ingest/backfill_test.go index 748332ee..1bbeeef6 100644 --- a/internal/ingest/backfill_test.go +++ b/internal/ingest/backfill_test.go @@ -49,8 +49,8 @@ func TestDecryptGroupText_Success(t *testing.T) { if result.Payload.Sender != "ded" || result.Payload.Text != "hello" { t.Errorf("expected decrypted sender=ded text=hello, got sender=%s text=%s", result.Payload.Sender, result.Payload.Text) } - if !result.NewMessage { - t.Error("expected NewMessage true when InsertChannelMessage reports a new insert") + if result.Message == nil { + t.Error("expected a message when InsertChannelMessage reports a new insert") } if db.upsertChannelCalls != 1 { t.Errorf("expected UpsertChannel to be called once, got %d", db.upsertChannelCalls) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index fa505324..289a0def 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -131,7 +131,7 @@ type DB interface { // UpsertNodeShortID upserts a node_short_ids row for path resolution. UpsertNodeShortID(ctx context.Context, nodeID uuid.UUID, iata string, prefix4 []byte) error - // InsertChannelMessage stores a decrypted group text message. Returns insert success and an error. + // InsertChannelMessage stores a decrypted group text message. Returns the inserted message, nil for a duplicate, and an error. InsertChannelMessage(ctx context.Context, m InsertChannelMessageParams) (*InsertedChannelMessage, error) // UpdateObserverStatus updates the observer row from a /status message. Returns the OberserID diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 6f809fef..09132424 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -54,7 +54,7 @@ type channelMessageEvent struct { Content string `json:"content"` SentAt int64 `json:"sentAt"` // epoch ms Scope *string `json:"scope"` - ScopeStatus api.ChannelScopeStatus `json:"scopeStatus"` + ScopeStatus api.ChannelScopeStatus `json:"scopeStatus" enums:"matched,unscoped,unknown,unavailable"` } // nodeUpdateEvent is the JSON payload for a nodeUpdate WS event. @@ -215,7 +215,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc return } - if result.NewMessage { + if result.Message != nil { evt := channelMessageEvent{ ID: result.Message.ID, Scope: result.Message.Scope, From 3b806c1c395373149769db88116b14cfcfa87a50 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 03:12:13 -0400 Subject: [PATCH 083/141] fix(ingest): resolve advert endpoints after node updates --- .../ingest/advert_endpoint_update_test.go | 144 ++++++++++++++++++ internal/ingest/endpoint_matching_test.go | 4 + internal/ingest/packet.go | 14 +- 3 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 internal/ingest/advert_endpoint_update_test.go diff --git a/internal/ingest/advert_endpoint_update_test.go b/internal/ingest/advert_endpoint_update_test.go new file mode 100644 index 00000000..aea19c4d --- /dev/null +++ b/internal/ingest/advert_endpoint_update_test.go @@ -0,0 +1,144 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +type advertEndpointDB struct { + *endpointCaptureDB + found, duplicate bool + lookups, upserts int +} + +func (s *advertEndpointDB) GetNodeByPubkey(context.Context, []byte) (uuid.UUID, error) { + if !s.found { + return uuid.Nil, errors.New("node not found") + } + return s.node.ID, nil +} + +func (s *advertEndpointDB) GetNodesByIDs(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]*api.ResolvedNode, error) { + s.lookups++ + return s.endpointCaptureDB.GetNodesByIDs(ctx, ids) +} + +func (s *advertEndpointDB) UpsertNode(_ context.Context, n UpsertNodeParams, _ RadioSettings) (uuid.UUID, error) { + s.node.Name = &n.Name + s.found = true + s.upserts++ + return s.node.ID, nil +} + +func (s *advertEndpointDB) InsertObservation(context.Context, InsertObservationParams) (bool, error) { + return !s.duplicate, nil +} + +func TestAdvertEndpointUsesUpdatedName(t *testing.T) { + for _, existing := range []bool{false, true} { + t.Run(map[bool]string{false: "first advert", true: "renamed node"}[existing], func(t *testing.T) { + w, base := newTestWorker() + old := "Previous repeater" + store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: base, node: api.ResolvedNode{ID: uuid.New(), Name: &old, PublicKey: "aa"}}, found: existing} + w.db = store + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + w.hub.AddScope(client, "rename", hub.Scope{Events: []hub.EventType{hub.EventPacketObservation, hub.EventObserverStatus}}) + defer w.hub.Remove(client) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := buildAdvertPacketWithData(t, append([]byte{meshcore.AdvertTypeRepeater | meshcore.AdvertNameMask}, []byte("Renamed repeater")...), false) + w.handlePacket(ctx, "YYZ", "0102", packetEnvelope(t, packet)) + for { + select { + case event := <-client.Send: + if event.Type != hub.EventPacketObservation { + continue + } + var got packetObservationEvent + if err := json.Unmarshal(event.Payload, &got); err != nil { + t.Fatal(err) + } + hop := got.Observation.ResolvedSource + if hop == nil || len(hop.Nodes) != 1 || hop.Nodes[0].Name == nil || *hop.Nodes[0].Name != "Renamed repeater" { + t.Fatalf("live source did not use the newly saved name: %+v", hop) + } + if store.upserts != 1 || hop.Nodes[0].ID != store.node.ID || hop.Confidence != "high" { + t.Fatal("advert identity or confidence changed") + } + return + case <-ctx.Done(): + t.Fatal("packet event missing") + } + } + }) + } +} + +func TestDuplicateAdvertSkipsLiveEndpointLookup(t *testing.T) { + w, base := newTestWorker() + store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: base, node: api.ResolvedNode{ID: uuid.New(), PublicKey: "aa"}}, found: true, duplicate: true} + w.db = store + w.handlePacket(context.Background(), "YYZ", "0102", packetEnvelope(t, buildAdvertPacket(t, false))) + if store.lookups != 0 || store.upserts != 0 { + t.Fatalf("duplicate observation performed live-only work: lookups=%d upserts=%d", store.lookups, store.upserts) + } +} + +func TestRepeatAdvertUsesCurrentIdentityWithoutReapplyingAdvert(t *testing.T) { + r := newRepeatHarness(t, true) + store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: r.db.stubDB, node: api.ResolvedNode{ID: uuid.New(), PublicKey: "aa"}}} + r.w.db = store + packet := buildAdvertPacketWithData(t, append([]byte{meshcore.AdvertTypeRepeater | meshcore.AdvertNameMask}, []byte("Advert name")...), false) + hear := func(path byte) []packetObservationEvent { + packet.Path, packet.PathLength = []byte{path}, 1 + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(t, packet)) + r.w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + var events []packetObservationEvent + for { + select { + case event := <-r.client.Send: + if event.Type == hub.EventObserverStatus { + return events + } + var payload packetObservationEvent + if err := json.Unmarshal(event.Payload, &payload); err != nil { + t.Fatal(err) + } + events = append(events, payload) + case <-r.ctx.Done(): + t.Fatal("repeat marker missing") + } + } + } + first := hear(0x11) + if len(first) != 1 || first[0].Packet.IsRepeat || store.upserts != 1 { + t.Fatal("first advert did not produce one stored hearing") + } + // A later advert has updated the identity before this older packet is heard again. + current := "Current name" + store.node.Name, store.duplicate = ¤t, true + repeated := hear(0x22) + if len(repeated) != 1 || !repeated[0].Packet.IsRepeat || repeated[0].Observation.ResolvedSource == nil { + t.Fatal("new-path repeat lost its resolved source") + } + if got := repeated[0].Observation.ResolvedSource.Nodes[0].Name; got == nil || *got != current || store.upserts != 1 { + t.Fatal("repeat reapplied the old advert or returned a stale identity") + } + lookups := store.lookups + if got := hear(0x22); len(got) != 0 || store.lookups != lookups || store.upserts != 1 { + t.Fatal("suppressed repeat performed live-only work") + } +} diff --git a/internal/ingest/endpoint_matching_test.go b/internal/ingest/endpoint_matching_test.go index 52d12195..24596fb7 100644 --- a/internal/ingest/endpoint_matching_test.go +++ b/internal/ingest/endpoint_matching_test.go @@ -19,6 +19,10 @@ type endpointRoutingDB struct { paths []string } +func (s *endpointRoutingDB) InsertObservation(context.Context, InsertObservationParams) (bool, error) { + return true, nil +} + func (s *endpointRoutingDB) ResolveEndpointHashes(_ context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { for _, hash := range hashes { s.endpoints = append(s.endpoints, iata+":"+hex.EncodeToString(hash)) diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 1a99b5e6..086f3271 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -857,7 +857,13 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ // A duplicate is streamed, never stored, to includeRepeats clients when its path is new. repeat := !inserted && w.hub.RepeatsWanted() && w.hub.MarkSent(packetHash[:], id[:], packet.Path) if inserted || repeat { - // Endpoints for the live event only; stored rows resolve them at read time. + if inserted { + w.handlePayloadTypeSideEffects(ctx, packet, iata, packetHash[:], radio, scopeID, matchedScope, pubkeyBytes, float32(parseNumber(envelope.SNR))) + if w.hub.RepeatsWanted() { + w.hub.MarkSent(packetHash[:], id[:], packet.Path) // so broker copies of it aren't repeats + } + } + // Resolve after advert updates; suppressed copies need no endpoint lookup. var resolvedSource, resolvedDestination *api.ResolvedHop if packet.PayloadType() == meshcore.PayloadTypeAdvert && originPubkey != nil { // Exact match: ADVERT carries the sender's real identity pubkey, not a @@ -880,12 +886,6 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ resolvedDestination = &hop } } - if inserted { - w.handlePayloadTypeSideEffects(ctx, packet, iata, packetHash[:], radio, scopeID, matchedScope, pubkeyBytes, float32(parseNumber(envelope.SNR))) - if w.hub.RepeatsWanted() { - w.hub.MarkSent(packetHash[:], id[:], packet.Path) // so broker copies of it aren't repeats - } - } evt := packetObservationEvent{} evt.PacketHash = hex.EncodeToString(packetHash[:]) evt.Packet.PayloadType = packet.PayloadType() From 97cc22a5a0aa2256b4f8f9abe163bd3bc9c8deca Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 18:27:42 -0400 Subject: [PATCH 084/141] fix: address maintainer review for #166 --- internal/ingest/endpoint_matching_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/ingest/endpoint_matching_test.go b/internal/ingest/endpoint_matching_test.go index 24596fb7..52d12195 100644 --- a/internal/ingest/endpoint_matching_test.go +++ b/internal/ingest/endpoint_matching_test.go @@ -19,10 +19,6 @@ type endpointRoutingDB struct { paths []string } -func (s *endpointRoutingDB) InsertObservation(context.Context, InsertObservationParams) (bool, error) { - return true, nil -} - func (s *endpointRoutingDB) ResolveEndpointHashes(_ context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { for _, hash := range hashes { s.endpoints = append(s.endpoints, iata+":"+hex.EncodeToString(hash)) From 89a376c29198ea0bd006953ff3b319e69d9a4082 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 18:37:14 -0400 Subject: [PATCH 085/141] fix(ingest): accept RFC3339 timestamp offsets and precision --- internal/ingest/packet.go | 13 +------ internal/ingest/timestamp_test.go | 64 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 internal/ingest/timestamp_test.go diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 086f3271..10222bae 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -730,20 +730,11 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ w.log.Error(fmt.Sprintf("db: upsert packet failed from %s/%s", iata, pubkeyHex), "error", err) return } - // Try parsing with timezone offset first - heardAt, err := time.Parse("2006-01-02T15:04:05.000000-07:00", envelope.Timestamp) - if err != nil { - heardAt, err = time.Parse("2006-01-02T15:04:05.000000", envelope.Timestamp) - } - if err != nil { - heardAt, err = time.Parse("2006-01-02T15:04:05.000000Z", envelope.Timestamp) - } + // Prefer an explicit RFC3339 offset; legacy timezone-less observers use UTC. + heardAt, err := time.Parse(time.RFC3339Nano, envelope.Timestamp) if err != nil { heardAt, err = time.Parse("2006-01-02T15:04:05", envelope.Timestamp) } - if err != nil { - heardAt, err = time.Parse("2006-01-02T15:04:05Z", envelope.Timestamp) - } if err != nil { w.log.Warn(fmt.Sprintf("failed to parse timestamp %q", envelope.Timestamp), "error", err) heardAt = time.Now().UTC() diff --git a/internal/ingest/timestamp_test.go b/internal/ingest/timestamp_test.go new file mode 100644 index 00000000..1b5daf66 --- /dev/null +++ b/internal/ingest/timestamp_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" +) + +func TestPacketTimestampFormatsAndSkew(t *testing.T) { + want := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Second).Add(123456789 * time.Nanosecond) + capture := func(t *testing.T, timestamp string) time.Time { + t.Helper() + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} + w.db = db + var envelope map[string]string + if err := json.Unmarshal(packetEnvelope(t, buildGrpTxtPacket(t, 0x1a, make([]byte, 16))), &envelope); err != nil { + t.Fatal(err) + } + envelope["timestamp"] = timestamp + body, err := json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + w.handlePacket(context.Background(), "YOW", strings.Repeat("01", 32), body) + if len(db.observed) != 1 { + t.Fatal("timestamp prevented packet storage") + } + return db.observed[0].HeardAt + } + for _, zone := range []*time.Location{time.UTC, time.FixedZone("zero", 0), time.FixedZone("west", -4*60*60), time.FixedZone("east", 19800)} { + for _, precision := range []time.Duration{time.Second, time.Millisecond, time.Microsecond, time.Nanosecond} { + expected := want.Truncate(precision) + stamp := expected.In(zone).Format("2006-01-02T15:04:05.999999999-07:00") + if zone == time.UTC { + stamp = expected.Format(time.RFC3339Nano) + } + t.Run(stamp, func(t *testing.T) { + if got := capture(t, stamp); !got.Equal(expected) { + t.Fatalf("reported time changed: got %s want %s", got, expected) + } + }) + } + } + for _, precision := range []time.Duration{time.Second, time.Microsecond, time.Nanosecond} { + expected := want.Truncate(precision) + stamp := expected.Format("2006-01-02T15:04:05.999999999") + if got := capture(t, stamp); !got.Equal(expected) { + t.Fatalf("timezone-less UTC changed: got %s want %s", got, expected) + } + } + for _, stamp := range []string{"", "invalid", "2026-09-29T25:00:00+00:00", time.Now().Add(31 * time.Minute).Format(time.RFC3339), time.Now().Add(-31 * time.Minute).Format(time.RFC3339)} { + before := time.Now().UTC() + got := capture(t, stamp) + if got.Before(before) || got.After(time.Now().UTC()) { + t.Fatalf("invalid/skewed time escaped server-time guard: %s => %s", stamp, got) + } + } +} From c9b7010082d780eafa4b185c280ab7138e6bc605 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 07:37:40 -0400 Subject: [PATCH 086/141] chore: regenerate sqlc and document saved-route evidence --- README.md | 6 +++++- db/sqlc/querier.go | 2 -- db/sqlc/queries.sql.go | 3 --- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3ecf2d55..cae5c801 100644 --- a/README.md +++ b/README.md @@ -570,4 +570,8 @@ The initial analytics archive migration includes unknown-type activity. Existing development previews that used the earlier draft require a separate operator repair; radio samples already discarded for those legacy rows cannot be recovered. -Saved-route evidence: see [the operator guide](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/app_documentation/saved-route-evidence.md). +### Saved-route evidence + +`GET /api/v1/routes/{iata}/{pathKey}/observations` returns retained observations that match a saved +route's full path bytes, hash size and hop count within its IATA. `range` defaults to `24h` (max `720h`), +`limit` to 50 (max 200); page with `nextPageCursor` → `pageCursor`. Evidence expires with raw packets. diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 1f80ab61..ef65fd7b 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -89,8 +89,6 @@ type Querier interface { GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) - // Copyright 2026 Beacon Contributors - // SPDX-License-Identifier: AGPL-3.0-or-later GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) GetScopeNames(ctx context.Context) ([]string, error) // Aggregate matching observations once, separately from node memberships to avoid diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index e6fb1edc..3b3def70 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1510,7 +1510,6 @@ func (q *Queries) GetScopeByName(ctx context.Context, name string) (GetScopeByNa } const getScopeCatalogue = `-- name: GetScopeCatalogue :one - SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2 ` @@ -1519,8 +1518,6 @@ type GetScopeCatalogueParams struct { Url string `json:"url"` } -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later func (q *Queries) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) { row := q.db.QueryRow(ctx, getScopeCatalogue, arg.Iata, arg.Url) var i MeshmapperScopeCatalogue From fd57c8b00c903de06a894bd294e6eac423884115 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 07:37:40 -0400 Subject: [PATCH 087/141] fix(ingest): accept space-separated and colonless-offset timestamps --- internal/ingest/packet.go | 28 +++++++++++++++++++----- internal/ingest/timestamp_test.go | 36 ++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 10222bae..fda54975 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -730,11 +730,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ w.log.Error(fmt.Sprintf("db: upsert packet failed from %s/%s", iata, pubkeyHex), "error", err) return } - // Prefer an explicit RFC3339 offset; legacy timezone-less observers use UTC. - heardAt, err := time.Parse(time.RFC3339Nano, envelope.Timestamp) - if err != nil { - heardAt, err = time.Parse("2006-01-02T15:04:05", envelope.Timestamp) - } + heardAt, err := parseHeardAt(envelope.Timestamp) if err != nil { w.log.Warn(fmt.Sprintf("failed to parse timestamp %q", envelope.Timestamp), "error", err) heardAt = time.Now().UTC() @@ -959,3 +955,25 @@ func computeTransportCode(key []byte, payloadType uint8, payload []byte) uint16 } return code } + +// heardAtLayouts lists offset forms first; timezone-less observers are read as UTC. +// Fractional seconds are optional in every layout. +var heardAtLayouts = []string{ + time.RFC3339Nano, + "2006-01-02 15:04:05.999999999Z07:00", + "2006-01-02T15:04:05.999999999Z0700", + "2006-01-02 15:04:05.999999999Z0700", + "2006-01-02T15:04:05", + "2006-01-02 15:04:05", +} + +func parseHeardAt(s string) (time.Time, error) { + var err error + for _, layout := range heardAtLayouts { + var t time.Time + if t, err = time.Parse(layout, s); err == nil { + return t, nil + } + } + return time.Time{}, err +} diff --git a/internal/ingest/timestamp_test.go b/internal/ingest/timestamp_test.go index 1b5daf66..2010f63e 100644 --- a/internal/ingest/timestamp_test.go +++ b/internal/ingest/timestamp_test.go @@ -49,16 +49,36 @@ func TestPacketTimestampFormatsAndSkew(t *testing.T) { } for _, precision := range []time.Duration{time.Second, time.Microsecond, time.Nanosecond} { expected := want.Truncate(precision) - stamp := expected.Format("2006-01-02T15:04:05.999999999") - if got := capture(t, stamp); !got.Equal(expected) { - t.Fatalf("timezone-less UTC changed: got %s want %s", got, expected) + for _, layout := range []string{"2006-01-02T15:04:05.999999999", "2006-01-02 15:04:05.999999999"} { + stamp := expected.Format(layout) + t.Run("naive "+stamp, func(t *testing.T) { + if got := capture(t, stamp); !got.Equal(expected) { + t.Fatalf("timezone-less UTC changed: got %s want %s", got, expected) + } + }) } } + west := time.FixedZone("west", -4*60*60) + for _, layout := range []string{ + "2006-01-02 15:04:05.999999-07:00", // Python str(datetime) + "2006-01-02T15:04:05.999-0700", + "2006-01-02 15:04:05-0700", + } { + stamp := want.Truncate(time.Microsecond).In(west).Format(layout) + expected, _ := time.Parse(layout, stamp) + t.Run(stamp, func(t *testing.T) { + if got := capture(t, stamp); !got.Equal(expected) { + t.Fatalf("reported time changed: got %s want %s", got, expected) + } + }) + } for _, stamp := range []string{"", "invalid", "2026-09-29T25:00:00+00:00", time.Now().Add(31 * time.Minute).Format(time.RFC3339), time.Now().Add(-31 * time.Minute).Format(time.RFC3339)} { - before := time.Now().UTC() - got := capture(t, stamp) - if got.Before(before) || got.After(time.Now().UTC()) { - t.Fatalf("invalid/skewed time escaped server-time guard: %s => %s", stamp, got) - } + t.Run("guard "+stamp, func(t *testing.T) { + before := time.Now().UTC() + got := capture(t, stamp) + if got.Before(before) || got.After(time.Now().UTC()) { + t.Fatalf("invalid/skewed time escaped server-time guard: %s => %s", stamp, got) + } + }) } } From 32644ae09f447b4c5227c3b921de3edf44cf5bd2 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 07:38:41 -0400 Subject: [PATCH 088/141] chore: set MrAlders0n as default code owner --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fc14c65d..b8732e9d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,5 +1,5 @@ # Default owners for everything -* @446564 +* @MrAlders0n # EXAMPLE Specific ownership by directory From 5cf060114a8d8134a1f65d2005809e589d8cf623 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Wed, 30 Sep 2026 07:52:59 -0400 Subject: [PATCH 089/141] feat: add bounded CPU profiling (#182) * feat: add bounded CPU profiling * fix(profiling): defer cooldown-skipped samples and configure triggers --- PROFILING.md | 80 ++++++ cmd/beacon/main.go | 5 + cmd/beacon/profiling.go | 31 +++ env.example | 4 + internal/profiling/cpu_other.go | 8 + internal/profiling/cpu_unix.go | 19 ++ internal/profiling/profiling.go | 315 ++++++++++++++++++++++++ internal/profiling/profiling_test.go | 355 +++++++++++++++++++++++++++ 8 files changed, 817 insertions(+) create mode 100644 PROFILING.md create mode 100644 cmd/beacon/profiling.go create mode 100644 internal/profiling/cpu_other.go create mode 100644 internal/profiling/cpu_unix.go create mode 100644 internal/profiling/profiling.go create mode 100644 internal/profiling/profiling_test.go diff --git a/PROFILING.md b/PROFILING.md new file mode 100644 index 00000000..5e3667ee --- /dev/null +++ b/PROFILING.md @@ -0,0 +1,80 @@ +# Production CPU profiling + +CPU profiling is supported on Linux and macOS and is disabled by default. It writes private files inside the container; +there is no HTTP endpoint or published profiling port. Profiling failures stop the +recorder and log a warning without stopping ingest. + +## Enable a bounded session + +Create a dedicated directory writable by the container user with mode `0700`. +Mount it into the app container and set both environment variables: + +```yaml +services: + app: + environment: + BEACON_CPU_PROFILE_DIR: /profiles + BEACON_CPU_PROFILE_UNTIL: "2026-10-02T12:00:00Z" # Replace with your fixed deadline. + volumes: + - ./profiles:/profiles +``` + +Choose an RFC3339 deadline no more than 72 hours in the future. The same deadline +remains in effect after restarts; expired settings are inactive. Do not generate a +fresh deadline automatically on each startup. Restarting the app is required to +change these settings. Preserve the rest of the deployment configuration. + +Start with a deadline about one minute away to assess the first capture's overhead. +Check process CPU, packet freshness, queue drops and database timeouts against a +comparable period with profiling disabled. After reviewing that capture, an operator +can enable a longer session. Profiling adds overhead while a capture is active. + +## Capture behavior + +- One 30-second sample immediately, then every 30 minutes. +- Route reconfirmation requests an additional sample when the maintenance task starts. + This includes the retention step before route validation. A five-minute cooldown + between capture starts prevents overlap and repeated triggers from increasing load; + a periodic sample due during the cooldown runs when it ends. +- Background task stacks carry a `task` label while profiling is enabled. +- Shutdown or expiry stops the active sample and saves the shorter profile. +- Each profile is limited to 8 MiB. The dedicated directory is limited to 256 MiB + and 512 files, including metadata and files left by interrupted runs. The recorder + reserves space for a full capture before starting and stops when a limit is reached. + Files are never automatically deleted. Existing unrelated files consume the budget. +- Profiles and metadata are written with mode `0600`. A `.partial` file indicates + an interrupted capture and is not a completed profile. + +Only one Beacon process should write to a profiling directory. Keep the mount +private and out of web roots, backups intended for public download, and source control. +Do not run another CPU profiler in the same process during a session. + +## Interpret the output + +Each `.pprof` has a `.pprof.json` sidecar containing UTC start/end times, the trigger, +Go version, process CPU counters (Linux/macOS), goroutine count, and database-pool +counters before and after the capture. Pool acquire duration and counts are cumulative; +use differences between the two snapshots. They do not contain SQL text or credentials. + +Copy completed files privately off the host. Record the container's image revision +and digest alongside them. On a workstation with Go installed: + +```sh +go tool pprof -top capture.pprof +go tool pprof -top -cum capture.pprof +go tool pprof -tags capture.pprof +``` + +CPU samples show work inside Beacon, not CPU used by PostgreSQL or time waiting on +locks. Correlate each UTC interval with separately collected host/container CPU and +I/O, PostgreSQL wait states, observation throughput, and Beacon timeout/drop logs. +Do not enable SQL text logging or run full-table counts for this purpose. The recorder +makes no diagnostic database queries and does not change the ingest path. + +Review captures from quiet periods, bursts, and maintenance separately before +combining them. Additional maintenance samples deliberately bias an aggregate profile. +Thirty-second windows can miss brief stalls; the absence of a stack is not proof that +it never consumes CPU. + +At the deadline, check for `CPU profiling stopped` in the app log. Export the files, +then remove the environment variables and mount during the next planned deployment. diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 1fa2ea38..24e60847 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -318,6 +318,11 @@ func main() { if scopeImporter != nil { tasks = append(tasks, background.Task{Name: "meshmapper.scopes", Interval: meshmapper.PollInterval, Run: scopeImporter.Refresh}) } + profiles := configureProfiling(ctx, pool) + defer profiles.Stop() + for i := range tasks { + tasks[i].Run = profiles.WrapTask(tasks[i].Name, tasks[i].Run) + } scheduler := background.New(tasks) go scheduler.Start(ctx) diff --git a/cmd/beacon/profiling.go b/cmd/beacon/profiling.go new file mode 100644 index 00000000..df65d4cb --- /dev/null +++ b/cmd/beacon/profiling.go @@ -0,0 +1,31 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "context" + "log/slog" + "os" + + "github.com/MeshCore-Beacon/beacon-server/internal/profiling" + "github.com/jackc/pgx/v5/pgxpool" +) + +func configureProfiling(ctx context.Context, pool *pgxpool.Pool) *profiling.Recorder { + r, err := profiling.Start(ctx, os.Getenv("BEACON_CPU_PROFILE_DIR"), os.Getenv("BEACON_CPU_PROFILE_UNTIL"), []string{"reconfirm"}, func() any { + s := pool.Stat() + return struct { + Acquired, Idle, Total, Max int32 + Acquires, EmptyAcquires, CanceledAcquires int64 + AcquireDurationNS int64 + }{ + s.AcquiredConns(), s.IdleConns(), s.TotalConns(), s.MaxConns(), + s.AcquireCount(), s.EmptyAcquireCount(), s.CanceledAcquireCount(), int64(s.AcquireDuration()), + } + }) + if err != nil { + slog.Warn("CPU profiling unavailable", "component", "profiling", "error", err) + } + return r +} diff --git a/env.example b/env.example index 97d7cbc4..b918e6e3 100644 --- a/env.example +++ b/env.example @@ -21,3 +21,7 @@ MQTT_BROKER_1_PASSWORD= MQTT_BROKER_2_URL=wss://mqtt2.meshcore.ca:443 MQTT_BROKER_2_USERNAME= MQTT_BROKER_2_PASSWORD= + +# Optional private CPU captures. Set both; the deadline must be within 72 hours. +# BEACON_CPU_PROFILE_DIR=/profiles +# BEACON_CPU_PROFILE_UNTIL=2026-10-02T12:00:00Z diff --git a/internal/profiling/cpu_other.go b/internal/profiling/cpu_other.go new file mode 100644 index 00000000..9c94f146 --- /dev/null +++ b/internal/profiling/cpu_other.go @@ -0,0 +1,8 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build !linux && !darwin + +package profiling + +func cpuUsage() *processCPU { return nil } diff --git a/internal/profiling/cpu_unix.go b/internal/profiling/cpu_unix.go new file mode 100644 index 00000000..77da9f73 --- /dev/null +++ b/internal/profiling/cpu_unix.go @@ -0,0 +1,19 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +//go:build linux || darwin + +package profiling + +import "syscall" + +func cpuUsage() *processCPU { + var usage syscall.Rusage + if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) != nil { + return nil + } + return &processCPU{ + UserSeconds: float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6, + SystemSeconds: float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6, + } +} diff --git a/internal/profiling/profiling.go b/internal/profiling/profiling.go new file mode 100644 index 00000000..ca836d9d --- /dev/null +++ b/internal/profiling/profiling.go @@ -0,0 +1,315 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package profiling records bounded, opt-in CPU samples to private files. +package profiling + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "time" +) + +const ( + captureDuration = 30 * time.Second + captureInterval = 30 * time.Minute + captureCooldown = 5 * time.Minute + maxProfileBytes = 8 << 20 + maxMetadataBytes = 16 << 10 + maxDirectoryBytes = 256 << 20 + maxDirectoryFiles = 512 +) + +type Recorder struct { + root *os.Root + snapshot func() any + triggers map[string]bool + requests chan string + cancel context.CancelFunc + done chan struct{} +} + +type snapshot struct { + ProcessCPU *processCPU `json:",omitempty"` + Goroutines int + Pool any `json:",omitempty"` +} + +type metadata struct { + Reason string + StartedAt time.Time + FinishedAt time.Time + GoVersion string + Before snapshot + After snapshot +} + +// Start leaves expired settings inactive so restarts cannot extend a capture window. +// Tasks named in triggers request an extra capture when they start. +func Start(parent context.Context, dir, until string, triggers []string, poolSnapshot func() any) (*Recorder, error) { + if dir == "" && until == "" { + return nil, nil + } + if dir == "" || until == "" { + return nil, errors.New("profiling requires both directory and deadline") + } + deadline, err := time.Parse(time.RFC3339Nano, until) + if err != nil { + return nil, fmt.Errorf("invalid profiling deadline: %w", err) + } + if !deadline.After(time.Now()) { + slog.Info("CPU profiling deadline has passed; profiling stays off", "component", "profiling", "until", deadline.UTC()) + return nil, nil + } + if time.Until(deadline) > 72*time.Hour { + return nil, errors.New("profiling deadline must be within 72 hours") + } + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + return nil, errors.New("private CPU profiling is supported on Linux and macOS") + } + if !filepath.IsAbs(dir) { + return nil, errors.New("profiling directory must be absolute") + } + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, err + } + info, err := os.Lstat(dir) + if err != nil { + return nil, err + } + if !info.IsDir() || info.Mode().Perm()&0077 != 0 { + return nil, errors.New("profiling directory must be private (0700) and not a symlink") + } + root, err := os.OpenRoot(dir) + if err != nil { + return nil, err + } + ctx, cancel := context.WithDeadline(parent, deadline) + r := &Recorder{root: root, snapshot: poolSnapshot, triggers: make(map[string]bool, len(triggers)), requests: make(chan string, 1), cancel: cancel, done: make(chan struct{})} + for _, name := range triggers { + r.triggers[name] = true + } + slog.Info("CPU profiling enabled", "component", "profiling", "until", deadline.UTC()) + go r.run(ctx) + return r, nil +} + +func (r *Recorder) Stop() { + if r == nil { + return + } + r.cancel() + <-r.done +} + +// request never holds up maintenance or queues a backlog of captures. +func (r *Recorder) request(reason string) { + if r == nil { + return + } + select { + case <-r.done: + return + default: + } + select { + case r.requests <- reason: + default: + } +} + +func (r *Recorder) WrapTask(name string, run func(context.Context) error) func(context.Context) error { + if r == nil { + return run + } + return func(ctx context.Context) (err error) { + select { + case <-r.done: + return run(ctx) + default: + } + if r.triggers[name] { + r.request(name) + } + pprof.Do(ctx, pprof.Labels("task", name), func(ctx context.Context) { err = run(ctx) }) + return err + } +} + +func (r *Recorder) run(ctx context.Context) { + defer close(r.done) + defer r.root.Close() + defer r.cancel() + defer slog.Info("CPU profiling stopped", "component", "profiling") + if err := schedule(ctx, r.requests, r.capture); err != nil { + slog.Warn("CPU profiling stopped after capture failure", "component", "profiling", "error", err) + } +} + +func schedule(ctx context.Context, requests <-chan string, capture func(context.Context, string) error) error { + ticker := time.NewTicker(captureInterval) + defer ticker.Stop() + reason := "periodic" + var last time.Time + var deferred <-chan time.Time + for { + if ctx.Err() != nil { + return nil + } + if wait := captureCooldown - time.Since(last); last.IsZero() || wait <= 0 { + last = time.Now() + if err := capture(ctx, reason); err != nil { + return err + } + } else if reason == "periodic" && deferred == nil { + // Run a periodic sample after the cooldown rather than skipping a whole interval. + deferred = time.After(wait) + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + reason = "periodic" + case <-deferred: + deferred = nil + reason = "periodic" + case reason = <-requests: + } + } +} + +func (r *Recorder) takeSnapshot() snapshot { + s := snapshot{Goroutines: runtime.NumGoroutine()} + s.ProcessCPU = cpuUsage() + if r.snapshot != nil { + s.Pool = r.snapshot() + } + return s +} + +func (r *Recorder) checkBudget() error { + d, err := r.root.Open(".") + if err != nil { + return err + } + entries, err := d.ReadDir(maxDirectoryFiles + 1) + d.Close() + if err != nil && err != io.EOF { + return err + } + if len(entries) > maxDirectoryFiles-2 { + return errors.New("profiling file budget exhausted") + } + var total int64 + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return errors.New("profiling directory contains a non-regular file") + } + total += info.Size() + } + if total > maxDirectoryBytes-maxProfileBytes-maxMetadataBytes { + return errors.New("profiling storage budget exhausted") + } + return nil +} + +func (r *Recorder) capture(ctx context.Context, reason string) error { + if err := r.checkBudget(); err != nil { + return err + } + started := time.Now().UTC() + name := started.Format("20060102T150405.000000000Z") + ".pprof" + partial := name + ".partial" + f, err := r.root.OpenFile(partial, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer r.root.Remove(partial) + w := &limitedWriter{writer: f, remaining: maxProfileBytes} + m := metadata{Reason: reason, StartedAt: started, GoVersion: runtime.Version(), Before: r.takeSnapshot()} + if err := pprof.StartCPUProfile(w); err != nil { + f.Close() + return err + } + slog.Info("CPU profiling capture started", "component", "profiling", "file", name, "reason", reason) + timer := time.NewTimer(captureDuration) + select { + case <-ctx.Done(): + case <-timer.C: + } + timer.Stop() + pprof.StopCPUProfile() + m.FinishedAt = time.Now().UTC() + m.After = r.takeSnapshot() + closeErr := f.Close() + if w.err != nil { + return w.err + } + if closeErr != nil { + return closeErr + } + if err := r.root.Rename(partial, name); err != nil { + return err + } + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + if len(data) > maxMetadataBytes { + return errors.New("profiling metadata budget exhausted") + } + meta, err := r.root.OpenFile(name+".json", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + _, err = meta.Write(data) + closeErr = meta.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + slog.Info("CPU profile saved", "component", "profiling", "file", name, "reason", reason, "duration", m.FinishedAt.Sub(started)) + return nil +} + +type limitedWriter struct { + writer io.Writer + remaining int + err error +} + +func (w *limitedWriter) Write(p []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + if len(p) > w.remaining { + w.err = errors.New("CPU profile exceeded size limit") + return 0, w.err + } + n, err := w.writer.Write(p) + w.remaining -= n + if err == nil && n != len(p) { + err = io.ErrShortWrite + } + w.err = err + return n, err +} + +type processCPU struct { + UserSeconds float64 + SystemSeconds float64 +} diff --git a/internal/profiling/profiling_test.go b/internal/profiling/profiling_test.go new file mode 100644 index 00000000..4a307db6 --- /dev/null +++ b/internal/profiling/profiling_test.go @@ -0,0 +1,355 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package profiling + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "reflect" + "runtime" + "runtime/pprof" + "testing" + "testing/synctest" + "time" +) + +func TestDisabledAndExpired(t *testing.T) { + for _, tc := range []struct{ dir, until string }{ + {}, {filepath.Join(t.TempDir(), "unused"), time.Now().Add(-time.Hour).Format(time.RFC3339)}, + } { + r, err := Start(context.Background(), tc.dir, tc.until, nil, nil) + if err != nil || r != nil { + t.Fatalf("got %v, %v", r, err) + } + if tc.dir != "" { + if _, err := os.Stat(tc.dir); !os.IsNotExist(err) { + t.Fatalf("expired capture created directory: %v", err) + } + } + } +} + +func TestRejectUnsafeConfiguration(t *testing.T) { + for _, tc := range []struct{ name, dir, until string }{ + {"missing deadline", t.TempDir(), ""}, + {"missing directory", "", time.Now().Add(time.Hour).Format(time.RFC3339)}, + {"invalid deadline", t.TempDir(), "tomorrow"}, + {"unbounded window", t.TempDir(), time.Now().Add(73 * time.Hour).Format(time.RFC3339)}, + {"relative directory", "profiles", time.Now().Add(time.Hour).Format(time.RFC3339)}, + } { + t.Run(tc.name, func(t *testing.T) { + r, err := Start(context.Background(), tc.dir, tc.until, nil, nil) + if r != nil { + r.Stop() + } + if err == nil { + t.Fatal("accepted unsafe configuration") + } + }) + } + dir := t.TempDir() + if err := os.Chmod(dir, 0755); err != nil { + t.Fatal(err) + } + r, err := Start(context.Background(), dir, time.Now().Add(time.Hour).Format(time.RFC3339), nil, nil) + if r != nil { + r.Stop() + } + if err == nil { + t.Fatal("accepted public directory") + } +} + +func TestDeadlineProducesPrivateProfileAndMetadata(t *testing.T) { + requireSupportedPlatform(t) + dir := filepath.Join(t.TempDir(), "profiles") + until := time.Now().Add(500 * time.Millisecond) + r, err := Start(context.Background(), dir, until.Format(time.RFC3339Nano), nil, func() any { return map[string]int{"acquired": 2} }) + if err != nil { + t.Fatal(err) + } + select { + case <-r.done: + case <-time.After(3 * time.Second): + r.Stop() + t.Fatal("did not stop at deadline") + } + files, err := filepath.Glob(filepath.Join(dir, "*.pprof")) + if err != nil || len(files) != 1 { + t.Fatalf("profiles %v, %v", files, err) + } + b, err := os.ReadFile(files[0]) + if err != nil { + t.Fatal(err) + } + z, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(z) + z.Close() + if err != nil || len(data) == 0 { + t.Fatalf("invalid profile: %v", err) + } + for _, name := range []string{files[0], files[0] + ".json"} { + info, err := os.Stat(name) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Fatalf("permissions %v", info.Mode()) + } + } + var m struct { + Reason string + StartedAt, FinishedAt time.Time + Before, After struct{ Pool map[string]int } + } + b, err = os.ReadFile(files[0] + ".json") + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + if m.Reason != "periodic" || !m.FinishedAt.After(m.StartedAt) || m.Before.Pool["acquired"] != 2 || m.After.Pool["acquired"] != 2 { + t.Fatalf("bad metadata %+v", m) + } + r.request("reconfirm") + r.Stop() + files, _ = filepath.Glob(filepath.Join(dir, "*.pprof")) + if len(files) != 1 { + t.Fatal("capture after expiry") + } +} + +func TestStopReleasesProfiler(t *testing.T) { + requireSupportedPlatform(t) + active := make(chan struct{}, 2) + old := slog.Default() + slog.SetDefault(slog.New(captureObserver{Handler: slog.NewTextHandler(io.Discard, nil), active: active})) + defer slog.SetDefault(old) + dir := filepath.Join(t.TempDir(), "profiles") + r, err := Start(context.Background(), dir, time.Now().Add(time.Hour).Format(time.RFC3339), nil, nil) + if err != nil { + t.Fatal(err) + } + select { + case <-active: + case <-time.After(3 * time.Second): + r.Stop() + t.Fatal("capture did not start") + } + r.Stop() + r.Stop() + first, _ := filepath.Glob(filepath.Join(dir, "*.pprof")) + if len(first) != 1 { + t.Fatal("active capture was not saved on shutdown") + } + b, err := os.ReadFile(first[0]) + if err != nil { + t.Fatal(err) + } + z, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(io.Discard, z); err != nil { + t.Fatal(err) + } + z.Close() + dir2 := filepath.Join(t.TempDir(), "profiles") + r2, err := Start(context.Background(), dir2, time.Now().Add(400*time.Millisecond).Format(time.RFC3339Nano), nil, nil) + if err != nil { + t.Fatal(err) + } + <-r2.done + files, _ := filepath.Glob(filepath.Join(dir2, "*.pprof")) + if len(files) != 1 { + t.Fatal("profiler was not released") + } +} + +func TestProfileWriteLimit(t *testing.T) { + var b bytes.Buffer + w := &limitedWriter{writer: &b, remaining: 4} + if _, err := w.Write([]byte("12345")); err == nil { + t.Fatal("accepted oversized profile") + } + if b.Len() > 4 { + t.Fatal("exceeded file budget") + } + if w.err == nil { + t.Fatal("write failure lost") + } +} + +func TestDirectoryBudgetSurvivesRestart(t *testing.T) { + requireSupportedPlatform(t) + dir := filepath.Join(t.TempDir(), "profiles") + if err := os.Mkdir(dir, 0700); err != nil { + t.Fatal(err) + } + f, err := os.Create(filepath.Join(dir, "old.pprof")) + if err != nil { + t.Fatal(err) + } + if err := f.Truncate(256 << 20); err != nil { + t.Fatal(err) + } + f.Close() + r, err := Start(context.Background(), dir, time.Now().Add(time.Hour).Format(time.RFC3339), nil, nil) + if err != nil { + t.Fatal(err) + } + select { + case <-r.done: + case <-time.After(2 * time.Second): + r.Stop() + t.Fatal("budget did not stop recording") + } + entries, _ := os.ReadDir(dir) + if len(entries) != 1 { + t.Fatal("wrote past directory budget") + } +} + +func TestScheduleBoundsExtraCaptures(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + requests := make(chan string, 1) + type event struct { + at time.Duration + reason string + } + var events []event + start := time.Now() + done := make(chan error, 1) + go func() { + done <- schedule(ctx, requests, func(ctx context.Context, reason string) error { + events = append(events, event{time.Since(start), reason}) + time.Sleep(30 * time.Second) + return nil + }) + }() + synctest.Wait() + time.Sleep(time.Minute) + requests <- "reconfirm" + synctest.Wait() + if len(events) != 1 { + t.Fatalf("ignored cooldown: %+v", events) + } + time.Sleep(4 * time.Minute) + requests <- "reconfirm" + synctest.Wait() + time.Sleep(56 * time.Minute) + cancel() + synctest.Wait() + if err := <-done; err != nil { + t.Fatal(err) + } + want := []event{{0, "periodic"}, {5 * time.Minute, "reconfirm"}, {30 * time.Minute, "periodic"}, {60 * time.Minute, "periodic"}} + if !reflect.DeepEqual(events, want) { + t.Fatalf("captures %+v, want %+v", events, want) + } + }) +} + +func TestScheduleDefersPeriodicCaptureInCooldown(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + requests := make(chan string, 1) + var at []time.Duration + var reasons []string + start := time.Now() + done := make(chan error, 1) + go func() { + done <- schedule(ctx, requests, func(ctx context.Context, reason string) error { + at = append(at, time.Since(start)) + reasons = append(reasons, reason) + time.Sleep(30 * time.Second) + return nil + }) + }() + synctest.Wait() + time.Sleep(28 * time.Minute) + requests <- "reconfirm" + synctest.Wait() + time.Sleep(33 * time.Minute) + cancel() + synctest.Wait() + if err := <-done; err != nil { + t.Fatal(err) + } + wantAt := []time.Duration{0, 28 * time.Minute, 33 * time.Minute, 60 * time.Minute} + wantReasons := []string{"periodic", "reconfirm", "periodic", "periodic"} + if !reflect.DeepEqual(at, wantAt) || !reflect.DeepEqual(reasons, wantReasons) { + t.Fatalf("captures %v %v, want %v %v", at, reasons, wantAt, wantReasons) + } + }) +} + +func TestTriggersAreConfigurable(t *testing.T) { + r := &Recorder{triggers: map[string]bool{"custom": true}, requests: make(chan string, 1), done: make(chan struct{})} + noop := func(context.Context) error { return nil } + _ = r.WrapTask("reconfirm", noop)(context.Background()) + select { + case got := <-r.requests: + t.Fatalf("untriggered task requested %q", got) + default: + } + _ = r.WrapTask("custom", noop)(context.Background()) + if got := <-r.requests; got != "custom" { + t.Fatalf("got %q", got) + } +} + +func TestWrappedTaskPreservesContextAndError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + want := errors.New("task failed") + r := &Recorder{triggers: map[string]bool{"reconfirm": true}, requests: make(chan string, 1), done: make(chan struct{})} + run := r.WrapTask("reconfirm", func(got context.Context) error { + if got.Err() != context.Canceled { + t.Fatal("lost cancellation") + } + if label, ok := pprof.Label(got, "task"); !ok || label != "reconfirm" { + t.Fatal("missing task label") + } + return want + }) + for i := 0; i < 10; i++ { + if err := run(ctx); err != want { + t.Fatalf("lost task error: %v", err) + } + } +} + +func requireSupportedPlatform(t *testing.T) { + t.Helper() + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("private profiling requires Unix permissions") + } +} + +type captureObserver struct { + slog.Handler + active chan<- struct{} +} + +func (h captureObserver) Handle(ctx context.Context, record slog.Record) error { + if record.Message == "CPU profiling capture started" { + h.active <- struct{}{} + } + return h.Handler.Handle(ctx, record) +} From d733975165056eccced110d27b4ffb30c7b0c177 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:02 -0400 Subject: [PATCH 090/141] fix(backup): abort exports on shutdown and sweep stale staging dirs srv.Shutdown never cancelled the request context, so a SIGTERM mid-export left the plaintext dump and pg_service.conf in TMPDIR. The handler now derives its context from the app context and the router removes staging dirs older than 20 minutes at startup. --- cmd/beacon/main.go | 2 +- internal/api/handlers/backup.go | 26 ++++++-- internal/api/handlers/backup_test.go | 60 +++++++++++++++++-- .../api/router/backup_integration_test.go | 2 +- internal/api/router/backup_test.go | 3 +- 5 files changed, 82 insertions(+), 11 deletions(-) diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 24e60847..fc98fd9b 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -337,7 +337,7 @@ func main() { CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, AdminRoutes: map[string]http.Handler{ "/accounts": handlers.AccountsRouter(store), - "/backup": handlers.BackupRouter(backupOpts), + "/backup": handlers.BackupRouter(backupOpts, ctx), }, }) diff --git a/internal/api/handlers/backup.go b/internal/api/handlers/backup.go index 67f9641d..e9a85fd9 100644 --- a/internal/api/handlers/backup.go +++ b/internal/api/handlers/backup.go @@ -22,12 +22,25 @@ import ( // BackupRouter returns an operator-only subrouter with one bounded export or // transfer at a time. Its caller must apply the admin authentication middleware. // The options are startup-owned; the request selects no targets or paths. -func BackupRouter(opts backup.Options) http.Handler { +// shutdown aborts an in-flight export so its staging files are removed. +func BackupRouter(opts backup.Options, shutdown context.Context) http.Handler { + sweepStaleStaging() r := chi.NewRouter() - r.Get("/", backupDownload(opts, backup.Export)) + r.Get("/", backupDownload(opts, backup.Export, shutdown)) return r } +// sweepStaleStaging removes staging dirs an earlier process left behind (SIGKILL, crash). +func sweepStaleStaging() { + matches, _ := filepath.Glob(filepath.Join(os.TempDir(), "beacon-download-*")) + cutoff := time.Now().Add(-2 * backup.DefaultTimeout) + for _, dir := range matches { + if info, err := os.Stat(dir); err == nil && info.IsDir() && info.ModTime().Before(cutoff) { + _ = os.RemoveAll(dir) + } + } +} + // backupDownload godoc // // @Summary Download a private database and saved-config backup @@ -45,7 +58,7 @@ func BackupRouter(opts backup.Options) http.Handler { // @Failure 504 {object} map[string]APIError // @Failure 507 {object} map[string]APIError // @Router /admin/backup [get] -func backupDownload(opts backup.Options, export func(context.Context, backup.Options) error) http.HandlerFunc { +func backupDownload(opts backup.Options, export func(context.Context, backup.Options) error, shutdown context.Context) http.HandlerFunc { var busy atomic.Bool return func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-store") @@ -70,11 +83,16 @@ func backupDownload(opts backup.Options, export func(context.Context, backup.Opt defer os.RemoveAll(dir) requestOpts := opts requestOpts.OutputPath = filepath.Join(dir, "backup.tar.gz") - if err := export(r.Context(), requestOpts); err != nil { + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + defer context.AfterFunc(shutdown, cancel)() + if err := export(ctx, requestOpts); err != nil { if errors.Is(err, backup.ErrTooLarge) { respondError(w, http.StatusInsufficientStorage, "backup exceeds the configured export size limit") } else if errors.Is(err, context.DeadlineExceeded) { respondError(w, 504, "backup export timed out") + } else if shutdown.Err() != nil { + respondError(w, http.StatusServiceUnavailable, "server is shutting down") } else if r.Context().Err() == nil { // Export discards pg_dump stderr and never returns connection settings. slog.Error("backup export failed", "component", "backup", "error", err) diff --git a/internal/api/handlers/backup_test.go b/internal/api/handlers/backup_test.go index 64c77247..0ad1e9cb 100644 --- a/internal/api/handlers/backup_test.go +++ b/internal/api/handlers/backup_test.go @@ -27,7 +27,7 @@ func TestBackupDownload(t *testing.T) { output = got.OutputPath return os.WriteFile(output, []byte("completed fixture"), 0600) } - handler := backupDownload(opts, export) + handler := backupDownload(opts, export, context.Background()) for _, query := range []string{"?database=other", "?path=/private", "?command=anything"} { w := httptest.NewRecorder() handler(w, httptest.NewRequest("GET", "/backup"+query, nil)) @@ -53,7 +53,7 @@ func TestBackupDownload(t *testing.T) { handler = backupDownload(opts, func(_ context.Context, got backup.Options) error { output = got.OutputPath return failure - }) + }, context.Background()) w = httptest.NewRecorder() handler(w, httptest.NewRequest("GET", "/backup", nil)) want := 500 @@ -71,7 +71,7 @@ func TestBackupDownload(t *testing.T) { } } w = httptest.NewRecorder() - BackupRouter(backup.Options{}).ServeHTTP(w, httptest.NewRequest("GET", "/", nil)) + BackupRouter(backup.Options{}, context.Background()).ServeHTTP(w, httptest.NewRequest("GET", "/", nil)) if w.Code != 503 { t.Fatal("unconfigured download enabled") } @@ -85,7 +85,7 @@ func TestBackupDownloadConcurrencyAndCancellation(t *testing.T) { close(started) <-ctx.Done() return ctx.Err() - }) + }, context.Background()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { @@ -104,3 +104,55 @@ func TestBackupDownloadConcurrencyAndCancellation(t *testing.T) { t.Fatal("cancelled staging retained") } } + +func TestBackupDownloadAbortsOnShutdown(t *testing.T) { + shutdown, stop := context.WithCancel(context.Background()) + stagedCh := make(chan string, 1) + handler := backupDownload(backup.Options{ConnectionService: "fixture"}, func(ctx context.Context, opts backup.Options) error { + stagedCh <- filepath.Dir(opts.OutputPath) + <-ctx.Done() + return ctx.Err() + }, shutdown) + rec := httptest.NewRecorder() + done := make(chan struct{}) + go func() { + handler.ServeHTTP(rec, httptest.NewRequest("GET", "/", nil)) + close(done) + }() + staged := <-stagedCh + stop() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("export did not stop on shutdown") + } + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status %d, want 503", rec.Code) + } + if _, err := os.Stat(staged); !os.IsNotExist(err) { + t.Fatalf("staging dir survived shutdown: %v", err) + } +} + +func TestBackupRouterSweepsStaleStaging(t *testing.T) { + t.Setenv("TMPDIR", t.TempDir()) + stale, err := os.MkdirTemp("", "beacon-download-") + if err != nil { + t.Fatal(err) + } + old := time.Now().Add(-3 * backup.DefaultTimeout) + if err := os.Chtimes(stale, old, old); err != nil { + t.Fatal(err) + } + fresh, err := os.MkdirTemp("", "beacon-download-") + if err != nil { + t.Fatal(err) + } + BackupRouter(backup.Options{}, context.Background()) + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatal("stale staging dir not swept") + } + if _, err := os.Stat(fresh); err != nil { + t.Fatal("fresh staging dir swept") + } +} diff --git a/internal/api/router/backup_integration_test.go b/internal/api/router/backup_integration_test.go index 33965a4b..18f61830 100644 --- a/internal/api/router/backup_integration_test.go +++ b/internal/api/router/backup_integration_test.go @@ -91,7 +91,7 @@ func TestBackupDownloadPostgres(t *testing.T) { Auth: config.AuthConfig{APIKey: "fixture-key"}, AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ ConnectionService: service, ConfigPath: configPath, MaxBytes: 16 << 20, Timeout: 30 * time.Second, Version: "http-fixture", - })}, + }, context.Background())}, })) defer server.Close() request, _ := http.NewRequestWithContext(ctx, "GET", server.URL+"/api/v1/admin/backup", nil) diff --git a/internal/api/router/backup_test.go b/internal/api/router/backup_test.go index 76bdf6e7..2c026529 100644 --- a/internal/api/router/backup_test.go +++ b/internal/api/router/backup_test.go @@ -4,6 +4,7 @@ package router import ( + "context" "net/http" "net/http/httptest" "testing" @@ -15,7 +16,7 @@ import ( func TestBackupAuthenticationBeforeExport(t *testing.T) { for _, key := range []string{"", "fixture-key"} { - handler := New(nil, nil, nil, Options{Auth: config.AuthConfig{APIKey: key}, AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ConnectionService: "fixture"})}}) + handler := New(nil, nil, nil, Options{Auth: config.AuthConfig{APIKey: key}, AdminRoutes: map[string]http.Handler{"/backup": handlers.BackupRouter(backup.Options{ConnectionService: "fixture"}, context.Background())}}) for _, token := range []string{"", "wrong", "fixture-key"} { r := httptest.NewRequest("GET", "/api/v1/admin/backup?database=other", nil) if token != "" { From 6852074180269b9bb520ccbbc1553b464c7ab206 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 091/141] fix(ingest): record hearing paths right after the insert MarkSent ran after the side effects, so a broker copy on another worker could win it and be streamed as a repeat of the first hearing. It now runs before any further DB work, unconditionally so the set is warm before anyone opts in, and never for TRACE, whose path bytes are per-hop SNR. --- internal/api/packets.go | 2 +- internal/hub/hub.go | 15 ++++--- internal/hub/sent.go | 2 +- internal/ingest/ingest_test.go | 4 ++ internal/ingest/packet.go | 12 +++--- internal/ingest/repeats_test.go | 69 +++++++++++++++++++++++++++++++++ 6 files changed, 88 insertions(+), 16 deletions(-) diff --git a/internal/api/packets.go b/internal/api/packets.go index 2b2e4ebf..e5f47fe8 100644 --- a/internal/api/packets.go +++ b/internal/api/packets.go @@ -40,7 +40,7 @@ type PacketSummary struct { LastHeardAt int64 `json:"lastHeardAt"` // epoch ms ObservationCount int32 `json:"observationCount"` LatestObserver *PacketLatestObserver `json:"latestObserver,omitempty"` - Summary *string `json:"summary,omitempty"` // advert name from this packet; omitted when unavailable or unsupported + Summary *string `json:"summary,omitempty"` // advert name, or the ACK/TRACE/PING summary; omitted when unavailable } // PacketPathLength is the decoded path_length byte from a packet observation. diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 09c22077..28061264 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -38,11 +38,10 @@ const ( // Event is a single fan-out unit. Payload is pre-serialised JSON so the // broadcast loop never touches encoding — it's done once by the ingest path. // -// PayloadResolved is an optional second serialization carrying additional -// fields for clients that opted into them via configure (currently just -// resolvedPath on packetObservation events). Left nil for event types that -// don't have an opt-in variant; the hub falls back to Payload in that case. -// The WithKey variants also carry observerPublicKey. +// PayloadResolved/PayloadWithKey/PayloadResolvedWithKey are optional variants +// for clients that opted in via configure (resolvedPath, observerPublicKey); +// nil falls back to Payload. Repeat marks later hearings that only +// includeRepeats clients receive. type Event struct { Type EventType Payload json.RawMessage @@ -239,13 +238,13 @@ func (h *Hub) ObserverKeyWanted() bool { return h.observerKeyClients.Load() > 0 } -// RepeatsWanted lets ingest skip repeat work when nobody wants it. +// RepeatsWanted lets ingest skip broadcasting repeats when nobody wants them. func (h *Hub) RepeatsWanted() bool { return h.repeatClients.Load() > 0 } -// MarkSent records a hearing's path and reports whether it was not already sent recently, -// so broker copies and same-path duplicates go out once. +// MarkSent records every hearing's path and reports whether it is new within the TTL, +// so broker copies and same-path duplicates are never streamed as repeats. func (h *Hub) MarkSent(packetHash, observerID, path []byte) bool { return h.sent.mark(packetHash, observerID, path, time.Now()) } diff --git a/internal/hub/sent.go b/internal/hub/sent.go index f1e04112..45ac8015 100644 --- a/internal/hub/sent.go +++ b/internal/hub/sent.go @@ -14,7 +14,7 @@ const ( sentPathsMax = 200_000 ) -// sentPaths remembers recently sent (packet, observer, path) hearings. It is shared by all +// sentPaths remembers recently heard (packet, observer, path) hearings. It is shared by all // broker workers; keys are hashed so memory stays small at the size bound. type sentPaths struct { mu sync.Mutex diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 248d3a4b..aa8f892a 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -184,6 +184,7 @@ type stubDB struct { observationInserted bool insertChannelMessageResult bool // configurable return for InsertChannelMessage; default false undecryptedPackets []UndecryptedPacket + dbHook func() // runs in UpsertChannelIATA, the first DB call after InsertObservation } type setCapabilityCall struct { @@ -279,6 +280,9 @@ func (s *stubDB) ListUndecryptedGroupTextPackets(_ context.Context) ([]Undecrypt } func (s *stubDB) UpsertChannelIATA(_ context.Context, _ []byte, _ string, _ time.Time) error { + if s.dbHook != nil { + s.dbHook() + } s.upsertChannelIATACalls++ return nil } diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index fda54975..ad5336eb 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -81,7 +81,7 @@ type packetObservationEvent struct { IsFirstObservation bool `json:"isFirstObservation"` ObservationCount int64 `json:"observationCount"` Scope *string `json:"scope,omitempty"` - Summary *string `json:"summary,omitempty"` // same advert name as REST list/backfill rows + Summary *string `json:"summary,omitempty"` // advert name, or "ACK …"/"TRACE …"/"PING …" for those types // Only set on later hearings sent to includeRepeats clients; those carry observationCount 0. IsRepeat bool `json:"isRepeat,omitempty"` } `json:"packet"` @@ -780,6 +780,11 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ w.log.Error(fmt.Sprintf("db: insert observation failed from %s/%s", iata, pubkeyHex), "error", err) return } + // Record the path before any more DB work so a broker copy racing through it can't be + // streamed as a repeat of this hearing; TRACE reuses Path for per-hop SNR. + pathNew := packet.PayloadType() != meshcore.PayloadTypeTrace && w.hub.MarkSent(packetHash[:], id[:], packet.Path) + // A duplicate is streamed, never stored, to includeRepeats clients when its path is new. + repeat := !inserted && pathNew && w.hub.RepeatsWanted() if scopeID != nil && inserted { if err := w.db.UpsertObserverScope(ctx, id, *scopeID); err != nil { @@ -841,14 +846,9 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } w.runCapabilityDetection(ctx, packet.PayloadType(), packet.PathHashSize(), resolvedIDs) - // A duplicate is streamed, never stored, to includeRepeats clients when its path is new. - repeat := !inserted && w.hub.RepeatsWanted() && w.hub.MarkSent(packetHash[:], id[:], packet.Path) if inserted || repeat { if inserted { w.handlePayloadTypeSideEffects(ctx, packet, iata, packetHash[:], radio, scopeID, matchedScope, pubkeyBytes, float32(parseNumber(envelope.SNR))) - if w.hub.RepeatsWanted() { - w.hub.MarkSent(packetHash[:], id[:], packet.Path) // so broker copies of it aren't repeats - } } // Resolve after advert updates; suppressed copies need no endpoint lookup. var resolvedSource, resolvedDestination *api.ResolvedHop diff --git a/internal/ingest/repeats_test.go b/internal/ingest/repeats_test.go index 5766c099..0f9e73d9 100644 --- a/internal/ingest/repeats_test.go +++ b/internal/ingest/repeats_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/google/uuid" ) type countQueryDB struct { @@ -130,3 +131,71 @@ func TestHandlePacket_RepeatsOffByDefault(t *testing.T) { t.Fatalf("duplicate ran a count query (%d total)", r.db.countQueries) } } + +// A broker copy racing through the inserted hearing's DB work must find its path already +// recorded, otherwise it would be streamed as a repeat of the first hearing. +func TestHandlePacket_PathMarkedRightAfterInsert(t *testing.T) { + r := newRepeatHarness(t, true) + packet := buildGrpTxtPacket(t, 0x1a, make([]byte, 16)) + packet.Path = []byte{0x11} + packet.PathLength = 1 + hash := packet.PacketHash() + var fresh bool + r.db.dbHook = func() { + // stubDB.UpsertObserver returns uuid.Nil, so that is the observer id ingest marks with + fresh = r.w.hub.MarkSent(hash[:], uuid.Nil[:], packet.Path) + } + r.db.observationInserted = true + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(r.t, packet)) + if fresh { + t.Fatal("path was not marked right after the insert") + } +} + +func TestHandlePacket_PathsRecordedWithoutSubscribers(t *testing.T) { + r := newRepeatHarness(t, false) + if got := r.hear(true, 0x11); len(got) != 1 { + t.Fatalf("first hearing: %s", got) + } + r.w.hub.Configure(r.client, hub.ClientOptions{IncludeRepeats: true}) + for !r.w.hub.RepeatsWanted() { + if r.ctx.Err() != nil { + t.Fatal("repeats opt-in never registered") + } + time.Sleep(time.Millisecond) + } + if got := r.hear(false, 0x11); len(got) != 0 { + t.Fatalf("same path streamed as repeat after late opt-in: %s", got) + } +} + +func TestHandlePacket_TraceNeverRepeats(t *testing.T) { + r := newRepeatHarness(t, true) + packet := buildTracePacket(t) + packet.Path = []byte{0x05, 0x06} + packet.PathLength = 2 + r.db.observationInserted = true + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(r.t, packet)) + r.db.observationInserted = false + packet.Path = append([]byte{}, packet.Path...) + packet.Path[0] ^= 0xff // a different per-hop SNR byte, same route + r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(r.t, packet)) + r.w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + repeats := 0 + for { + select { + case evt := <-r.client.Send: + if evt.Type == hub.EventObserverStatus { + if repeats != 0 { + t.Fatalf("TRACE copy streamed as %d repeat(s)", repeats) + } + return + } + if strings.Contains(string(evt.Payload), "isRepeat") { + repeats++ + } + case <-r.ctx.Done(): + t.Fatal("marker event not delivered") + } + } +} From c952afdd29eaced96da9e621ba6fca8d997ed205 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 092/141] fix(ingest): let paho auto-ack MQTT deliveries Sessions are clean and client IDs random, so deferring acks never bought redelivery; it only capped delivery at the broker's inflight window and pushed backlog into the broker queue, where drops are silent. --- internal/ingest/ingest.go | 8 ++------ internal/ingest/queue_test.go | 15 ++------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 289a0def..dcbe2bb8 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -249,7 +249,6 @@ func (w *Worker) Start(ctx context.Context) { } queue := newMessageQueue(8, 2048, 32<<20, func(ctx context.Context, msg mqtt.Message) { w.handleMessageContext(ctx, msg) - msg.Ack() }) reportDrops := func() { if n := queue.takeDropped(); n > 0 { @@ -268,13 +267,13 @@ func (w *Worker) Start(ctx context.Context) { // Isolate workers across deployments; Paho reuses this ID on reconnect. // Keep it alphanumeric and within MQTT 3.1's 23-character client ID limit. + // Auto-ack: sessions are clean, so deferring acks only throttled the broker's inflight window. opts := mqtt.NewClientOptions(). AddBroker(w.cfg.URL). SetClientID(rand.Text()[:23]). SetUsername(w.cfg.Username). SetPassword(w.cfg.Password). SetAutoReconnect(true). - SetAutoAckDisabled(true). SetMaxReconnectInterval(30 * time.Second). SetKeepAlive(30 * time.Second). SetPingTimeout(10 * time.Second). @@ -335,10 +334,7 @@ func (w *Worker) subscribe(client mqtt.Client, queue *messageQueue) { // meshcore/{IATA}/{pubkey}/status // We do NOT subscribe to /internal (Role 2 access). tok := client.Subscribe("meshcore/#", 1, func(_ mqtt.Client, msg mqtt.Message) { - if !queue.enqueue(msg) { - // Counted drops must release the broker's inflight slot. - msg.Ack() - } + queue.enqueue(msg) }) if tok.Wait() && tok.Error() != nil { w.log.Error("subscribe error", "error", tok.Error()) diff --git a/internal/ingest/queue_test.go b/internal/ingest/queue_test.go index 130a17a2..6b43fe49 100644 --- a/internal/ingest/queue_test.go +++ b/internal/ingest/queue_test.go @@ -141,24 +141,13 @@ type completedSubscribe struct{ mqtt.Token } func (completedSubscribe) Wait() bool { return true } func (completedSubscribe) Error() error { return nil } -type ackMessage struct { - mqtt.Message - acked bool -} - -func (m *ackMessage) Ack() { m.acked = true } - -func TestSubscribeReleasesRejectedMessage(t *testing.T) { +func TestSubscribeCountsRejectedMessage(t *testing.T) { w, _ := newTestWorker() q := newMessageQueue(1, 1, 1, func(context.Context, mqtt.Message) { t.Error("rejected message processed") }) defer q.close(context.Background()) client := &subscribeCapture{} w.subscribe(client, q) - m := &ackMessage{Message: queueMessage("aa", "packets", 0)} - client.callback(client, m) - if !m.acked { - t.Fatal("rejected message still occupies the broker inflight window") - } + client.callback(client, queueMessage("aa", "packets", 0)) if q.takeDropped() != 1 { t.Fatal("rejected message was not counted") } From 44b9a0f6d8c4f7062e39d2b5b88adcfe534a6ad2 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 093/141] ci: run packet summary and reconfirm tests, read analytics on 039 views --- .github/workflows/ci.yml | 2 +- db/analytics_retention_integration_test.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 328b4528..9574594d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/db/analytics_retention_integration_test.go b/db/analytics_retention_integration_test.go index 10e33339..2bfbdfe6 100644 --- a/db/analytics_retention_integration_test.go +++ b/db/analytics_retention_integration_test.go @@ -167,6 +167,24 @@ func TestAnalyticsRetentionPostgres(t *testing.T) { t.Fatalf("missing unfiltered time index %s: %s %v", name, definition, err) } } + // The signal/path stores must read the 039 view definitions, not just 035/036. + store := &Store{q: sqlc.New(tx)} + hour := time.Now().UTC().Truncate(time.Hour) + signal, err := store.GetSignalStats(ctx, hour.Add(-6*24*time.Hour), hour, nil) + if err != nil { + t.Fatal(err) + } + // 12 observations; only the n=1 rows carry a signal (snr 0, rssi -100). + if signal.Receptions != 12 || signal.SNR.Samples != 4 || signal.RSSI.Samples != 4 { + t.Fatalf("signal stats on 039 views: %+v", signal) + } + paths, err := store.GetPathStats(ctx, hour.Add(-6*24*time.Hour), hour, nil) + if err != nil { + t.Fatal(err) + } + if paths.Receptions != 12 { + t.Fatalf("path stats on 039 views: %+v", paths) + } before := analyticsSnapshot(t, ctx, tx) q := sqlc.New(tx) cutoff := time.Now().Add(-72 * time.Hour) From e7061f5e10e787121c914172c2a0b5f4497b2813 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 094/141] fix(api): log the cause of account 500s --- internal/api/handlers/accounts.go | 2 ++ internal/api/handlers/accounts_test.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/internal/api/handlers/accounts.go b/internal/api/handlers/accounts.go index 92c1fc20..3f4fb287 100644 --- a/internal/api/handlers/accounts.go +++ b/internal/api/handlers/accounts.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "io" + "log/slog" "mime" "net/http" @@ -168,6 +169,7 @@ func accountError(w http.ResponseWriter, err error) { case errors.Is(err, api.ErrAccountInactive): respondError(w, 409, api.ErrAccountInactive.Error()) default: + slog.Error("account request failed", "component", "api", "error", err) respondError(w, 500, "internal server error") } } diff --git a/internal/api/handlers/accounts_test.go b/internal/api/handlers/accounts_test.go index d6b96833..50aaa090 100644 --- a/internal/api/handlers/accounts_test.go +++ b/internal/api/handlers/accounts_test.go @@ -4,10 +4,12 @@ package handlers import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "log/slog" "net/http" "net/http/httptest" "strings" @@ -145,3 +147,18 @@ func TestAccountListAndAuthorization(t *testing.T) { } } } + +func TestAccountErrorLogsCause(t *testing.T) { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + rec := httptest.NewRecorder() + accountError(rec, errors.New("private-error")) + if rec.Code != 500 || strings.Contains(rec.Body.String(), "private-error") { + t.Fatalf("response leaked or wrong status: %d %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(buf.String(), "private-error") { + t.Fatalf("cause not logged: %s", buf.String()) + } +} From e1b78b6458ab9c696dc61368f20b54d842ba0934 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 095/141] fix(profiling): offset periodic captures and harden the recorder Periodic ticks now start five minutes off the task tickers so the hourly reconfirm trigger stops colliding with the cooldown, any capture clears a deferred periodic, subdirectories no longer stop the recorder, and the metadata sidecar is published before the profile. --- PROFILING.md | 9 ++-- internal/profiling/profiling.go | 41 ++++++++++++++---- internal/profiling/profiling_test.go | 65 ++++++++++++++++++++++++++-- 3 files changed, 99 insertions(+), 16 deletions(-) diff --git a/PROFILING.md b/PROFILING.md index 5e3667ee..a9989b4a 100644 --- a/PROFILING.md +++ b/PROFILING.md @@ -31,17 +31,20 @@ can enable a longer session. Profiling adds overhead while a capture is active. ## Capture behavior -- One 30-second sample immediately, then every 30 minutes. +- One 30-second sample immediately, then every 30 minutes, offset five minutes past + the half hour so a maintenance trigger due on the hour is not lost to the cooldown. - Route reconfirmation requests an additional sample when the maintenance task starts. This includes the retention step before route validation. A five-minute cooldown between capture starts prevents overlap and repeated triggers from increasing load; - a periodic sample due during the cooldown runs when it ends. + a periodic sample due during the cooldown runs when it ends. A triggered sample + during the cooldown is skipped, and any sample cancels a periodic one waiting on + the cooldown. - Background task stacks carry a `task` label while profiling is enabled. - Shutdown or expiry stops the active sample and saves the shorter profile. - Each profile is limited to 8 MiB. The dedicated directory is limited to 256 MiB and 512 files, including metadata and files left by interrupted runs. The recorder reserves space for a full capture before starting and stops when a limit is reached. - Files are never automatically deleted. Existing unrelated files consume the budget. + Files are never automatically deleted. Existing unrelated regular files consume the budget; subdirectories are ignored. - Profiles and metadata are written with mode `0600`. A `.partial` file indicates an interrupted capture and is not a completed profile. diff --git a/internal/profiling/profiling.go b/internal/profiling/profiling.go index ca836d9d..0d61a933 100644 --- a/internal/profiling/profiling.go +++ b/internal/profiling/profiling.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "io/fs" "log/slog" "os" "path/filepath" @@ -155,8 +156,12 @@ func (r *Recorder) run(ctx context.Context) { } func schedule(ctx context.Context, requests <-chan string, capture func(context.Context, string) error) error { - ticker := time.NewTicker(captureInterval) - defer ticker.Stop() + // Offset periodic ticks from the task tickers started alongside the recorder, so a + // task's trigger isn't swallowed by a periodic capture that began seconds earlier. + first := time.NewTimer(captureInterval + captureCooldown) + defer first.Stop() + var ticker *time.Ticker + tick := first.C reason := "periodic" var last time.Time var deferred <-chan time.Time @@ -164,8 +169,9 @@ func schedule(ctx context.Context, requests <-chan string, capture func(context. if ctx.Err() != nil { return nil } - if wait := captureCooldown - time.Since(last); last.IsZero() || wait <= 0 { + if wait := captureCooldown - time.Since(last); wait <= 0 { last = time.Now() + deferred = nil // this sample covers any periodic one waiting on the cooldown if err := capture(ctx, reason); err != nil { return err } @@ -175,8 +181,15 @@ func schedule(ctx context.Context, requests <-chan string, capture func(context. } select { case <-ctx.Done(): + if ticker != nil { + ticker.Stop() + } return nil - case <-ticker.C: + case <-tick: + if ticker == nil { + ticker = time.NewTicker(captureInterval) + tick = ticker.C + } reason = "periodic" case <-deferred: deferred = nil @@ -211,11 +224,14 @@ func (r *Recorder) checkBudget() error { var total int64 for _, entry := range entries { info, err := entry.Info() + if errors.Is(err, fs.ErrNotExist) { + continue // removed between ReadDir and Info + } if err != nil { return err } if !info.Mode().IsRegular() { - return errors.New("profiling directory contains a non-regular file") + continue // subdirectories and specials don't hold profiles } total += info.Size() } @@ -260,9 +276,6 @@ func (r *Recorder) capture(ctx context.Context, reason string) error { if closeErr != nil { return closeErr } - if err := r.root.Rename(partial, name); err != nil { - return err - } data, err := json.MarshalIndent(m, "", " ") if err != nil { return err @@ -270,10 +283,12 @@ func (r *Recorder) capture(ctx context.Context, reason string) error { if len(data) > maxMetadataBytes { return errors.New("profiling metadata budget exhausted") } - meta, err := r.root.OpenFile(name+".json", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + metaPartial := name + ".json.partial" + meta, err := r.root.OpenFile(metaPartial, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } + defer r.root.Remove(metaPartial) _, err = meta.Write(data) closeErr = meta.Close() if err != nil { @@ -282,6 +297,14 @@ func (r *Recorder) capture(ctx context.Context, reason string) error { if closeErr != nil { return closeErr } + // Sidecar first: a .pprof without .json would look complete but be unexplained. + if err := r.root.Rename(metaPartial, name+".json"); err != nil { + return err + } + if err := r.root.Rename(partial, name); err != nil { + r.root.Remove(name + ".json") + return err + } slog.Info("CPU profile saved", "component", "profiling", "file", name, "reason", reason, "duration", m.FinishedAt.Sub(started)) return nil } diff --git a/internal/profiling/profiling_test.go b/internal/profiling/profiling_test.go index 4a307db6..51b490de 100644 --- a/internal/profiling/profiling_test.go +++ b/internal/profiling/profiling_test.go @@ -251,13 +251,13 @@ func TestScheduleBoundsExtraCaptures(t *testing.T) { time.Sleep(4 * time.Minute) requests <- "reconfirm" synctest.Wait() - time.Sleep(56 * time.Minute) + time.Sleep(61 * time.Minute) cancel() synctest.Wait() if err := <-done; err != nil { t.Fatal(err) } - want := []event{{0, "periodic"}, {5 * time.Minute, "reconfirm"}, {30 * time.Minute, "periodic"}, {60 * time.Minute, "periodic"}} + want := []event{{0, "periodic"}, {5 * time.Minute, "reconfirm"}, {35 * time.Minute, "periodic"}, {65 * time.Minute, "periodic"}} if !reflect.DeepEqual(events, want) { t.Fatalf("captures %+v, want %+v", events, want) } @@ -282,7 +282,7 @@ func TestScheduleDefersPeriodicCaptureInCooldown(t *testing.T) { }) }() synctest.Wait() - time.Sleep(28 * time.Minute) + time.Sleep(33 * time.Minute) requests <- "reconfirm" synctest.Wait() time.Sleep(33 * time.Minute) @@ -291,7 +291,7 @@ func TestScheduleDefersPeriodicCaptureInCooldown(t *testing.T) { if err := <-done; err != nil { t.Fatal(err) } - wantAt := []time.Duration{0, 28 * time.Minute, 33 * time.Minute, 60 * time.Minute} + wantAt := []time.Duration{0, 33 * time.Minute, 38 * time.Minute, 65 * time.Minute} wantReasons := []string{"periodic", "reconfirm", "periodic", "periodic"} if !reflect.DeepEqual(at, wantAt) || !reflect.DeepEqual(reasons, wantReasons) { t.Fatalf("captures %v %v, want %v %v", at, reasons, wantAt, wantReasons) @@ -299,6 +299,63 @@ func TestScheduleDefersPeriodicCaptureInCooldown(t *testing.T) { }) } +func TestScheduleOneCaptureCoversDeferredAndTrigger(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + requests := make(chan string, 1) + var at []time.Duration + start := time.Now() + done := make(chan error, 1) + go func() { + done <- schedule(ctx, requests, func(ctx context.Context, reason string) error { + at = append(at, time.Since(start)) + time.Sleep(30 * time.Second) + return nil + }) + }() + synctest.Wait() + time.Sleep(33 * time.Minute) + requests <- "reconfirm" // 33m: captured; the 35m tick is deferred to 38m + synctest.Wait() + time.Sleep(5 * time.Minute) // 38m: the deferred periodic and a fresh trigger are both due + requests <- "reconfirm" + synctest.Wait() + time.Sleep(28 * time.Minute) // past the 65m tick so cancel can't race it + cancel() + synctest.Wait() + if err := <-done; err != nil { + t.Fatal(err) + } + want := []time.Duration{0, 33 * time.Minute, 38 * time.Minute, 65 * time.Minute} + if !reflect.DeepEqual(at, want) { + t.Fatalf("captures %v, want %v", at, want) + } + }) +} + +func TestBudgetIgnoresNonRegularEntries(t *testing.T) { + requireSupportedPlatform(t) + dir := filepath.Join(t.TempDir(), "profiles") + if err := os.MkdirAll(filepath.Join(dir, "lost+found"), 0700); err != nil { + t.Fatal(err) + } + r, err := Start(context.Background(), dir, time.Now().Add(500*time.Millisecond).Format(time.RFC3339Nano), nil, nil) + if err != nil { + t.Fatal(err) + } + select { + case <-r.done: + case <-time.After(3 * time.Second): + r.Stop() + t.Fatal("did not stop at deadline") + } + profiles, _ := filepath.Glob(filepath.Join(dir, "*.pprof")) + if len(profiles) != 1 { + t.Fatalf("subdirectory stopped the recorder: %v", profiles) + } +} + func TestTriggersAreConfigurable(t *testing.T) { r := &Recorder{triggers: map[string]bool{"custom": true}, requests: make(chan string, 1), done: make(chan struct{})} noop := func(context.Context) error { return nil } From ab27334f15ebbec7337f0cc1ad733b1602eddb7f Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 096/141] fix(backup): report trailing archive data as corruption --- internal/backup/verify.go | 7 +++++++ internal/backup/verify_test.go | 13 +++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/internal/backup/verify.go b/internal/backup/verify.go index f2bf52ae..68847452 100644 --- a/internal/backup/verify.go +++ b/internal/backup/verify.go @@ -53,6 +53,10 @@ func Verify(ctx context.Context, input io.Reader, maxBytes int64) (*Manifest, er // Bound Next itself: it otherwise hides and allocates extension records. decoded.limit = before + padding + 3*512 h, err := tr.Next() + if errors.Is(err, ErrTooLarge) { + // The 3-block header budget is a structure check, not the size limit. + return nil, verificationError(ctx, nil, "tar header") + } if err == io.EOF { // tar.Reader also accepts EOF without both closing zero blocks. if decoded.n-before != padding+2*512 { @@ -106,6 +110,9 @@ func Verify(ctx context.Context, input io.Reader, maxBytes int64) (*Manifest, er decoded.limit = decoded.n var tail [1]byte if _, err := decoded.Read(tail[:]); err != io.EOF { + if errors.Is(err, ErrTooLarge) { + err = nil // bytes after the tar terminator are corruption, not oversize + } return nil, verificationError(ctx, err, "gzip completion or trailing tar data") } if _, err := buffered.ReadByte(); err != io.EOF { diff --git a/internal/backup/verify_test.go b/internal/backup/verify_test.go index fb0e9f1e..06eb1239 100644 --- a/internal/backup/verify_test.go +++ b/internal/backup/verify_test.go @@ -170,9 +170,14 @@ func TestVerifyRejectsInvalidMembersAndManifest(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { data := verifyGzip(t, verifyTar(t, tc.change(verifyFixture(t)))) - if _, err := Verify(context.Background(), bytes.NewReader(data), int64(len(testSQL))); err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") { + _, err := Verify(context.Background(), bytes.NewReader(data), int64(len(testSQL))) + if err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") { t.Fatalf("expected a sanitized rejection, got %v", err) } + // PAX header size errors should be corruption, not size limit. + if tc.name == "hidden oversized metadata" && errors.Is(err, ErrTooLarge) { + t.Fatalf("oversized header reported as size limit: %v", err) + } }) } } @@ -190,9 +195,13 @@ func TestVerifyFraming(t *testing.T) { } for name, data := range cases { t.Run(name, func(t *testing.T) { - if _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes); err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") { + _, err := Verify(context.Background(), bytes.NewReader(data), DefaultMaxBytes) + if err == nil || strings.Contains(err.Error(), "PRIVATE_CANARY") { t.Fatalf("expected sanitized framing error, got %v", err) } + if errors.Is(err, ErrTooLarge) { + t.Fatalf("framing corruption reported as size limit: %v", err) + } }) } } From 1c2841a3c94cd65978d1290b41100723caebb1ee Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 097/141] perf(db): compute route ambiguity once per reconfirm run The four-way GROUP BY over node_short_ids ran once per 1000-route batch, up to 750 times a run, and could eat the whole 5s batch budget. The task now fetches the set once and passes it to each batch. --- db/queries/queries.sql | 24 +++++++---- db/reconfirm_integration_test.go | 34 +++++++++------ db/routes.go | 26 ++++++++++- db/sqlc/mock/querier.go | 15 +++++++ db/sqlc/querier.go | 5 ++- db/sqlc/queries.sql.go | 62 ++++++++++++++++++++++----- internal/background/reconfirm_test.go | 13 +++++- internal/background/tasks.go | 9 +++- 8 files changed, 152 insertions(+), 36 deletions(-) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 328960e5..77f1c517 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1482,10 +1482,21 @@ REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets; -- name: RefreshObserverActivity :exec REFRESH MATERIALIZED VIEW CONCURRENTLY mv_observer_activity_hourly; +-- name: AmbiguousPrefixes :many +-- Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. +SELECT iata::text AS iata, 1::int AS len, prefix_1 AS prefix FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 2::int, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 3::int, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 4::int, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1; + -- name: ReconfirmRoutes :one -- Checks one batch of least-recently-reconfirmed routes: deletes those with a departed -- hop node or a hop prefix now matching >1 node in that IATA (length-aware: --- 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. +-- 1/2/3/4-byte hop prefixes check prefix_1/2/3/4; ambiguity set supplied by AmbiguousPrefixes), +-- and stamps the survivors. WITH batch AS MATERIALIZED ( SELECT r.iata, r.path_key, r.node_ids, r.hash_prefix FROM known_routes r @@ -1494,14 +1505,9 @@ WITH batch AS MATERIALIZED ( LIMIT @batch_size FOR UPDATE OF r SKIP LOCKED ), -amb AS MATERIALIZED ( - SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 - UNION ALL - SELECT iata, 2, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1 - UNION ALL - SELECT iata, 3, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1 - UNION ALL - SELECT iata, 4, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1 +amb AS ( + SELECT a.iata::char(3) AS iata, a.len, a.p + FROM ROWS FROM (unnest(@amb_iata::text[]), unnest(@amb_len::int[]), unnest(@amb_prefix::bytea[])) AS a(iata, len, p) ), dead AS ( SELECT b.iata, b.path_key diff --git a/db/reconfirm_integration_test.go b/db/reconfirm_integration_test.go index 7aeed34e..0b52206b 100644 --- a/db/reconfirm_integration_test.go +++ b/db/reconfirm_integration_test.go @@ -17,10 +17,19 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) +func ambiguity(t *testing.T, ctx context.Context, store *Store) AmbiguousPrefixes { + t.Helper() + amb, err := store.AmbiguousPrefixes(ctx) + if err != nil { + t.Fatal(err) + } + return amb +} + func TestReconfirmReleasesEarlierBatchPostgres(t *testing.T) { ctx, pool, store := reconfirmPool(t) before := time.Now() - if n, err := store.ReconfirmRoutes(ctx, 1, before); err != nil || n != 1 { + if n, err := store.ReconfirmRoutes(ctx, 1, before, ambiguity(t, ctx, store)); err != nil || n != 1 { t.Fatalf("first batch: %d, %v", n, err) } gate, err := pool.Begin(ctx) @@ -40,8 +49,9 @@ CREATE TRIGGER pause_second_route BEFORE UPDATE ON known_routes FOR EACH ROW EXE t.Fatal(err) } finished := make(chan error, 1) + amb := ambiguity(t, ctx, store) go func() { - _, err := store.ReconfirmRoutes(ctx, 1, before) + _, err := store.ReconfirmRoutes(ctx, 1, before, amb) finished <- err }() deadline := time.Now().Add(3 * time.Second) @@ -131,7 +141,7 @@ UPDATE known_routes SET node_ids=node_ids || md5('missing')::uuid, hash_prefix=h original := snapshot() var checked int64 for range 10 { - n, err := store.ReconfirmRoutes(ctx, 3, before) + n, err := store.ReconfirmRoutes(ctx, 3, before, ambiguity(t, ctx, store)) if err != nil { t.Fatal(err) } @@ -165,7 +175,7 @@ UPDATE known_routes SET node_ids=node_ids || md5('missing')::uuid, hash_prefix=h func TestReconfirmCancelledBatchRollsBackPostgres(t *testing.T) { ctx, pool, store := reconfirmPool(t) before := time.Now() - if _, err := store.ReconfirmRoutes(ctx, 1, before); err != nil { + if _, err := store.ReconfirmRoutes(ctx, 1, before, ambiguity(t, ctx, store)); err != nil { t.Fatal(err) } _, err := pool.Exec(ctx, ` @@ -177,7 +187,7 @@ CREATE TRIGGER pause_reconfirm BEFORE UPDATE ON known_routes FOR EACH ROW EXECUT } batchCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) defer cancel() - if _, err := store.ReconfirmRoutes(batchCtx, 1, before); err == nil { + if _, err := store.ReconfirmRoutes(batchCtx, 1, before, ambiguity(t, ctx, store)); err == nil { t.Fatal("expected batch cancellation") } // Waiting for this DDL also waits for cancellation to release the row lock. @@ -191,7 +201,7 @@ CREATE TRIGGER pause_reconfirm BEFORE UPDATE ON known_routes FOR EACH ROW EXECUT if total != 2 || stamped != 1 { t.Fatalf("cancellation changed committed work: total %d, checked %d", total, stamped) } - if n, err := store.ReconfirmRoutes(ctx, 1, before); err != nil || n != 1 { + if n, err := store.ReconfirmRoutes(ctx, 1, before, ambiguity(t, ctx, store)); err != nil || n != 1 { t.Fatalf("cancelled route was not available to retry: %d, %v", n, err) } } @@ -199,10 +209,10 @@ CREATE TRIGGER pause_reconfirm BEFORE UPDATE ON known_routes FOR EACH ROW EXECUT func TestReconfirmFutureCutoffDoesNotRepeatPostgres(t *testing.T) { ctx, _, store := reconfirmPool(t) before := time.Now().Add(time.Hour) - if n, err := store.ReconfirmRoutes(ctx, 2, before); err != nil || n != 2 { + if n, err := store.ReconfirmRoutes(ctx, 2, before, ambiguity(t, ctx, store)); err != nil || n != 2 { t.Fatalf("first batch: %d, %v", n, err) } - if n, err := store.ReconfirmRoutes(ctx, 2, before); err != nil || n != 0 { + if n, err := store.ReconfirmRoutes(ctx, 2, before, ambiguity(t, ctx, store)); err != nil || n != 0 { t.Fatalf("completed routes consumed the run budget twice: %d, %v", n, err) } } @@ -220,7 +230,7 @@ CREATE TRIGGER pause_valid_route BEFORE UPDATE ON known_routes FOR EACH ROW EXEC before := time.Now() batchCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) defer cancel() - if _, err := store.ReconfirmRoutes(batchCtx, 2, before); err == nil { + if _, err := store.ReconfirmRoutes(batchCtx, 2, before, ambiguity(t, ctx, store)); err == nil { t.Fatal("expected batch cancellation") } if _, err := pool.Exec(ctx, `DROP TRIGGER pause_valid_route ON known_routes`); err != nil { @@ -230,7 +240,7 @@ CREATE TRIGGER pause_valid_route BEFORE UPDATE ON known_routes FOR EACH ROW EXEC if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes`).Scan(&count); err != nil || count != 2 { t.Fatalf("cancelled batch partially deleted routes: %d, %v", count, err) } - if n, err := store.ReconfirmRoutes(ctx, 2, before); err != nil || n != 2 { + if n, err := store.ReconfirmRoutes(ctx, 2, before, ambiguity(t, ctx, store)); err != nil || n != 2 { t.Fatalf("retry did not process both routes: %d, %v", n, err) } if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes WHERE path_key=int4send(2)`).Scan(&count); err != nil || count != 1 { @@ -299,7 +309,7 @@ func TestReconfirmSkipsBusyRoutePostgres(t *testing.T) { } batchCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) defer cancel() - if _, err := store.ReconfirmRoutes(batchCtx, 1, time.Now()); err != nil { + if _, err := store.ReconfirmRoutes(batchCtx, 1, time.Now(), ambiguity(t, ctx, store)); err != nil { t.Fatalf("maintenance should skip the busy route and validate the next one: %v", err) } var checked, preserved int @@ -312,7 +322,7 @@ func TestReconfirmSkipsBusyRoutePostgres(t *testing.T) { if err := busy.Rollback(ctx); err != nil { t.Fatal(err) } - if _, err := store.ReconfirmRoutes(ctx, 1, time.Now()); err != nil { + if _, err := store.ReconfirmRoutes(ctx, 1, time.Now(), ambiguity(t, ctx, store)); err != nil { t.Fatal(err) } if err := pool.QueryRow(ctx, `SELECT count(*) FROM known_routes WHERE last_reconfirmed_at > '2026-02-01'`).Scan(&checked); err != nil { diff --git a/db/routes.go b/db/routes.go index 090d2a00..69183fb4 100644 --- a/db/routes.go +++ b/db/routes.go @@ -287,12 +287,36 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t return results, nil } +// AmbiguousPrefixes is the per-IATA set of hop prefixes that resolve to more than one node. +type AmbiguousPrefixes struct { + IATAs []string + Lens []int32 + Prefixes [][]byte +} + +func (s *Store) AmbiguousPrefixes(ctx context.Context) (AmbiguousPrefixes, error) { + rows, err := s.q.AmbiguousPrefixes(ctx) + if err != nil { + return AmbiguousPrefixes{}, err + } + amb := AmbiguousPrefixes{IATAs: make([]string, 0, len(rows)), Lens: make([]int32, 0, len(rows)), Prefixes: make([][]byte, 0, len(rows))} + for _, r := range rows { + amb.IATAs = append(amb.IATAs, r.Iata) + amb.Lens = append(amb.Lens, r.Len) + amb.Prefixes = append(amb.Prefixes, r.Prefix) + } + return amb, nil +} + // ReconfirmRoutes checks the batchSize least-recently-reconfirmed routes, // deleting stale or ambiguous ones and stamping the survivors. -func (s *Store) ReconfirmRoutes(ctx context.Context, batchSize int32, before time.Time) (int64, error) { +func (s *Store) ReconfirmRoutes(ctx context.Context, batchSize int32, before time.Time, amb AmbiguousPrefixes) (int64, error) { return s.q.ReconfirmRoutes(ctx, sqlc.ReconfirmRoutesParams{ BatchSize: batchSize, Before: pgtype.Timestamptz{Time: before, Valid: true}, + AmbIata: amb.IATAs, + AmbLen: amb.Lens, + AmbPrefix: amb.Prefixes, }) } diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 8a9a7efe..82d16603 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -43,6 +43,21 @@ func (m *MockQuerier) EXPECT() *MockQuerierMockRecorder { return m.recorder } +// AmbiguousPrefixes mocks base method. +func (m *MockQuerier) AmbiguousPrefixes(ctx context.Context) ([]db.AmbiguousPrefixesRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AmbiguousPrefixes", ctx) + ret0, _ := ret[0].([]db.AmbiguousPrefixesRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AmbiguousPrefixes indicates an expected call of AmbiguousPrefixes. +func (mr *MockQuerierMockRecorder) AmbiguousPrefixes(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AmbiguousPrefixes", reflect.TypeOf((*MockQuerier)(nil).AmbiguousPrefixes), ctx) +} + // CreateAccount mocks base method. func (m *MockQuerier) CreateAccount(ctx context.Context, name string) (db.Account, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index ef65fd7b..6d699a94 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -12,6 +12,8 @@ import ( ) type Querier interface { + // Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. + AmbiguousPrefixes(ctx context.Context) ([]AmbiguousPrefixesRow, error) CreateAccount(ctx context.Context, name string) (Account, error) // Lock the current row before deciding the outcome, including when another // deactivation commits while this statement is waiting for its row lock. @@ -223,7 +225,8 @@ type Querier interface { ReconfirmNeighbors(ctx context.Context) error // Checks one batch of least-recently-reconfirmed routes: deletes those with a departed // hop node or a hop prefix now matching >1 node in that IATA (length-aware: - // 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. + // 1/2/3/4-byte hop prefixes check prefix_1/2/3/4; ambiguity set supplied by AmbiguousPrefixes), + // and stamps the survivors. ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams) (int64, error) RefreshHourlyStats(ctx context.Context) error RefreshObserverActivity(ctx context.Context) error diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 3b3def70..dcd52c23 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -12,6 +12,43 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const ambiguousPrefixes = `-- name: AmbiguousPrefixes :many +SELECT iata::text AS iata, 1::int AS len, prefix_1 AS prefix FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 2::int, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 3::int, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1 +UNION ALL +SELECT iata::text, 4::int, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1 +` + +type AmbiguousPrefixesRow struct { + Iata string `json:"iata"` + Len int32 `json:"len"` + Prefix []byte `json:"prefix"` +} + +// Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. +func (q *Queries) AmbiguousPrefixes(ctx context.Context) ([]AmbiguousPrefixesRow, error) { + rows, err := q.db.Query(ctx, ambiguousPrefixes) + if err != nil { + return nil, err + } + defer rows.Close() + items := []AmbiguousPrefixesRow{} + for rows.Next() { + var i AmbiguousPrefixesRow + if err := rows.Scan(&i.Iata, &i.Len, &i.Prefix); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const createAccount = `-- name: CreateAccount :one INSERT INTO accounts (name) VALUES ($1) ON CONFLICT (name) WHERE deactivated_at IS NULL DO NOTHING @@ -3972,14 +4009,9 @@ WITH batch AS MATERIALIZED ( LIMIT $2 FOR UPDATE OF r SKIP LOCKED ), -amb AS MATERIALIZED ( - SELECT iata, 1 AS len, prefix_1 AS p FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 - UNION ALL - SELECT iata, 2, prefix_2 FROM node_short_ids GROUP BY iata, prefix_2 HAVING COUNT(*) > 1 - UNION ALL - SELECT iata, 3, prefix_3 FROM node_short_ids GROUP BY iata, prefix_3 HAVING COUNT(*) > 1 - UNION ALL - SELECT iata, 4, prefix_4 FROM node_short_ids GROUP BY iata, prefix_4 HAVING COUNT(*) > 1 +amb AS ( + SELECT a.iata::char(3) AS iata, a.len, a.p + FROM ROWS FROM (unnest($3::text[]), unnest($4::int[]), unnest($5::bytea[])) AS a(iata, len, p) ), dead AS ( SELECT b.iata, b.path_key @@ -4020,13 +4052,23 @@ SELECT count(*) FROM batch type ReconfirmRoutesParams struct { Before pgtype.Timestamptz `json:"before"` BatchSize int32 `json:"batch_size"` + AmbIata []string `json:"amb_iata"` + AmbLen []int32 `json:"amb_len"` + AmbPrefix [][]byte `json:"amb_prefix"` } // Checks one batch of least-recently-reconfirmed routes: deletes those with a departed // hop node or a hop prefix now matching >1 node in that IATA (length-aware: -// 1/2/3/4-byte hop prefixes check prefix_1/2/3/4), and stamps the survivors. +// 1/2/3/4-byte hop prefixes check prefix_1/2/3/4; ambiguity set supplied by AmbiguousPrefixes), +// and stamps the survivors. func (q *Queries) ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams) (int64, error) { - row := q.db.QueryRow(ctx, reconfirmRoutes, arg.Before, arg.BatchSize) + row := q.db.QueryRow(ctx, reconfirmRoutes, + arg.Before, + arg.BatchSize, + arg.AmbIata, + arg.AmbLen, + arg.AmbPrefix, + ) var count int64 err := row.Scan(&count) return count, err diff --git a/internal/background/reconfirm_test.go b/internal/background/reconfirm_test.go index 2012688f..991a7c4c 100644 --- a/internal/background/reconfirm_test.go +++ b/internal/background/reconfirm_test.go @@ -9,10 +9,13 @@ import ( "testing" "testing/synctest" "time" + + "github.com/MeshCore-Beacon/beacon-server/db" ) type reconfirmStore struct { run func(context.Context, int32, time.Time) (int64, error) + ambCalls int neighbors bool } @@ -20,7 +23,12 @@ func (*reconfirmStore) DeleteOldRoutes(context.Context, time.Time, int64, time.T return nil } -func (s *reconfirmStore) ReconfirmRoutes(ctx context.Context, n int32, before time.Time) (int64, error) { +func (s *reconfirmStore) AmbiguousPrefixes(context.Context) (db.AmbiguousPrefixes, error) { + s.ambCalls++ + return db.AmbiguousPrefixes{}, nil +} + +func (s *reconfirmStore) ReconfirmRoutes(ctx context.Context, n int32, before time.Time, _ db.AmbiguousPrefixes) (int64, error) { return s.run(ctx, n, before) } @@ -60,6 +68,9 @@ func TestReconfirmCoverage(t *testing.T) { if left != tc.wantLeft || !s.neighbors { t.Fatalf("remaining %d, neighbor cleanup %t", left, s.neighbors) } + if s.ambCalls != 1 { + t.Fatalf("ambiguity computed %d times, want once per run", s.ambCalls) + } }) } } diff --git a/internal/background/tasks.go b/internal/background/tasks.go index 7d82fdfa..951dca75 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -95,7 +95,8 @@ const ( type routeMaintainer interface { DeleteOldRoutes(context.Context, time.Time, int64, time.Time) error - ReconfirmRoutes(context.Context, int32, time.Time) (int64, error) + AmbiguousPrefixes(context.Context) (db.AmbiguousPrefixes, error) + ReconfirmRoutes(context.Context, int32, time.Time, db.AmbiguousPrefixes) (int64, error) ReconfirmNeighbors(context.Context) error } @@ -137,10 +138,14 @@ func ReconfirmTask(store routeMaintainer, routeRetention, routeGrace time.Durati if err := store.DeleteOldRoutes(ctx, now.Add(-routeRetention), routeMinObservations, now.Add(-routeGrace)); err != nil { return fmt.Errorf("route retention: %w", err) } + amb, err := store.AmbiguousPrefixes(ctx) + if err != nil { + return fmt.Errorf("ambiguous prefixes: %w", err) + } for remaining := int64(reconfirmRunLimit); remaining > 0; { limit := min(int64(reconfirmBatchSize), remaining) batchCtx, cancel := context.WithTimeout(ctx, reconfirmBatchTimeout) - n, err := store.ReconfirmRoutes(batchCtx, int32(limit), now) + n, err := store.ReconfirmRoutes(batchCtx, int32(limit), now, amb) cancel() if err != nil { return fmt.Errorf("routes: %w", err) From 3b548900a08d4bde6fded82eb89d80af07b5d31d Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 098/141] fix(api): tolerate client clock skew and drop the dead route evidence cursor --- db/route_evidence.go | 2 +- internal/api/handlers/limits.go | 3 +++ internal/api/handlers/observers.go | 2 +- internal/api/handlers/observers_test.go | 9 ++++++++ internal/api/handlers/route_evidence.go | 4 ++-- internal/api/handlers/route_evidence_test.go | 11 +++++++++- internal/api/route_evidence.go | 23 ++++++++++---------- 7 files changed, 38 insertions(+), 16 deletions(-) diff --git a/db/route_evidence.go b/db/route_evidence.go index fc7209e0..e7017ea5 100644 --- a/db/route_evidence.go +++ b/db/route_evidence.go @@ -51,7 +51,7 @@ func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query ap } route := toKnownRoutes([]knownRouteRow{{ID: row.ID, NodeIds: row.NodeIds, HashPrefix: row.HashPrefix, Iata: row.Iata, HopCount: row.HopCount, FirstSeen: row.FirstSeen, LastSeen: row.LastSeen, ObservationCount: row.ObservationCount}}, nodes)[0] route.PathKey = hex.EncodeToString(row.PathKey) - out := &api.RouteEvidence{Page: api.Page[api.RouteObservation]{Items: []api.RouteObservation{}}, Route: route, WindowStart: query.Since.UnixMilli(), WindowEnd: query.Until.UnixMilli(), GeneratedAt: time.Now().UnixMilli(), MatchType: "saved_path_prefixes"} + out := &api.RouteEvidence{Items: []api.RouteObservation{}, Route: route, WindowStart: query.Since.UnixMilli(), WindowEnd: query.Until.UnixMilli(), GeneratedAt: time.Now().UnixMilli(), MatchType: "saved_path_prefixes"} width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) if !valid { return out, nil diff --git a/internal/api/handlers/limits.go b/internal/api/handlers/limits.go index dfc48233..80bda95b 100644 --- a/internal/api/handlers/limits.go +++ b/internal/api/handlers/limits.go @@ -8,10 +8,13 @@ import ( "log/slog" "net/http" "strconv" + "time" ) const maxListLimit int32 = 200 +const clockSkewTolerance = 5 * time.Minute // client clocks run ahead; accept a small future 'until' + // parseLimit caps the requested result size while retaining each endpoint's default. func parseLimit(r *http.Request, defaultLimit int32) (int32, error) { value := r.URL.Query().Get("limit") diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 66bddd8d..89fddd68 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -300,7 +300,7 @@ func getObserverActivity(reader api.Reader) http.HandlerFunc { if values, ok := r.URL.Query()["until"]; ok { n, err := strconv.ParseInt(values[0], 10, 64) now := time.Now() - if len(values) != 1 || err != nil || n < 0 || n > now.UnixMilli() || n < now.Add(-720*time.Hour).UnixMilli() { + if len(values) != 1 || err != nil || n < 0 || n > now.Add(clockSkewTolerance).UnixMilli() || n < now.Add(-720*time.Hour).UnixMilli() { respondError(w, http.StatusBadRequest, "until must be one past epoch-millisecond timestamp within 30 days") return } diff --git a/internal/api/handlers/observers_test.go b/internal/api/handlers/observers_test.go index cfef1c9c..7344526e 100644 --- a/internal/api/handlers/observers_test.go +++ b/internal/api/handlers/observers_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" "testing" @@ -225,6 +226,14 @@ func TestGetObserverActivity_Defaults(t *testing.T) { if body.Range != "24h" || body.Interval != "15m" { t.Errorf("expected range 24h interval 15m, got %q/%q", body.Range, body.Interval) } + + skewed := time.Now().Add(time.Minute).UnixMilli() + req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/observers/%s/activity?until=%d", observerID.String(), skewed), nil) + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200 for until within clock skew, got %d (%s)", w.Code, w.Body.String()) + } } func TestGetObserverActivity_CustomRangeAndInterval(t *testing.T) { diff --git a/internal/api/handlers/route_evidence.go b/internal/api/handlers/route_evidence.go index 6c156fef..a194306c 100644 --- a/internal/api/handlers/route_evidence.go +++ b/internal/api/handlers/route_evidence.go @@ -67,7 +67,7 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R return q, api.ErrRouteEvidenceInput } } - if !api.ValidRouteEvidenceWindow(q.Since, q.Until) || q.Until.After(now) { + if !api.ValidRouteEvidenceWindow(q.Since, q.Until) || q.Until.After(now.Add(clockSkewTolerance)) { return q, api.ErrRouteEvidenceInput } return q, nil @@ -83,7 +83,7 @@ func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.R // @Param pathKey path string true "Stable 32-hex pathKey from a known-route response" // @Param range query string false "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor" // @Param since query int false "Window start epoch ms; provide with until (default last 24h)" -// @Param until query int false "Exclusive window end epoch ms; maximum span 30d, no future end" +// @Param until query int false "Exclusive window end epoch ms; maximum span 30d, end may be up to 5 minutes ahead of server time (clock skew tolerance)" // @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; window is pinned" // @Param limit query int false "Default 50; positive, capped at 200" minimum(1) maximum(200) // @Success 200 {object} api.RouteEvidence diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go index 8340d85b..f2268d86 100644 --- a/internal/api/handlers/route_evidence_test.go +++ b/internal/api/handlers/route_evidence_test.go @@ -5,6 +5,7 @@ package handlers import ( "context" "errors" + "fmt" "net/http/httptest" "net/url" "testing" @@ -44,6 +45,14 @@ func TestRouteEvidenceQuery(t *testing.T) { t.Errorf("accepted %s", query) } } + ahead := now.Add(time.Minute) + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", fmt.Sprintf("/?since=%d&until=%d", ahead.Add(-time.Hour).UnixMilli(), ahead.UnixMilli()), nil), "YOW", evidenceKey, now); err != nil { + t.Fatalf("rejected until within clock skew: %v", err) + } + far := now.Add(10 * time.Minute) + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", fmt.Sprintf("/?since=%d&until=%d", far.Add(-time.Hour).UnixMilli(), far.UnixMilli()), nil), "YOW", evidenceKey, now); err == nil { + t.Fatal("accepted until far in the future") + } } func TestRouteEvidenceHandler(t *testing.T) { @@ -64,7 +73,7 @@ func TestRouteEvidenceHandler(t *testing.T) { if iata != "YOW" || key != evidenceKey || q.Limit != 50 { t.Fatal("wrong reader scope") } - return &api.RouteEvidence{Page: api.Page[api.RouteObservation]{Items: []api.RouteObservation{}}}, tc.err + return &api.RouteEvidence{Items: []api.RouteObservation{}}, tc.err }}) w := httptest.NewRecorder() router.ServeHTTP(w, httptest.NewRequest("GET", tc.path, nil)) diff --git a/internal/api/route_evidence.go b/internal/api/route_evidence.go index cadc056a..d320ade5 100644 --- a/internal/api/route_evidence.go +++ b/internal/api/route_evidence.go @@ -30,18 +30,19 @@ type RouteObservation struct { SNR *float32 `json:"snr,omitempty"` } -// RouteEvidence uses nextPageCursor rather than the lossy legacy numeric cursor. +// RouteEvidence pages by nextPageCursor, which keeps full timestamp precision. type RouteEvidence struct { - Page[RouteObservation] - Route KnownRoute `json:"route"` - WindowStart int64 `json:"windowStart"` - WindowEnd int64 `json:"windowEnd"` - GeneratedAt int64 `json:"generatedAt"` - MatchType string `json:"matchType"` - MatchAvailable bool `json:"matchAvailable"` - HashSize int16 `json:"hashSize"` - PathBytes string `json:"pathBytes"` - NextPageCursor *string `json:"nextPageCursor,omitempty"` + Items []RouteObservation `json:"items"` + HasMore bool `json:"hasMore"` + Route KnownRoute `json:"route"` + WindowStart int64 `json:"windowStart"` + WindowEnd int64 `json:"windowEnd"` + GeneratedAt int64 `json:"generatedAt"` + MatchType string `json:"matchType"` + MatchAvailable bool `json:"matchAvailable"` + HashSize int16 `json:"hashSize"` + PathBytes string `json:"pathBytes"` + NextPageCursor *string `json:"nextPageCursor,omitempty"` } type RouteEvidenceQuery struct { From 9ce4e8d10c31a05e79eddcb2077f0710e5e2486e Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 099/141] fix(cache): invalidate derived scope keys on catalogue import --- internal/cache/cache.go | 18 ++++++++++++++++++ internal/cache/cache_test.go | 13 +++++++++++++ internal/cache/reader.go | 4 +++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 92377544..48fb5694 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -76,6 +76,24 @@ func (c *Client) del(ctx context.Context, keys ...string) { c.rdb.Del(ctx, keys...) } +// delPrefix SCANs the whole keyspace; call it rarely. +func (c *Client) delPrefix(ctx context.Context, prefix string) { + var cursor uint64 + for { + keys, next, err := c.rdb.Scan(ctx, cursor, prefix+"*", 100).Result() + if err != nil { + return + } + if len(keys) > 0 { + c.rdb.Del(ctx, keys...) + } + if next == 0 { + return + } + cursor = next + } +} + // getOrSet retrieves a cached value by key, deserialising it into T. // On a cache miss it calls fetch, stores the result under key with the given // TTL, and returns it. If Redis is unavailable or returns an unexpected error, diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 8184a17b..22515312 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -383,6 +383,19 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { } } +func TestInvalidateScopeNamesClearsDerivedKeys(t *testing.T) { + c, mr := newTestClient(t) + cr := &CachedReader{inner: &stubReader{}, c: c, ttl: CacheTTLs{}} + for _, k := range []string{keyScopeNames, keyScopesByIATAsPrefix + "YOW", keyScopesByIATAsPrefix + "YOW,YVR", keyScopeByNamePrefix + "#yow"} { + mr.Set(k, "x") + } + mr.Set("beacon:iatas", "keep") + cr.InvalidateScopeNames(context.Background()) + if got := mr.Keys(); len(got) != 1 || got[0] != "beacon:iatas" { + t.Fatalf("keys left: %v", got) + } +} + func TestCachedReader_InvalidateNode(t *testing.T) { c, mr := newTestClient(t) nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000001") diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 515adc93..2dd3dda2 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -80,9 +80,11 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { } } -// InvalidateScopeNames makes newly committed catalogue names available to filters. +// InvalidateScopeNames makes newly committed catalogue names visible to every scope read. func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { cr.c.del(ctx, keyScopeNames) + cr.c.delPrefix(ctx, keyScopesByIATAsPrefix) + cr.c.delPrefix(ctx, keyScopeByNamePrefix) } // InvalidateNode removes the cached entries for a node by UUID. From 8a488fde95b065ed1e8b52ab22016cb7655657e9 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 100/141] fix(meshmapper): retry failed catalogue fetches sooner A transient failure pushed the next attempt out by the full refresh interval. Failures now retry in five minutes, with a server Retry-After still honoured as a ceiling. --- internal/meshmapper/scopes.go | 15 +++++++++--- internal/meshmapper/scopes_test.go | 39 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index 23363e69..5eb0d7bb 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -26,6 +26,8 @@ const ( MaxScopes = 64 // per source; at most 16 configured sources (1024 imported identities) MaxBody = 64 << 10 PollInterval = 15 * time.Second // one source per tick, at most four requests/minute + + failureRetry = 5 * time.Minute // transient failures shouldn't cost a whole refresh interval ) // Cache stores source provenance and freshness separately from packet evidence. @@ -142,6 +144,7 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error response, err := i.client.Do(request) var entries []scopestore.Entry var generated time.Time + var retryAfter time.Time if err != nil { if ctx.Err() != nil { return ctx.Err() @@ -169,10 +172,7 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error if response.StatusCode == http.StatusTooManyRequests || response.StatusCode == http.StatusServiceUnavailable { // The published contract uses seconds. Ignore invalid/overflowing headers. if seconds, parseErr := strconv.ParseInt(response.Header.Get("Retry-After"), 10, 64); parseErr == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) { - until := now.Add(time.Duration(seconds) * time.Second) - if until.After(update.NextAttempt) { - update.NextAttempt = until - } + retryAfter = now.Add(time.Duration(seconds) * time.Second) } } } @@ -191,6 +191,13 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error } } } + if update.LastError != "" { + // Transient failures shouldn't cost a whole refresh interval, but never beat a Retry-After. + update.NextAttempt = now.Add(min(i.interval, failureRetry)) + if retryAfter.After(update.NextAttempt) { + update.NextAttempt = retryAfter + } + } if update.LastError == "HTTP 429" { i.retryAfter = update.NextAttempt } diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go index 858ea582..b8e1b24d 100644 --- a/internal/meshmapper/scopes_test.go +++ b/internal/meshmapper/scopes_test.go @@ -278,3 +278,42 @@ func TestInvalidSavedCatalogueDoesNotBlockStartupOrRefresh(t *testing.T) { t.Fatalf("normal shutdown is a task failure: %v", err) } } + +func TestRetryAfterShorterThanIntervalIsHonoured(t *testing.T) { + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "1800") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} + imp, err := New(ctx, cfg, &memoryStore{rows: map[string]Cache{}}, scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { + t.Fatal(err) + } + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != 30*time.Minute { + t.Fatalf("next attempt after 429 in %v, want %v (server Retry-After must not be overridden by the 1h interval)", got, 30*time.Minute) + } +} + +func TestFailedRefreshRetriesSoon(t *testing.T) { + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) })) + defer server.Close() + cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} + imp, err := New(ctx, cfg, &memoryStore{rows: map[string]Cache{}}, scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { + t.Fatal(err) + } + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != failureRetry { + t.Fatalf("next attempt after failure in %v, want %v", got, failureRetry) + } +} From 50d35d8626601b598fdc4da5553705ac9abba401 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 101/141] fix(api): emit observer statusMetadata as JSON --- db/observers.go | 3 ++- db/observers_test.go | 12 ++++++++++++ internal/api/observers.go | 18 +++++++++++------- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/db/observers.go b/db/observers.go index fc10efad..f312aca1 100644 --- a/db/observers.go +++ b/db/observers.go @@ -6,6 +6,7 @@ package db import ( "context" "encoding/hex" + "encoding/json" "fmt" "log/slog" "math" @@ -116,7 +117,7 @@ func (s *Store) GetObserver(ctx context.Context, observerID uuid.UUID) (*api.Obs RadioCR: obs.RadioCr, BatteryLevel: obs.BatteryLevel, UptimeSeconds: obs.UptimeSeconds, - StatusMetadata: obs.StatusMetadata, + StatusMetadata: json.RawMessage(obs.StatusMetadata), FirstSeen: obs.FirstSeen.Time.UnixMilli(), LastSeen: obs.LastSeen.Time.UnixMilli(), ObservationCount: *obs.ObservationCount, diff --git a/db/observers_test.go b/db/observers_test.go index f6c67d3d..e5b98506 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -5,7 +5,9 @@ package db import ( "context" + "encoding/json" "errors" + "strings" "testing" "time" @@ -178,6 +180,7 @@ func TestGetObserver_OnlineStatus(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") obsCount := int64(10) + statusMetadata := []byte(`{"stats":{"noise_floor":-97}}`) mock.EXPECT(). GetObserverByID(gomock.Any(), observerID). @@ -188,6 +191,7 @@ func TestGetObserver_OnlineStatus(t *testing.T) { FirstSeen: pgtype.Timestamptz{Time: time.Now().Add(-time.Hour), Valid: true}, LastSeen: pgtype.Timestamptz{Time: time.Now().Add(-time.Minute), Valid: true}, LastStatusAt: pgtype.Timestamptz{Time: time.Now().Add(-time.Minute), Valid: true}, + StatusMetadata: statusMetadata, }, nil) mock.EXPECT(). @@ -213,6 +217,14 @@ func TestGetObserver_OnlineStatus(t *testing.T) { if observer.IATA != "YVR" { t.Errorf("expected IATA YVR, got %s", observer.IATA) } + + out, err := json.Marshal(observer) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), `"statusMetadata":{"stats":{"noise_floor":-97}}`) { + t.Fatalf("statusMetadata not an object: %s", out) + } } func TestGetObserver_OfflineStatus(t *testing.T) { diff --git a/internal/api/observers.go b/internal/api/observers.go index 2da680c1..080b74d7 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -3,7 +3,11 @@ package api -import "github.com/google/uuid" +import ( + "encoding/json" + + "github.com/google/uuid" +) // ObserverSummary is the minimal observer representation used in list responses. type ObserverSummary struct { @@ -40,12 +44,12 @@ type Observer struct { RadioCR *int16 `json:"radioCr,omitempty"` // coding rate denominator BatteryLevel *float32 `json:"batteryLevel,omitempty"` // volts, nil if mains powered UptimeSeconds *int64 `json:"uptimeSeconds,omitempty"` - StatusMetadata any `json:"statusMetadata,omitempty"` // raw /status JSON payload - LastStatusAt *int64 `json:"lastStatusAt,omitempty"` // epoch ms - FirstSeen int64 `json:"firstSeen"` // epoch ms - LastSeen int64 `json:"lastSeen"` // epoch ms - ObservationCount int64 `json:"observationCount"` // legacy cumulative presence counter; includes non-packet events - Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on + StatusMetadata json.RawMessage `json:"statusMetadata,omitempty" swaggertype:"object"` // raw /status JSON payload + LastStatusAt *int64 `json:"lastStatusAt,omitempty"` // epoch ms + FirstSeen int64 `json:"firstSeen"` // epoch ms + LastSeen int64 `json:"lastSeen"` // epoch ms + ObservationCount int64 `json:"observationCount"` // legacy cumulative presence counter; includes non-packet events + Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on } // ObserverTelemetryPoint is a single telemetry snapshot for an observer. From f6fe177a4e6180ce68d539fd6345bc8181b6edf7 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 10:28:03 -0400 Subject: [PATCH 102/141] docs: regenerate swagger --- docs/docs.go | 10 ++++------ docs/swagger.json | 10 ++++------ docs/swagger.yaml | 8 ++++---- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 288a70b6..df1ccc9e 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2170,7 +2170,7 @@ const docTemplate = `{ }, { "type": "integer", - "description": "Exclusive window end epoch ms; maximum span 30d, no future end", + "description": "Exclusive window end epoch ms; maximum span 30d, end may be up to 5 minutes ahead of server time (clock skew tolerance)", "name": "until", "in": "query" }, @@ -4080,7 +4080,8 @@ const docTemplate = `{ "type": "string" }, "statusMetadata": { - "description": "raw /status JSON payload" + "description": "raw /status JSON payload", + "type": "object" }, "uptimeSeconds": { "type": "integer" @@ -4631,7 +4632,7 @@ const docTemplate = `{ "type": "string" }, "summary": { - "description": "advert name from this packet; omitted when unavailable or unsupported", + "description": "advert name, or the ACK/TRACE/PING summary; omitted when unavailable", "type": "string" } } @@ -4973,9 +4974,6 @@ const docTemplate = `{ "matchType": { "type": "string" }, - "nextCursor": { - "type": "integer" - }, "nextPageCursor": { "type": "string" }, diff --git a/docs/swagger.json b/docs/swagger.json index 56794446..183611af 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2168,7 +2168,7 @@ }, { "type": "integer", - "description": "Exclusive window end epoch ms; maximum span 30d, no future end", + "description": "Exclusive window end epoch ms; maximum span 30d, end may be up to 5 minutes ahead of server time (clock skew tolerance)", "name": "until", "in": "query" }, @@ -4078,7 +4078,8 @@ "type": "string" }, "statusMetadata": { - "description": "raw /status JSON payload" + "description": "raw /status JSON payload", + "type": "object" }, "uptimeSeconds": { "type": "integer" @@ -4629,7 +4630,7 @@ "type": "string" }, "summary": { - "description": "advert name from this packet; omitted when unavailable or unsupported", + "description": "advert name, or the ACK/TRACE/PING summary; omitted when unavailable", "type": "string" } } @@ -4971,9 +4972,6 @@ "matchType": { "type": "string" }, - "nextCursor": { - "type": "integer" - }, "nextPageCursor": { "type": "string" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index a58626f2..ca79d350 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -570,6 +570,7 @@ definitions: type: string statusMetadata: description: raw /status JSON payload + type: object uptimeSeconds: type: integer type: object @@ -956,7 +957,7 @@ definitions: description: matched transport scope name e.g. "#bc" type: string summary: - description: advert name from this packet; omitted when unavailable or unsupported + description: advert name, or the ACK/TRACE/PING summary; omitted when unavailable type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.PacketTransportCodes: @@ -1183,8 +1184,6 @@ definitions: type: boolean matchType: type: string - nextCursor: - type: integer nextPageCursor: type: string pathBytes: @@ -2894,7 +2893,8 @@ paths: in: query name: since type: integer - - description: Exclusive window end epoch ms; maximum span 30d, no future end + - description: Exclusive window end epoch ms; maximum span 30d, end may be up + to 5 minutes ahead of server time (clock skew tolerance) in: query name: until type: integer From 689bc2326a0462bb387c6087f5cb3514cca636f1 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 11:26:34 -0400 Subject: [PATCH 103/141] fix(ingest): clear a node's location on an explicit 0/0 advert A repeater that advertises 0/0 is hiding its position. It was stored as a literal 0/0 that every consumer had to special-case; now the stored latitude, longitude and location source are cleared, an omitted location still preserves the stored one, and the nodeUpdate event sends explicit null lat/lng so clients can clear too. Migration 043 drops rows stored as 0/0 before this change. --- .github/workflows/ci.yml | 2 +- README.md | 9 ++- db/migrations/043_clear_zero_locations.sql | 6 ++ db/node_location_integration_test.go | 61 ++++++++++++----- db/nodes.go | 1 + db/queries/queries.sql | 8 +-- db/sqlc/queries.sql.go | 10 +-- internal/borders/borders.go | 4 +- internal/borders/borders_test.go | 2 +- internal/ingest/side_effects.go | 20 ++++-- internal/ingest/side_effects_test.go | 77 +++++++++++++++++++--- 11 files changed, 151 insertions(+), 49 deletions(-) create mode 100644 db/migrations/043_clear_zero_locations.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9574594d..4dc16322 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index cae5c801..dfcec34f 100644 --- a/README.md +++ b/README.md @@ -313,15 +313,18 @@ invalid geometry fails startup. Border changes require a restart. Inside any polygon (including its edges) means `false`; outside the entire union means `true`. Hole interiors are outside; hole edges are local. Other node roles, -missing/invalid positions and the 0/0 location reset have no classification. +missing or invalid positions have no classification (an explicit 0/0 advert +clears the stored position). This is a hint based on reported position, not proof of a repeater's origin. Packet ingestion, heard-in IATAs and route matching remain unchanged. Node list/detail reads apply the current geometry after cache reads, so existing historical nodes need no backfill. A `nodeUpdate` includes `possiblyForeign` when its advert provides a position: a boolean for known positions, `null` to -clear an unknown/reset position. Omission retains the previous value when a -repeater's advert omits its position. A change to another role also sends `null`. +clear it when the advert reports 0/0 or an invalid position. Omission retains the +previous value when a repeater's advert omits its position. A change to another +role also sends `null`. `lat`/`lng` follow the same rule: omitted keeps the +client's position, explicit `null` clears it. The field is omitted everywhere when the feature is disabled (the default). Use longitude/latitude coordinate order and split antimeridian-crossing borders diff --git a/db/migrations/043_clear_zero_locations.sql b/db/migrations/043_clear_zero_locations.sql new file mode 100644 index 00000000..b4a83b41 --- /dev/null +++ b/db/migrations/043_clear_zero_locations.sql @@ -0,0 +1,6 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- An explicit 0/0 advert now clears the stored position; drop old 0/0 rows and stale sources. +UPDATE nodes SET latitude = NULL, longitude = NULL, location_source = NULL +WHERE location_source = 'advert' AND (latitude IS NULL OR (latitude = 0 AND longitude = 0)); diff --git a/db/node_location_integration_test.go b/db/node_location_integration_test.go index dd060011..51f700c4 100644 --- a/db/node_location_integration_test.go +++ b/db/node_location_integration_test.go @@ -41,31 +41,58 @@ func TestNodeLocationResetPostgres(t *testing.T) { if err != nil { t.Fatal(err) } + f := func(v float64) *float64 { return &v } + advert := "advert" for _, tc := range []struct { - name string - present bool - lat, lon float64 + name string + lat, lon *float64 + clear bool + wantLat *float64 + wantLon *float64 + wantSource *string }{ - {"omission preserves location", false, 45, -75}, - {"explicit reset replaces location", true, 0, 0}, - {"omission preserves reset", false, 0, 0}, + {"omission preserves location", nil, nil, false, f(45), f(-75), &advert}, + {"explicit reset clears location", nil, nil, true, nil, nil, nil}, + {"omission preserves cleared location", nil, nil, false, nil, nil, nil}, + {"new coordinates restore location", f(46), f(-76), false, f(46), f(-76), &advert}, } { t.Run(tc.name, func(t *testing.T) { - params.Latitude, params.Longitude = nil, nil - if tc.present { - params.Latitude, params.Longitude = &tc.lat, &tc.lon - } + params.Latitude, params.Longitude, params.ClearLocation = tc.lat, tc.lon, tc.clear updatedID, err := store.UpsertNode(ctx, params, ingest.RadioSettings{}) if err != nil || updatedID != id { t.Fatalf("node identity changed: %v", err) } - var gotLat, gotLon float64 - if err := tx.QueryRow(ctx, "SELECT latitude,longitude FROM nodes WHERE id=$1", id).Scan(&gotLat, &gotLon); err != nil { - t.Fatal(err) - } - if gotLat != tc.lat || gotLon != tc.lon { - t.Fatalf("location = (%g,%g), want (%g,%g)", gotLat, gotLon, tc.lat, tc.lon) - } + assertNodeLocation(t, ctx, tx, []byte{0x97}, tc.wantLat, tc.wantLon, tc.wantSource) }) } + t.Run("new node with cleared location has no source", func(t *testing.T) { + fresh := ingest.UpsertNodeParams{PublicKey: []byte{0x98}, NodeType: 2, Name: "cleared fixture", ClearLocation: true} + if _, err := store.UpsertNode(ctx, fresh, ingest.RadioSettings{}); err != nil { + t.Fatal(err) + } + assertNodeLocation(t, ctx, tx, []byte{0x98}, nil, nil, nil) + }) +} + +func assertNodeLocation(t *testing.T, ctx context.Context, tx pgx.Tx, pubkey []byte, wantLat, wantLon *float64, wantSource *string) { + t.Helper() + var gotLat, gotLon *float64 + var gotSource *string + if err := tx.QueryRow(ctx, "SELECT latitude, longitude, location_source FROM nodes WHERE public_key=$1", pubkey).Scan(&gotLat, &gotLon, &gotSource); err != nil { + t.Fatal(err) + } + if !equalPtr(gotLat, wantLat) || !equalPtr(gotLon, wantLon) || !equalPtr(gotSource, wantSource) { + t.Fatalf("location = (%v,%v,%v), want (%v,%v,%v)", deref(gotLat), deref(gotLon), deref(gotSource), deref(wantLat), deref(wantLon), deref(wantSource)) + } +} + +func equalPtr[T comparable](a, b *T) bool { + return a == nil && b == nil || a != nil && b != nil && *a == *b +} + +func deref[T any](p *T) any { + if p == nil { + return nil + } + return *p } diff --git a/db/nodes.go b/db/nodes.go index 9cc53e33..a602cb3d 100644 --- a/db/nodes.go +++ b/db/nodes.go @@ -34,6 +34,7 @@ func (s *Store) UpsertNode(ctx context.Context, n ingest.UpsertNodeParams, radio Name: &n.Name, Latitude: n.Latitude, Longitude: n.Longitude, + ClearLocation: n.ClearLocation, DeviceClockDriftSeconds: driftSeconds, } if radio.FreqMHz != 0 { diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 77f1c517..c206a2f0 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -745,13 +745,13 @@ ORDER BY po.heard_at ASC; -- name: UpsertNode :one INSERT INTO nodes (public_key, node_type, name, latitude, longitude, location_source, last_advert_at, last_seen, radio_freq_mhz, radio_sf, radio_bw_khz, device_clock_drift_seconds) -VALUES ($1, $2, $3, $4, $5, 'advert', NOW(), NOW(), $6, $7, $8, $9) +VALUES (@public_key, @node_type, @name, @latitude, @longitude, CASE WHEN @latitude::double precision IS NOT NULL THEN 'advert' END, NOW(), NOW(), @radio_freq_mhz, @radio_sf, @radio_bw_khz, @device_clock_drift_seconds) ON CONFLICT (public_key) DO UPDATE SET node_type = EXCLUDED.node_type, name = COALESCE(EXCLUDED.name, nodes.name), - latitude = COALESCE(EXCLUDED.latitude, nodes.latitude), - longitude = COALESCE(EXCLUDED.longitude, nodes.longitude), - location_source = CASE WHEN EXCLUDED.latitude IS NOT NULL THEN 'advert' ELSE nodes.location_source END, + latitude = CASE WHEN @clear_location::bool THEN NULL ELSE COALESCE(EXCLUDED.latitude, nodes.latitude) END, + longitude = CASE WHEN @clear_location::bool THEN NULL ELSE COALESCE(EXCLUDED.longitude, nodes.longitude) END, + location_source = CASE WHEN @clear_location::bool THEN NULL WHEN EXCLUDED.latitude IS NOT NULL THEN 'advert' ELSE nodes.location_source END, last_advert_at = NOW(), last_seen = NOW(), radio_freq_mhz = EXCLUDED.radio_freq_mhz, diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index dcd52c23..0a5aa362 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -4961,13 +4961,13 @@ func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRoutePara const upsertNode = `-- name: UpsertNode :one INSERT INTO nodes (public_key, node_type, name, latitude, longitude, location_source, last_advert_at, last_seen, radio_freq_mhz, radio_sf, radio_bw_khz, device_clock_drift_seconds) -VALUES ($1, $2, $3, $4, $5, 'advert', NOW(), NOW(), $6, $7, $8, $9) +VALUES ($1, $2, $3, $4, $5, CASE WHEN $4::double precision IS NOT NULL THEN 'advert' END, NOW(), NOW(), $6, $7, $8, $9) ON CONFLICT (public_key) DO UPDATE SET node_type = EXCLUDED.node_type, name = COALESCE(EXCLUDED.name, nodes.name), - latitude = COALESCE(EXCLUDED.latitude, nodes.latitude), - longitude = COALESCE(EXCLUDED.longitude, nodes.longitude), - location_source = CASE WHEN EXCLUDED.latitude IS NOT NULL THEN 'advert' ELSE nodes.location_source END, + latitude = CASE WHEN $10::bool THEN NULL ELSE COALESCE(EXCLUDED.latitude, nodes.latitude) END, + longitude = CASE WHEN $10::bool THEN NULL ELSE COALESCE(EXCLUDED.longitude, nodes.longitude) END, + location_source = CASE WHEN $10::bool THEN NULL WHEN EXCLUDED.latitude IS NOT NULL THEN 'advert' ELSE nodes.location_source END, last_advert_at = NOW(), last_seen = NOW(), radio_freq_mhz = EXCLUDED.radio_freq_mhz, @@ -4987,6 +4987,7 @@ type UpsertNodeParams struct { RadioSf *int16 `json:"radio_sf"` RadioBwKhz *float32 `json:"radio_bw_khz"` DeviceClockDriftSeconds *int32 `json:"device_clock_drift_seconds"` + ClearLocation bool `json:"clear_location"` } // ============================================================ @@ -5003,6 +5004,7 @@ func (q *Queries) UpsertNode(ctx context.Context, arg UpsertNodeParams) (Node, e arg.RadioSf, arg.RadioBwKhz, arg.DeviceClockDriftSeconds, + arg.ClearLocation, ) var i Node err := row.Scan( diff --git a/internal/borders/borders.go b/internal/borders/borders.go index 1ab15016..ad2c560d 100644 --- a/internal/borders/borders.go +++ b/internal/borders/borders.go @@ -61,11 +61,11 @@ func validPosition(lat, lng float64) bool { } // PossiblyForeign is nil when disabled, the role is not repeater, or position -// is unknown/invalid (including the protocol's 0/0 location reset). +// is unknown/invalid. // All polygon boundaries, including hole edges, count as local; hole interiors // remain outside. Location is self-reported, so this is an indication only. func (local *Local) PossiblyForeign(role int16, lat, lng *float64) *bool { - if local == nil || role != int16(meshcore.AdvertTypeRepeater) || lat == nil || lng == nil || !validPosition(*lat, *lng) || *lat == 0 && *lng == 0 { + if local == nil || role != int16(meshcore.AdvertTypeRepeater) || lat == nil || lng == nil || !validPosition(*lat, *lng) { return nil } point := orb.Point{*lng, *lat} diff --git a/internal/borders/borders_test.go b/internal/borders/borders_test.go index a90aaf94..c5d4fdfc 100644 --- a/internal/borders/borders_test.go +++ b/internal/borders/borders_test.go @@ -34,7 +34,7 @@ func TestPossiblyForeign(t *testing.T) { {"companion", 1, 25, 25, nil}, {"room", 3, 25, 25, nil}, {"sensor", 4, 25, 25, nil}, - {"zero reset", 2, 0, 0, nil}, + {"null island is a position", 2, 0, 0, new(true)}, {"equator is valid", 2, 0, 25, new(true)}, {"prime meridian is valid", 2, 25, 0, new(true)}, {"invalid latitude", 2, 91, 25, nil}, diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 09132424..46fa8fe8 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -22,6 +22,8 @@ type UpsertNodeParams struct { NodeType uint8 // 1=companion, 2=repeater, 3=room server Latitude *float64 Longitude *float64 + // ClearLocation drops the stored position; set by an explicit 0/0 advert. + ClearLocation bool // AdvertTimestamp is the device's self-reported wall-clock time (epoch seconds) from the // signed advert body. Used to derive clock drift for repeaters/room servers; see // api.Node.ClockDriftSeconds. @@ -65,8 +67,8 @@ type nodeUpdateEvent struct { NodeType uint8 `json:"nodeType"` NodeTypeName string `json:"nodeTypeName"` IATA string `json:"iata"` - Lat *float64 `json:"lat,omitempty"` - Lng *float64 `json:"lng,omitempty"` + Lat **float64 `json:"lat,omitempty"` // Omitted keeps the client's position; explicit null clears it. + Lng **float64 `json:"lng,omitempty"` IsObserver bool `json:"isObserver"` IATAs []api.NodeIATA `json:"iatas"` DefaultScope *string `json:"defaultScope,omitempty"` @@ -92,8 +94,10 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc return } var lat, lon *float64 - // Presence, not value: an explicit 0/0 advert resets the stored location. - if advert.Flags()&meshcore.AdvertLatLonMask != 0 { + present := advert.Flags()&meshcore.AdvertLatLonMask != 0 + // Presence decides: omitted keeps the stored position, an explicit 0/0 clears it. + cleared := present && advert.AppData().Lat == 0 && advert.AppData().Lon == 0 + if present && !cleared { la := float64(advert.AppData().Lat) / 1e6 lo := float64(advert.AppData().Lon) / 1e6 lat = &la @@ -105,6 +109,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc NodeType: advert.Type(), Latitude: lat, Longitude: lon, + ClearLocation: cleared, AdvertTimestamp: advert.Timestamp, } var nodeRadio RadioSettings @@ -179,14 +184,15 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc NodeType: advert.Type(), NodeTypeName: api.NodeTypeName(int16(advert.Type())), IATA: iata, - Lat: lat, - Lng: lon, IsObserver: isObserver, IATAs: []api.NodeIATA{{IATA: iata, LastHeard: time.Now().UnixMilli()}}, DefaultScope: defaultScope, Radio: radioStr, } - if w.cfg.LocalBorders != nil && (lat != nil || advert.Type() != meshcore.AdvertTypeRepeater) { + if present { + evt.Lat, evt.Lng = &lat, &lon + } + if w.cfg.LocalBorders != nil && (present || advert.Type() != meshcore.AdvertTypeRepeater) { foreign := w.cfg.LocalBorders.PossiblyForeign(int16(advert.Type()), lat, lon) evt.PossiblyForeign = &foreign } diff --git a/internal/ingest/side_effects_test.go b/internal/ingest/side_effects_test.go index 782b79f8..93c5a4e3 100644 --- a/internal/ingest/side_effects_test.go +++ b/internal/ingest/side_effects_test.go @@ -9,9 +9,11 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" + "strings" "testing" "time" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" "github.com/MeshCore-Beacon/beacon-server/internal/keystore" "github.com/meshcore-go/meshcore-go" ) @@ -68,16 +70,16 @@ func buildAdvertPacketWithData(t *testing.T, data []byte, tamper bool) *meshcore func TestAdvertLocationPresence(t *testing.T) { for _, tc := range []struct { - name string - present, tamper bool - lat, lon int32 + name string + present, tamper, clear bool + lat, lon int32 }{ - {"ordinary location", true, false, 45000000, -75000000}, - {"explicit reset", true, false, 0, 0}, - {"zero latitude", true, false, 0, -75000000}, - {"zero longitude", true, false, 45000000, 0}, - {"no location", false, false, 0, 0}, - {"tampered reset", true, true, 0, 0}, + {"ordinary location", true, false, false, 45000000, -75000000}, + {"explicit reset", true, false, true, 0, 0}, + {"zero latitude", true, false, false, 0, -75000000}, + {"zero longitude", true, false, false, 45000000, 0}, + {"no location", false, false, false, 0, 0}, + {"tampered reset", true, true, false, 0, 0}, } { t.Run(tc.name, func(t *testing.T) { data := []byte{meshcore.AdvertTypeRepeater} @@ -101,7 +103,10 @@ func TestAdvertLocationPresence(t *testing.T) { t.Fatal("signed advert did not update the node") } got := store.upsertNodeParams - if !tc.present { + if got.ClearLocation != tc.clear { + t.Fatalf("ClearLocation = %v, want %v", got.ClearLocation, tc.clear) + } + if !tc.present || tc.clear { if got.Latitude != nil || got.Longitude != nil { t.Fatal("absent location became an update") } @@ -114,6 +119,58 @@ func TestAdvertLocationPresence(t *testing.T) { } } +func TestAdvertClearEmitsNullLocation(t *testing.T) { + w, _ := newTestWorker() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "clear", hub.Scope{Events: []hub.EventType{hub.EventNodeUpdate, hub.EventObserverStatus}}) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + waitForSummarySubscriber(t, ctx, w.hub, client) + for _, tc := range []struct { + name string + present, want bool + lat, lon int32 + wantLat string + wantLng string + }{ + {"explicit reset sends null", true, true, 0, 0, "null", "null"}, + {"omission sends nothing", false, false, 0, 0, "", ""}, + {"real location sends coordinates", true, true, 45000000, -75000000, "45", "-75"}, + } { + t.Run(tc.name, func(t *testing.T) { + data := []byte{meshcore.AdvertTypeRepeater} + if tc.present { + data[0] |= meshcore.AdvertLatLonMask + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lat)) + data = binary.LittleEndian.AppendUint32(data, uint32(tc.lon)) + } + w.handlePayloadTypeSideEffects(ctx, buildAdvertPacketWithData(t, data, false), "AAA", []byte{1}, RadioSettings{}, nil, nil, nil, 0) + for { + select { + case event := <-client.Send: + if event.Type != hub.EventNodeUpdate { + continue + } + payload := string(event.Payload) + for field, want := range map[string]string{`"lat"`: tc.wantLat, `"lng"`: tc.wantLng} { + if tc.want && !strings.Contains(payload, field+":"+want+",") { + t.Fatalf("%s != %s: %s", field, want, payload) + } + if !tc.want && strings.Contains(payload, field) { + t.Fatalf("%s present on omission: %s", field, payload) + } + } + return + case <-ctx.Done(): + t.Fatal("node update not received") + } + } + }) + } +} + func TestHandlePayloadTypeSideEffects_Advert_ValidSignature_UpsertsNode(t *testing.T) { w, db := newTestWorker() packet := buildAdvertPacket(t, false) From 0e242574fb5ff852412c8c741877e104c59c4c6d Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 12:32:52 -0400 Subject: [PATCH 104/141] feat(meshmapper): import region boundaries from the Zones API Imported boundaries override borderFile on the border map and for foreign marking, falling back to the manual border when MeshMapper has none. --- cmd/beacon/main.go | 38 +- config.yaml.example | 16 +- db/meshmapper.go | 37 ++ db/meshmapper_integration_test.go | 86 +++++ db/migrations/044_meshmapper_zones.sql | 15 + db/queries/queries.sql | 32 +- db/sqlc/mock/querier.go | 49 ++- db/sqlc/models.go | 11 + db/sqlc/querier.go | 14 +- db/sqlc/queries.sql.go | 114 +++++- internal/api/local_borders.go | 6 +- internal/api/local_borders_test.go | 6 +- internal/borders/borders_test.go | 25 ++ internal/borders/live.go | 28 ++ internal/cache/cache_test.go | 11 + internal/cache/reader.go | 5 + internal/config/config.go | 8 +- internal/config/foreign_borders.go | 55 ++- internal/config/foreign_borders_test.go | 40 ++- internal/config/meshmapper.go | 34 ++ internal/config/meshmapper_test.go | 34 ++ internal/ingest/foreign_nodes_test.go | 2 +- internal/ingest/ingest.go | 4 +- internal/meshmapper/scopes.go | 2 +- internal/meshmapper/zones.go | 443 ++++++++++++++++++++++++ internal/meshmapper/zones_test.go | 331 ++++++++++++++++++ 26 files changed, 1397 insertions(+), 49 deletions(-) create mode 100644 db/migrations/044_meshmapper_zones.sql create mode 100644 internal/borders/live.go create mode 100644 internal/meshmapper/zones.go create mode 100644 internal/meshmapper/zones_test.go diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index fc98fd9b..5017dffb 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -6,6 +6,7 @@ package main import ( "context" "encoding/hex" + "encoding/json" "fmt" "log/slog" "net/http" @@ -22,6 +23,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/api/router" "github.com/MeshCore-Beacon/beacon-server/internal/background" + "github.com/MeshCore-Beacon/beacon-server/internal/borders" "github.com/MeshCore-Beacon/beacon-server/internal/cache" "github.com/MeshCore-Beacon/beacon-server/internal/config" "github.com/MeshCore-Beacon/beacon-server/internal/hub" @@ -111,11 +113,16 @@ func main() { } resolved := config.Resolve(cfg) - localBorders, err := config.LoadLocalBorders(cfg) + borderFiles, err := config.LoadLocalBorders(cfg) if err != nil { slog.Error("invalid local border configuration", "component", "startup", "error", err) os.Exit(1) } + var localBorders *borders.Live + if cfg.Nodes.MarkForeign { + local, _ := config.BuildLocalBorders(borderFiles, nil) // checked by LoadLocalBorders + localBorders = borders.NewLive(local) + } slog.Info(fmt.Sprintf("config: loaded — %s", resolved), "component", "startup") @@ -201,6 +208,32 @@ func main() { } } + // Always restored, so disabling the import prunes it and file borders return. + zones := meshmapper.NewZones(cfg.MeshMapper.Zones, cfg.RegionIATAs(), store) + if cr, ok := reader.(*cache.CachedReader); ok { + zones.OnChange(cr.InvalidateIATABorder) + } + if localBorders != nil { + zones.OnUpdate(func(imported map[string]json.RawMessage) { + local, err := config.BuildLocalBorders(borderFiles, imported) + if err != nil { + slog.Error("MeshMapper boundaries rejected for foreign marking", "component", "meshmapper.zones", "error", err) + return + } + localBorders.Store(local) + }) + } + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + err = zones.Restore(restoreCtx) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper boundaries", "component", "startup", "error", err) + os.Exit(1) + } + if localBorders != nil && !localBorders.Ready() { + slog.Warn("nodes.mark_foreign has no boundaries yet; waiting for MeshMapper", "component", "startup") + } + // ── Build channel keystore ────────────────────────────────────────────── entries := make(map[string][]keystore.Entry) @@ -318,6 +351,9 @@ func main() { if scopeImporter != nil { tasks = append(tasks, background.Task{Name: "meshmapper.scopes", Interval: meshmapper.PollInterval, Run: scopeImporter.Refresh}) } + if cfg.MeshMapper.Zones.Enabled { + tasks = append(tasks, background.Task{Name: "meshmapper.zones", Interval: meshmapper.PollInterval, Run: zones.Refresh}) + } profiles := configureProfiling(ctx, pool) defer profiles.Stop() for i := range tasks { diff --git a/config.yaml.example b/config.yaml.example index 3e2a66fc..0c6eb6e2 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -47,6 +47,7 @@ iatas: lat: 49.1967 lng: -123.1815 # borderFile: borders/yvr.geojson # optional GeoJSON Feature (Polygon/MultiPolygon) + # # (meshmapper.zones overrides it when enabled) # # for the region border map; path is relative to # # this file's directory; validated at boot. YYJ: @@ -159,8 +160,9 @@ websocket: # Node staleness, deletion, and clock-drift thresholds. #nodes: # mark_foreign: false # opt in to possiblyForeign on repeater nodes -# # uses the union of configured iatas.*.borderFile polygons; -# # requires at least one valid border; no packet filtering +# # uses the union of configured iatas.*.borderFile polygons, +# # with meshmapper.zones boundaries replacing them; requires +# # a valid border file or meshmapper.zones; no packet filtering # stale_threshold: 24h # default: 24h # delete_after: 720h # default: 30 days # clock_drift_threshold: 5m # default: 5m @@ -228,6 +230,16 @@ cache: # refresh_interval: 1h # 5m-24h; one source checked per 15s tick # sources: # YOW: https://yow.meshmapper.net/get_scopes.php +# +# Optional MeshMapper region boundaries (Zones API). Every IATA in a configured +# region is looked up in its country's zone list, then its get_geojson.php outline +# is imported. An imported boundary overrides that IATA's borderFile on the border +# map and for nodes.mark_foreign. A missing or failed boundary keeps the last good +# import, or the borderFile when there is none. Disabling, or removing an IATA from +# every region, drops its import on restart. Logged under component=meshmapper.zones. +# zones: +# enabled: false +# refresh_interval: 24h # 1h-168h; one request per 15s tick, ETag-conditional # Background task intervals. # Shorter intervals are useful during initial deployment to confirm data is diff --git a/db/meshmapper.go b/db/meshmapper.go index 3ddbb28b..0e2bc289 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -47,3 +47,40 @@ func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache } return s.q.SaveScopeCatalogue(ctx, params) } + +func (s *Store) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { + if keep == nil { + keep = []string{} + } + return s.q.PruneZoneBoundaries(ctx, keep) +} + +func (s *Store) ListZoneBoundaries(ctx context.Context) ([]meshmapper.Boundary, error) { + rows, err := s.q.ListZoneBoundaries(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.Boundary, 0, len(rows)) + for _, row := range rows { + b := meshmapper.Boundary{IATA: row.Iata, URL: row.Url, Feature: row.Feature, CheckedAt: row.CheckedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError} + if row.Etag != nil { + b.ETag = *row.Etag + } + out = append(out, b) + } + return out, nil +} + +func (s *Store) SaveZoneBoundary(ctx context.Context, b meshmapper.Boundary) error { + params := sqlc.SaveZoneBoundaryParams{ + Iata: b.IATA, Url: b.URL, Feature: b.Feature, LastError: b.LastError, + CheckedAt: pgtype.Timestamptz{Time: b.CheckedAt, Valid: !b.CheckedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: b.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: b.NextAttempt, Valid: true}, + } + if !b.CheckedAt.IsZero() { + params.Etag = &b.ETag + } + return s.q.SaveZoneBoundary(ctx, params) +} diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go index 06d71096..7d8c2a11 100644 --- a/db/meshmapper_integration_test.go +++ b/db/meshmapper_integration_test.go @@ -6,6 +6,7 @@ package db import ( "context" "os" + "strings" "testing" "time" @@ -137,3 +138,88 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { t.Fatal("partially committed catalogue", count, err) } } + +func TestMeshMapperZoneBoundariesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"iata_codes", "meshmapper_zone_boundaries"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + manual := []byte(`{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,1],[0,0]]]},"properties":{"source":"file"}}`) + imported := []byte(`{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[0,0],[2,0],[2,2],[0,2],[0,0]]]},"properties":{"code":"YOW"}}`) + if err := store.UpsertIATABorder(ctx, "YOW", manual); err != nil { + t.Fatal(err) + } + if err := store.UpsertIATA(ctx, "YYZ"); err != nil { + t.Fatal(err) + } + border := func(iata string) string { + t.Helper() + got, err := store.GetIATABorder(ctx, iata) + if err != nil { + t.Fatal(err) + } + return string(got) + } + if !strings.Contains(border("YOW"), `"file"`) || border("YYZ") != "" { + t.Fatal("manual border not served without an import") + } + now := time.Now().UTC().Truncate(time.Second) + good := meshmapper.Boundary{IATA: "YOW", URL: "https://yow.meshmapper.net/", Feature: imported, ETag: `"v1"`, CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)} + if err := store.SaveZoneBoundary(ctx, good); err != nil { + t.Fatal(err) + } + if !strings.Contains(border("YOW"), `"YOW"`) { + t.Fatal("imported boundary did not override the manual border") + } + if err := store.UpsertIATABorder(ctx, "YOW", manual); err != nil { + t.Fatal(err) + } + failed := meshmapper.Boundary{IATA: "YOW", URL: good.URL, AttemptedAt: now.Add(time.Hour), NextAttempt: now.Add(2 * time.Hour), LastError: "HTTP 503"} + if err := store.SaveZoneBoundary(ctx, failed); err != nil { + t.Fatal(err) + } + rows, err := store.ListZoneBoundaries(ctx) + if err != nil || len(rows) != 1 || !strings.Contains(string(rows[0].Feature), `"YOW"`) || rows[0].ETag != `"v1"` || + !rows[0].CheckedAt.Equal(now) || rows[0].LastError != "HTTP 503" || !rows[0].NextAttempt.Equal(failed.NextAttempt) { + t.Fatal("failure or reseed lost the last good boundary", rows, err) + } + if !strings.Contains(border("YOW"), `"YOW"`) { + t.Fatal("reseeding the manual border undid the override") + } + if pruned, err := store.PruneZoneBoundaries(ctx, []string{"YOW"}); err != nil || len(pruned) != 0 { + t.Fatal("configured import pruned", pruned, err) + } + if pruned, err := store.PruneZoneBoundaries(ctx, nil); err != nil || len(pruned) != 1 || pruned[0] != "YOW" { + t.Fatal(pruned, err) + } + if !strings.Contains(border("YOW"), `"file"`) { + t.Fatal("manual border did not return after pruning") + } +} diff --git a/db/migrations/044_meshmapper_zones.sql b/db/migrations/044_meshmapper_zones.sql new file mode 100644 index 00000000..0103225f --- /dev/null +++ b/db/migrations/044_meshmapper_zones.sql @@ -0,0 +1,15 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Imported MeshMapper region boundaries. Kept apart from iata_codes.border so +-- startup seeding of manual borderFile borders never undoes the override. +CREATE TABLE IF NOT EXISTS meshmapper_zone_boundaries ( + iata TEXT PRIMARY KEY, + url TEXT NOT NULL, + feature JSONB, + etag TEXT, + checked_at TIMESTAMPTZ, + attempted_at TIMESTAMPTZ NOT NULL, + next_attempt TIMESTAMPTZ NOT NULL, + last_error TEXT NOT NULL DEFAULT '' +); diff --git a/db/queries/queries.sql b/db/queries/queries.sql index c206a2f0..941b340f 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -25,10 +25,13 @@ ON CONFLICT (iata) DO UPDATE SET approx_lng = EXCLUDED.approx_lng; -- name: GetIATABorder :one --- border is NULL when the IATA exists but has no border configured; a --- missing row (unknown IATA) is sql.ErrNoRows, same not-found distinction --- GetIATA already makes. -SELECT border FROM iata_codes WHERE iata = $1; +-- An imported MeshMapper boundary overrides the configured one. border is NULL +-- when neither exists; a missing row (unknown IATA) is sql.ErrNoRows, same +-- not-found distinction GetIATA already makes. +SELECT COALESCE(z.feature, i.border)::jsonb AS border +FROM iata_codes i +LEFT JOIN meshmapper_zone_boundaries z ON z.iata = i.iata +WHERE i.iata = $1; -- name: UpsertIATABorder :exec -- Written by the config-file-driven seeder (internal/config/seed.go), not a @@ -1627,3 +1630,24 @@ ON CONFLICT (iata, url) DO UPDATE SET attempted_at = EXCLUDED.attempted_at, next_attempt = EXCLUDED.next_attempt, last_error = EXCLUDED.last_error; + +-- name: ListZoneBoundaries :many +SELECT * FROM meshmapper_zone_boundaries ORDER BY iata; + +-- name: SaveZoneBoundary :exec +-- NULL feature/etag/checked_at retain the last good boundary after an error or 304. +INSERT INTO meshmapper_zone_boundaries (iata, url, feature, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(feature)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata) DO UPDATE SET + url = EXCLUDED.url, + feature = COALESCE(EXCLUDED.feature, meshmapper_zone_boundaries.feature), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_boundaries.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_zone_boundaries.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; + +-- name: PruneZoneBoundaries :many +-- Drops imports for IATAs no longer configured, so their manual border returns. +DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY(@keep::text[])) RETURNING iata; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 82d16603..86beeece 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -11,6 +11,7 @@ package mockdb import ( context "context" + json "encoding/json" reflect "reflect" db "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -265,10 +266,10 @@ func (mr *MockQuerierMockRecorder) GetIATA(ctx, iata any) *gomock.Call { } // GetIATABorder mocks base method. -func (m *MockQuerier) GetIATABorder(ctx context.Context, iata string) ([]byte, error) { +func (m *MockQuerier) GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetIATABorder", ctx, iata) - ret0, _ := ret[0].([]byte) + ret0, _ := ret[0].(json.RawMessage) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -1313,6 +1314,36 @@ func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPackets(ctx any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPackets", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPackets), ctx) } +// ListZoneBoundaries mocks base method. +func (m *MockQuerier) ListZoneBoundaries(ctx context.Context) ([]db.MeshmapperZoneBoundary, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListZoneBoundaries", ctx) + ret0, _ := ret[0].([]db.MeshmapperZoneBoundary) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListZoneBoundaries indicates an expected call of ListZoneBoundaries. +func (mr *MockQuerierMockRecorder) ListZoneBoundaries(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneBoundaries", reflect.TypeOf((*MockQuerier)(nil).ListZoneBoundaries), ctx) +} + +// PruneZoneBoundaries mocks base method. +func (m *MockQuerier) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PruneZoneBoundaries", ctx, keep) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PruneZoneBoundaries indicates an expected call of PruneZoneBoundaries. +func (mr *MockQuerierMockRecorder) PruneZoneBoundaries(ctx, keep any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneZoneBoundaries", reflect.TypeOf((*MockQuerier)(nil).PruneZoneBoundaries), ctx, keep) +} + // ReconfirmNeighbors mocks base method. func (m *MockQuerier) ReconfirmNeighbors(ctx context.Context) error { m.ctrl.T.Helper() @@ -1586,6 +1617,20 @@ func (mr *MockQuerierMockRecorder) SaveScopeCatalogue(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveScopeCatalogue), ctx, arg) } +// SaveZoneBoundary mocks base method. +func (m *MockQuerier) SaveZoneBoundary(ctx context.Context, arg db.SaveZoneBoundaryParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveZoneBoundary", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveZoneBoundary indicates an expected call of SaveZoneBoundary. +func (mr *MockQuerierMockRecorder) SaveZoneBoundary(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneBoundary", reflect.TypeOf((*MockQuerier)(nil).SaveZoneBoundary), ctx, arg) +} + // SearchKnownRoutes mocks base method. func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 86fe98ea..bf81a1d3 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -245,6 +245,17 @@ type MeshmapperScopeCatalogue struct { LastError string `json:"last_error"` } +type MeshmapperZoneBoundary struct { + Iata string `json:"iata"` + Url string `json:"url"` + Feature []byte `json:"feature"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + type MvHourlyIataStat struct { Iata string `json:"iata"` Hour pgtype.Timestamptz `json:"hour"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 6d699a94..364aea68 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -6,6 +6,7 @@ package db import ( "context" + "encoding/json" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -46,10 +47,10 @@ type Querier interface { GetCrossIATANeighbors(ctx context.Context, arg GetCrossIATANeighborsParams) ([]GetCrossIATANeighborsRow, error) GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]MvHourlyIataStat, error) GetIATA(ctx context.Context, iata string) (IataCode, error) - // border is NULL when the IATA exists but has no border configured; a - // missing row (unknown IATA) is sql.ErrNoRows, same not-found distinction - // GetIATA already makes. - GetIATABorder(ctx context.Context, iata string) ([]byte, error) + // An imported MeshMapper boundary overrides the configured one. border is NULL + // when neither exists; a missing row (unknown IATA) is sql.ErrNoRows, same + // not-found distinction GetIATA already makes. + GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error) GetNodeByID(ctx context.Context, id uuid.UUID) (GetNodeByIDRow, error) GetNodeByPubkey(ctx context.Context, publicKey []byte) (uuid.UUID, error) @@ -220,6 +221,9 @@ type Querier interface { // to the config after they'd already been ingested -- see // internal/ingest.BackfillChannelMessages. ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error) + ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) + // Drops imports for IATAs no longer configured, so their manual border returns. + PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) // Delete node_neighbors where the neighbor has departed from node_short_ids // for that IATA, or where its prefix_4 is now ambiguous. ReconfirmNeighbors(ctx context.Context) error @@ -260,6 +264,8 @@ type Querier interface { // Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data // after an error or 304. Imported names never replace existing manual metadata. SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error + // NULL feature/etag/checked_at retain the last good boundary after an error or 304. + SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryParams) error // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 0a5aa362..ce7bb12d 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -7,6 +7,7 @@ package db import ( "context" + "encoding/json" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -390,15 +391,18 @@ func (q *Queries) GetIATA(ctx context.Context, iata string) (IataCode, error) { } const getIATABorder = `-- name: GetIATABorder :one -SELECT border FROM iata_codes WHERE iata = $1 +SELECT COALESCE(z.feature, i.border)::jsonb AS border +FROM iata_codes i +LEFT JOIN meshmapper_zone_boundaries z ON z.iata = i.iata +WHERE i.iata = $1 ` -// border is NULL when the IATA exists but has no border configured; a -// missing row (unknown IATA) is sql.ErrNoRows, same not-found distinction -// GetIATA already makes. -func (q *Queries) GetIATABorder(ctx context.Context, iata string) ([]byte, error) { +// An imported MeshMapper boundary overrides the configured one. border is NULL +// when neither exists; a missing row (unknown IATA) is sql.ErrNoRows, same +// not-found distinction GetIATA already makes. +func (q *Queries) GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) { row := q.db.QueryRow(ctx, getIATABorder, iata) - var border []byte + var border json.RawMessage err := row.Scan(&border) return border, err } @@ -3975,6 +3979,64 @@ func (q *Queries) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUn return items, nil } +const listZoneBoundaries = `-- name: ListZoneBoundaries :many +SELECT iata, url, feature, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_boundaries ORDER BY iata +` + +func (q *Queries) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) { + rows, err := q.db.Query(ctx, listZoneBoundaries) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperZoneBoundary{} + for rows.Next() { + var i MeshmapperZoneBoundary + if err := rows.Scan( + &i.Iata, + &i.Url, + &i.Feature, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const pruneZoneBoundaries = `-- name: PruneZoneBoundaries :many +DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY($1::text[])) RETURNING iata +` + +// Drops imports for IATAs no longer configured, so their manual border returns. +func (q *Queries) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { + rows, err := q.db.Query(ctx, pruneZoneBoundaries, keep) + if err != nil { + return nil, err + } + defer rows.Close() + items := []string{} + for rows.Next() { + var iata string + if err := rows.Scan(&iata); err != nil { + return nil, err + } + items = append(items, iata) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const reconfirmNeighbors = `-- name: ReconfirmNeighbors :exec DELETE FROM node_neighbors nn WHERE NOT EXISTS ( @@ -4522,6 +4584,46 @@ func (q *Queries) SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogue return err } +const saveZoneBoundary = `-- name: SaveZoneBoundary :exec +INSERT INTO meshmapper_zone_boundaries (iata, url, feature, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata) DO UPDATE SET + url = EXCLUDED.url, + feature = COALESCE(EXCLUDED.feature, meshmapper_zone_boundaries.feature), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_boundaries.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_zone_boundaries.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveZoneBoundaryParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Feature []byte `json:"feature"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +// NULL feature/etag/checked_at retain the last good boundary after an error or 304. +func (q *Queries) SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryParams) error { + _, err := q.db.Exec(ctx, saveZoneBoundary, + arg.Iata, + arg.Url, + arg.Feature, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + ) + return err +} + const searchKnownRoutes = `-- name: SearchKnownRoutes :many SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes diff --git a/internal/api/local_borders.go b/internal/api/local_borders.go index 278f7f0a..65e44a0d 100644 --- a/internal/api/local_borders.go +++ b/internal/api/local_borders.go @@ -12,9 +12,9 @@ import ( ) // WithLocalBorders annotates node responses after cache reads. Historical rows -// use the current startup configuration without extra SQL or cached-policy drift. +// use the current boundaries without extra SQL or cached-policy drift. // Cached values are copied; the decorator never changes its inner reader's data. -func WithLocalBorders(reader Reader, local *borders.Local) Reader { +func WithLocalBorders(reader Reader, local *borders.Live) Reader { if local == nil { return reader } @@ -23,7 +23,7 @@ func WithLocalBorders(reader Reader, local *borders.Local) Reader { type localBorderReader struct { Reader - local *borders.Local + local *borders.Live } func (r *localBorderReader) ListNodes(ctx context.Context, nodeType int16, iatas []string, paths, traces *bool, pubkey []byte, prefix, name, scope string, cursor int64, limit int32, neighbors bool) (Page[NodeSummary], error) { diff --git a/internal/api/local_borders_test.go b/internal/api/local_borders_test.go index 033f37da..d7f0b013 100644 --- a/internal/api/local_borders_test.go +++ b/internal/api/local_borders_test.go @@ -52,8 +52,8 @@ func TestLocalBordersAfterCache(t *testing.T) { client := cache.NewClient(mr.Addr(), "", 0) t.Cleanup(func() { client.Close() }) cached := cache.NewCachedReader(base, client, cache.CacheTTLs{Nodes: time.Hour}) - first := api.WithLocalBorders(cached, local) - second := api.WithLocalBorders(cached, other) + first := api.WithLocalBorders(cached, borders.NewLive(local)) + second := api.WithLocalBorders(cached, borders.NewLive(other)) if api.WithLocalBorders(cached, nil) != cached { t.Fatal("disabled decorator changed reader") } @@ -110,7 +110,7 @@ func TestLocalBordersPreserveReaderErrors(t *testing.T) { } want := errors.New("reader unavailable") base := &borderReader{err: want} - reader := api.WithLocalBorders(base, local) + reader := api.WithLocalBorders(base, borders.NewLive(local)) if _, err := reader.GetNode(context.Background(), uuid.New()); !errors.Is(err, want) { t.Fatal(err) } diff --git a/internal/borders/borders_test.go b/internal/borders/borders_test.go index c5d4fdfc..687f84de 100644 --- a/internal/borders/borders_test.go +++ b/internal/borders/borders_test.go @@ -114,3 +114,28 @@ func BenchmarkPossiblyForeign(b *testing.B) { }) } } + +func TestLiveSwap(t *testing.T) { + var disabled *Live + if disabled.PossiblyForeign(2, new(15.0), new(15.0)) != nil { + t.Fatal("nil live classified") + } + live := NewLive(nil) + if disabled.Ready() || live.Ready() { + t.Fatal("ready without boundaries") + } + if live.PossiblyForeign(2, new(15.0), new(15.0)) != nil { + t.Fatal("empty live classified") + } + local, err := New([]orb.Polygon{{{{10, 10}, {20, 10}, {20, 20}, {10, 20}, {10, 10}}}}) + if err != nil { + t.Fatal(err) + } + live.Store(local) + if !live.Ready() { + t.Fatal("not ready after store") + } + if got := live.PossiblyForeign(2, new(15.0), new(15.0)); got == nil || *got { + t.Fatal("stored classifier not used", got) + } +} diff --git a/internal/borders/live.go b/internal/borders/live.go new file mode 100644 index 00000000..ae8294ed --- /dev/null +++ b/internal/borders/live.go @@ -0,0 +1,28 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package borders + +import "sync/atomic" + +// Live lets boundary refreshes replace the classifier under concurrent readers. +// A nil Live disables classification; an empty one reports unknown until stored. +type Live struct{ current atomic.Pointer[Local] } + +func NewLive(local *Local) *Live { + live := &Live{} + live.current.Store(local) + return live +} + +func (l *Live) Store(local *Local) { l.current.Store(local) } + +// Ready reports whether any boundary is loaded. +func (l *Live) Ready() bool { return l != nil && l.current.Load() != nil } + +func (l *Live) PossiblyForeign(role int16, lat, lng *float64) *bool { + if l == nil { + return nil + } + return l.current.Load().PossiblyForeign(role, lat, lng) +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 22515312..147b9da2 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -396,6 +396,17 @@ func TestInvalidateScopeNamesClearsDerivedKeys(t *testing.T) { } } +func TestInvalidateIATABorder(t *testing.T) { + c, mr := newTestClient(t) + cr := &CachedReader{inner: &stubReader{}, c: c, ttl: CacheTTLs{}} + mr.Set(keyIATABorderPrefix+"YOW", "x") + mr.Set(keyIATABorderPrefix+"YYZ", "keep") + cr.InvalidateIATABorder(context.Background(), "YOW") + if got := mr.Keys(); len(got) != 1 || got[0] != keyIATABorderPrefix+"YYZ" { + t.Fatalf("keys left: %v", got) + } +} + func TestCachedReader_InvalidateNode(t *testing.T) { c, mr := newTestClient(t) nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000001") diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 2dd3dda2..2273447a 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -87,6 +87,11 @@ func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { cr.c.delPrefix(ctx, keyScopeByNamePrefix) } +// InvalidateIATABorder makes an imported or removed MeshMapper boundary visible. +func (cr *CachedReader) InvalidateIATABorder(ctx context.Context, iata string) { + cr.c.del(ctx, keyIATABorderPrefix+iata) +} + // InvalidateNode removes the cached entries for a node by UUID. // Should be called from the ingest path after a node upsert. func (cr *CachedReader) InvalidateNode(ctx context.Context, nodeID uuid.UUID) { diff --git a/internal/config/config.go b/internal/config/config.go index b8b8fff8..85df4692 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -271,8 +271,9 @@ type RoutesConfig struct { // NodesConfig controls node-derived signal thresholds. type NodesConfig struct { - // MarkForeign annotates repeaters outside the union of configured IATA border - // files. Disabled by default; enabling it requires at least one usable border. + // MarkForeign annotates repeaters outside the union of IATA borders, with + // imported MeshMapper boundaries replacing border files. Disabled by default; + // enabling it requires a border file or meshmapper.zones. MarkForeign bool `yaml:"mark_foreign"` // ClockDriftThreshold is the |device clock - server clock| magnitude, measured from a // repeater/room server's ADVERT timestamp, above which the node API reports @@ -412,6 +413,9 @@ func Load(path string) (*Config, error) { if err := cfg.validateMeshMapper(); err != nil { return nil, err } + if cfg.Nodes.MarkForeign && !cfg.MeshMapper.Zones.Enabled && !cfg.hasBorderFile() { + return nil, fmt.Errorf("nodes.mark_foreign requires an iatas.*.borderFile or meshmapper.zones") + } for iata, details := range cfg.IATAs { if details.BorderFile != "" && !filepath.IsAbs(details.BorderFile) { details.BorderFile = filepath.Join(configDir, details.BorderFile) diff --git a/internal/config/foreign_borders.go b/internal/config/foreign_borders.go index a9ca62e1..eb5c410c 100644 --- a/internal/config/foreign_borders.go +++ b/internal/config/foreign_borders.go @@ -4,31 +4,38 @@ package config import ( + "encoding/json" "fmt" + "maps" "os" - "sort" + "slices" "github.com/MeshCore-Beacon/beacon-server/internal/borders" "github.com/paulmach/orb" "github.com/paulmach/orb/geojson" ) -// LoadLocalBorders builds one startup-only classifier from configured border -// files. IATA airport coordinates and auto-discovered IATAs are not boundaries. -func LoadLocalBorders(cfg *Config) (*borders.Local, error) { +func (c *Config) hasBorderFile() bool { + for _, details := range c.IATAs { + if details.BorderFile != "" { + return true + } + } + return false +} + +// LoadLocalBorders reads the validated border files used for foreign marking, +// keyed by IATA. IATA airport coordinates are not boundaries. Nil when disabled. +func LoadLocalBorders(cfg *Config) (map[string]json.RawMessage, error) { if !cfg.Nodes.MarkForeign { return nil, nil } - var codes []string + files := map[string]json.RawMessage{} for code, details := range cfg.IATAs { - if details.BorderFile != "" { - codes = append(codes, code) + if details.BorderFile == "" { + continue } - } - sort.Strings(codes) - var polygons []orb.Polygon - for _, code := range codes { - raw, err := os.ReadFile(cfg.IATAs[code].BorderFile) + raw, err := os.ReadFile(details.BorderFile) if err != nil { return nil, fmt.Errorf("nodes.mark_foreign: reading border for %s: %w", code, err) } @@ -36,7 +43,29 @@ func LoadLocalBorders(cfg *Config) (*borders.Local, error) { if err != nil { return nil, fmt.Errorf("nodes.mark_foreign: invalid border for %s: %w", code, err) } - feature, err := geojson.UnmarshalFeature(validated) + files[code] = validated + } + // Fail at startup rather than on the first rebuild. + if _, err := BuildLocalBorders(files, nil); err != nil { + return nil, err + } + return files, nil +} + +// BuildLocalBorders unions validated borders; an imported boundary replaces +// the same IATA's file. Nil when there are none. +func BuildLocalBorders(files, imported map[string]json.RawMessage) (*borders.Local, error) { + merged := maps.Clone(files) + if merged == nil { + merged = map[string]json.RawMessage{} + } + maps.Copy(merged, imported) + if len(merged) == 0 { + return nil, nil + } + var polygons []orb.Polygon + for _, code := range slices.Sorted(maps.Keys(merged)) { + feature, err := geojson.UnmarshalFeature(merged[code]) if err != nil { return nil, fmt.Errorf("nodes.mark_foreign: decoding border for %s: %w", code, err) } diff --git a/internal/config/foreign_borders_test.go b/internal/config/foreign_borders_test.go index d158f911..e39e6b77 100644 --- a/internal/config/foreign_borders_test.go +++ b/internal/config/foreign_borders_test.go @@ -4,6 +4,7 @@ package config import ( + "encoding/json" "os" "path/filepath" "strings" @@ -22,16 +23,27 @@ func TestLoadLocalBorders(t *testing.T) { if got, err := LoadLocalBorders(&Config{}); err != nil || got != nil { t.Fatalf("disabled: %v %v", got, err) } - if _, err := LoadLocalBorders(&Config{Nodes: NodesConfig{MarkForeign: true}}); err == nil { - t.Fatal("enabled with no border") - } dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + for _, tc := range []struct { + text string + bad bool + }{ + {"nodes:\n mark_foreign: true\n", true}, + {"nodes:\n mark_foreign: true\nregions:\n - slug: a\n iatas: [AAA]\nmeshmapper:\n zones:\n enabled: true\n", false}, + } { + if err := os.WriteFile(path, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); (err != nil) != tc.bad { + t.Fatalf("%q: %v", tc.text, err) + } + } border := filepath.Join(dir, "region.json") raw := `{"type":"Feature","geometry":{"type":"MultiPolygon","coordinates":[[[[10,10],[20,10],[20,20],[10,20],[10,10]]],[[[30,10],[40,10],[40,20],[30,20],[30,10]]]]}}` if err := os.WriteFile(border, []byte(raw), 0600); err != nil { t.Fatal(err) } - path := filepath.Join(dir, "config.yaml") if err := os.WriteFile(path, []byte("nodes:\n mark_foreign: true\niatas:\n AAA:\n borderFile: region.json\n BBB:\n name: Airport without an operating border\n"), 0600); err != nil { t.Fatal(err) } @@ -39,13 +51,31 @@ func TestLoadLocalBorders(t *testing.T) { if err != nil { t.Fatal(err) } - local, err := LoadLocalBorders(cfg) + files, err := LoadLocalBorders(cfg) + if err != nil || len(files) != 1 { + t.Fatal(files, err) + } + local, err := BuildLocalBorders(files, nil) if err != nil { t.Fatal(err) } if *local.PossiblyForeign(2, new(15.0), new(15.0)) || *local.PossiblyForeign(2, new(15.0), new(35.0)) || !*local.PossiblyForeign(2, new(25.0), new(25.0)) { t.Fatal("polygon union not used") } + imported := map[string]json.RawMessage{"AAA": json.RawMessage(`{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[50,50],[60,50],[60,60],[50,60],[50,50]]]}}`)} + replaced, err := BuildLocalBorders(files, imported) + if err != nil { + t.Fatal(err) + } + if !*replaced.PossiblyForeign(2, new(15.0), new(15.0)) || *replaced.PossiblyForeign(2, new(55.0), new(55.0)) { + t.Fatal("imported boundary did not replace the file border") + } + if len(files) != 1 || string(files["AAA"]) == string(imported["AAA"]) { + t.Fatal("build changed its inputs") + } + if empty, err := BuildLocalBorders(nil, nil); err != nil || empty != nil { + t.Fatal(empty, err) + } for _, bad := range []string{`not json`, `{"type":"Feature","geometry":null}`, `{"type":"Feature","geometry":{"type":"Polygon","coordinates":[[[179,10],[-179,10],[-179,20],[179,20],[179,10]]]}}`} { if err := os.WriteFile(border, []byte(bad), 0600); err != nil { t.Fatal(err) diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go index 5f367747..31bec785 100644 --- a/internal/config/meshmapper.go +++ b/internal/config/meshmapper.go @@ -7,6 +7,7 @@ import ( "fmt" "net/url" "regexp" + "slices" "time" ) @@ -15,6 +16,30 @@ const MaxScopeSources = 16 type MeshMapperConfig struct { Scopes MeshMapperScopesConfig `yaml:"scopes"` + Zones MeshMapperZonesConfig `yaml:"zones"` +} + +// MeshMapperZonesConfig imports each region IATA's published boundary, overriding borderFile. +type MeshMapperZonesConfig struct { + Enabled bool `yaml:"enabled"` + RefreshInterval duration `yaml:"refresh_interval"` +} + +func (c MeshMapperZonesConfig) Interval() time.Duration { + if c.RefreshInterval.Duration == 0 { + return 24 * time.Hour + } + return c.RefreshInterval.Duration +} + +// RegionIATAs returns the sorted, unique IATAs of every configured region. +func (c *Config) RegionIATAs() []string { + var iatas []string + for _, region := range c.Regions { + iatas = append(iatas, region.IATAs...) + } + slices.Sort(iatas) + return slices.Compact(iatas) } // MeshMapperScopesConfig augments, but never replaces, the manual scopes list. @@ -34,6 +59,15 @@ func (c MeshMapperScopesConfig) Interval() time.Duration { var meshMapperHost = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) func (c *Config) validateMeshMapper() error { + if z := c.MeshMapper.Zones; z.Enabled { + // The Zones API asks clients not to poll more than once an hour. + if z.Interval() < time.Hour || z.Interval() > 7*24*time.Hour { + return fmt.Errorf("meshmapper.zones.refresh_interval must be between 1h and 168h") + } + if len(c.RegionIATAs()) == 0 { + return fmt.Errorf("meshmapper.zones needs at least one configured region IATA") + } + } s := c.MeshMapper.Scopes if !s.Enabled { return nil diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go index 56a56345..d4e343ba 100644 --- a/internal/config/meshmapper_test.go +++ b/internal/config/meshmapper_test.go @@ -47,3 +47,37 @@ func TestMeshMapperConfig(t *testing.T) { t.Fatal("empty sources accepted") } } + +func TestMeshMapperZonesConfig(t *testing.T) { + valid := "regions:\n - slug: ottawa\n iatas: [YOW, YOW]\n - slug: toronto\n iatas: [YYZ]\nmeshmapper:\n zones:\n enabled: true\n" + for _, tc := range []struct { + name, text string + bad bool + interval time.Duration + }{ + {"default", valid, false, 24 * time.Hour}, + {"hourly", valid + " refresh_interval: 1h\n", false, time.Hour}, + {"too often", valid + " refresh_interval: 59m\n", true, 0}, + {"too rare", valid + " refresh_interval: 169h\n", true, 0}, + {"no regions", "meshmapper:\n zones:\n enabled: true\n", true, 0}, + {"disabled", "meshmapper:\n zones:\n enabled: false\n", false, 24 * time.Hour}, + } { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(p) + if (err != nil) != tc.bad { + t.Fatalf("error=%v", err) + } + if err == nil && cfg.MeshMapper.Zones.Interval() != tc.interval { + t.Fatal(cfg.MeshMapper.Zones.Interval()) + } + }) + } + cfg := &Config{Regions: []RegionConfig{{IATAs: []string{"YYZ", "YOW"}}, {IATAs: []string{"YOW"}}}} + if got := strings.Join(cfg.RegionIATAs(), ","); got != "YOW,YYZ" { + t.Fatal(got) + } +} diff --git a/internal/ingest/foreign_nodes_test.go b/internal/ingest/foreign_nodes_test.go index 20a3f78a..85f4feef 100644 --- a/internal/ingest/foreign_nodes_test.go +++ b/internal/ingest/foreign_nodes_test.go @@ -48,7 +48,7 @@ func TestForeignNodeUpdates(t *testing.T) { t.Run(tc.name, func(t *testing.T) { w.cfg.LocalBorders = nil if tc.enabled { - w.cfg.LocalBorders = local + w.cfg.LocalBorders = borders.NewLive(local) } data := []byte{tc.role} if tc.position { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index dcbe2bb8..523a1fb9 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -58,8 +58,8 @@ import ( // Config holds the connection parameters for one broker. type Config struct { - // LocalBorders is the same immutable classifier used by node API reads. - LocalBorders *borders.Local + // LocalBorders is the same live classifier used by node API reads. + LocalBorders *borders.Live // BrokerName is a short human-readable label ("mqtt1", "mqtt2") used in // log messages and stored in packet_observations.source_broker. BrokerName string diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index 5eb0d7bb..ac389a14 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -1,7 +1,7 @@ // Copyright 2026 Beacon Contributors // SPDX-License-Identifier: AGPL-3.0-or-later -// Package meshmapper imports the published regional scope catalogue, off the ingest path. +// Package meshmapper imports published regional scope catalogues and boundaries, off the ingest path. package meshmapper import ( diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go new file mode 100644 index 00000000..2d866615 --- /dev/null +++ b/internal/meshmapper/zones.go @@ -0,0 +1,443 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" +) + +const ( + ZonesURL = "https://meshmapper.net/get_zones.php" + MaxZoneList = 1 << 20 + MaxBoundaryBody = 8 << 20 // outlines are never simplified + + zoneListFresh = time.Hour // the API's own cache lifetime + zoneFailureRetry = time.Hour // fair use: no more than one poll an hour +) + +var zoneSite = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) + +// Boundary is one IATA's imported MeshMapper outline and its fetch state. +type Boundary struct { + IATA, URL string + Feature json.RawMessage + ETag string + CheckedAt, AttemptedAt, NextAttempt time.Time + LastError string +} + +type ZoneStore interface { + PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) + ListZoneBoundaries(ctx context.Context) ([]Boundary, error) + SaveZoneBoundary(ctx context.Context, b Boundary) error +} + +type zoneRegion struct { + country string + b Boundary +} + +type zoneEntry struct { + url string + hasBoundary bool +} + +type zoneList struct { + zones map[string]zoneEntry + etag string + fetchedAt, nextAttempt time.Time +} + +// Zones is owned by one background task; listeners are wired before Restore. +type Zones struct { + store ZoneStore + enabled bool + iatas []string + interval time.Duration + client *http.Client + listURL string + boundsURL func(site string) (string, bool) + regions []*zoneRegion + lists map[string]*zoneList + retryAfter time.Time + onChange func(ctx context.Context, iata string) + onUpdate func(imported map[string]json.RawMessage) +} + +func NewZones(cfg config.MeshMapperZonesConfig, iatas []string, store ZoneStore) *Zones { + return &Zones{store: store, enabled: cfg.Enabled, iatas: iatas, interval: cfg.Interval(), listURL: ZonesURL, + boundsURL: boundaryEndpoint, lists: map[string]*zoneList{}, client: &http.Client{ + Timeout: 30 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} +} + +// OnChange receives each IATA whose served border changed, for cache invalidation. +func (z *Zones) OnChange(fn func(ctx context.Context, iata string)) { z.onChange = fn } + +// OnUpdate receives every imported boundary whenever the set changes. +func (z *Zones) OnUpdate(fn func(imported map[string]json.RawMessage)) { z.onUpdate = fn } + +// Restore prunes imports for unconfigured IATAs (all of them when disabled) and +// loads saved boundaries, without making HTTP requests. +func (z *Zones) Restore(ctx context.Context) error { + keep := []string{} + if z.enabled { + keep = z.iatas + } + pruned, err := z.store.PruneZoneBoundaries(ctx, keep) + if err != nil { + return fmt.Errorf("prune MeshMapper boundaries: %w", err) + } + for _, iata := range pruned { + slog.Info("MeshMapper boundary removed", "component", "meshmapper.zones", "iata", iata) + z.changed(ctx, iata) + } + if !z.enabled { + return nil + } + saved, err := z.store.ListZoneBoundaries(ctx) + if err != nil { + return fmt.Errorf("restore MeshMapper boundaries: %w", err) + } + byIATA := map[string]Boundary{} + for _, b := range saved { + byIATA[b.IATA] = b + } + for _, iata := range z.iatas { + country := iatadb.CountryFor(iata) + if country == "" { + slog.Warn("MeshMapper boundary skipped: IATA has no known country", "component", "meshmapper.zones", "iata", iata) + continue + } + b, ok := byIATA[iata] + if !ok { + b = Boundary{IATA: iata} + } + if b.LastError == "HTTP 429" && b.NextAttempt.After(z.retryAfter) { + z.retryAfter = b.NextAttempt + } + z.regions = append(z.regions, &zoneRegion{country: country, b: b}) + z.log(b, "restored") + } + z.publish() + return nil +} + +// Refresh makes at most one request. The scheduler serializes calls every 15s. +func (z *Zones) Refresh(ctx context.Context) (err error) { + parent := ctx + defer func() { + if parent.Err() != nil { + err = nil + } + }() + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + now := time.Now().UTC() + if now.Before(z.retryAfter) { + return nil + } + for _, r := range z.regions { + if now.Before(r.b.NextAttempt) { + continue + } + list := z.lists[r.country] + if list == nil || now.Sub(list.fetchedAt) >= zoneListFresh { + if list != nil && now.Before(list.nextAttempt) { + continue + } + return z.fetchList(ctx, r.country, now) + } + return z.refresh(ctx, r, list, now) + } + return nil +} + +func (z *Zones) fetchList(ctx context.Context, country string, now time.Time) error { + list := z.lists[country] + if list == nil { + list = &zoneList{} + z.lists[country] = list + } + endpoint := z.listURL + "?country=" + url.QueryEscape(country) + etag := "" + if list.zones != nil { + etag = list.etag + } + status, body, header, err := z.get(ctx, endpoint, etag, MaxZoneList) + if err != nil { + return err + } + problem := "" + switch status { + case http.StatusOK: + zones, decodeErr := decodeZones(body, country) + if decodeErr != nil { + problem = "invalid response" + } else { + list.zones, list.etag = zones, header.Get("ETag") + } + case http.StatusNotModified: + if list.zones == nil { + problem = "304 without cached list" + } + default: + problem = z.statusProblem(status, header, now, &list.nextAttempt) + } + if problem == "" { + list.fetchedAt = now + } else if list.nextAttempt.Before(now.Add(zoneFailureRetry)) { + list.nextAttempt = now.Add(zoneFailureRetry) + } + level := slog.LevelInfo + if problem != "" { + level = slog.LevelWarn + } + slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country, + "zones", len(list.zones), "last_error", problem) + return nil +} + +func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now time.Time) error { + update := Boundary{IATA: r.b.IATA, URL: r.b.URL, AttemptedAt: now, NextAttempt: now.Add(z.interval)} + var retryAt time.Time + entry, listed := list.zones[r.b.IATA] + endpoint, valid := "", false + if listed { + update.URL = entry.url + endpoint, valid = z.boundsURL(entry.url) + } + switch { + case !listed: + update.LastError = "not listed" + case !entry.hasBoundary: + update.LastError = "no boundary" + case !valid: + update.LastError = "invalid site URL" + default: + etag := "" + if r.b.Feature != nil { + etag = r.b.ETag + } + status, body, header, err := z.get(ctx, endpoint, etag, MaxBoundaryBody) + if err != nil { + return err + } + switch status { + case http.StatusOK: + feature, decodeErr := decodeBoundary(body, r.b.IATA) + switch { + case decodeErr != nil: + update.LastError = "invalid response" // includes truncated bodies; retry + case feature == nil: + update.LastError = "no boundary" + default: + update.Feature = feature + } + case http.StatusNotModified: + if r.b.Feature == nil { + update.LastError = "304 without cached boundary" + } + default: + update.LastError = z.statusProblem(status, header, now, &retryAt) + } + if update.LastError == "" { + etag := header.Get("ETag") + if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") { + update.LastError, update.Feature = "invalid ETag", nil + } else { + if status == http.StatusNotModified && etag == "" { + etag = r.b.ETag + } + update.ETag, update.CheckedAt = etag, now + } + } + if update.LastError != "" && update.LastError != "no boundary" { + update.NextAttempt = now.Add(min(z.interval, zoneFailureRetry)) + if retryAt.After(update.NextAttempt) { + update.NextAttempt = retryAt + } + } + } + if err := z.store.SaveZoneBoundary(ctx, update); err != nil { + r.b.NextAttempt = update.NextAttempt // avoid retrying every tick during a DB outage + return fmt.Errorf("persist MeshMapper boundary %s: %w", r.b.IATA, err) + } + changed := update.Feature != nil && !bytes.Equal(update.Feature, r.b.Feature) + if update.Feature != nil { + r.b.Feature = update.Feature + } + if !update.CheckedAt.IsZero() { + r.b.CheckedAt, r.b.ETag = update.CheckedAt, update.ETag + } + r.b.URL, r.b.AttemptedAt, r.b.NextAttempt, r.b.LastError = update.URL, now, update.NextAttempt, update.LastError + if changed { + z.changed(ctx, r.b.IATA) + z.publish() + } + z.log(r.b, "checked") + return nil +} + +// get returns transport errors only when ctx ended; other failures become status 0. +func (z *Zones) get(ctx context.Context, endpoint, etag string, limit int64) (int, []byte, http.Header, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return 0, nil, nil, err + } + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", "Beacon-MeshMapper-Zones/1") + if etag != "" { + request.Header.Set("If-None-Match", etag) + } + response, err := z.client.Do(request) + if err != nil { + if ctx.Err() != nil { + return 0, nil, nil, ctx.Err() + } + return 0, nil, http.Header{}, nil + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return response.StatusCode, nil, response.Header, nil + } + body, err := io.ReadAll(io.LimitReader(response.Body, limit+1)) + if err != nil || int64(len(body)) > limit { + return response.StatusCode, nil, response.Header, nil + } + return response.StatusCode, body, response.Header, nil +} + +// statusProblem describes a failed response and honours Retry-After on 429/503. +func (z *Zones) statusProblem(status int, header http.Header, now time.Time, retryAt *time.Time) string { + if status == 0 { + return "request failed" // don't persist untrusted error text + } + if status == http.StatusTooManyRequests || status == http.StatusServiceUnavailable { + if seconds, err := strconv.ParseInt(header.Get("Retry-After"), 10, 64); err == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) { + *retryAt = now.Add(time.Duration(seconds) * time.Second) + } + // The rate limit is shared by both endpoints, so pause every request. + if status == http.StatusTooManyRequests { + z.retryAfter = *retryAt + if retryAt.IsZero() { + z.retryAfter = now.Add(time.Minute) + } + } + } + return fmt.Sprintf("HTTP %d", status) +} + +func (z *Zones) changed(ctx context.Context, iata string) { + if z.onChange != nil { + z.onChange(ctx, iata) + } +} + +func (z *Zones) publish() { + if z.onUpdate == nil { + return + } + imported := map[string]json.RawMessage{} + for _, r := range z.regions { + if r.b.Feature != nil { + imported[r.b.IATA] = r.b.Feature + } + } + z.onUpdate(imported) +} + +func (z *Zones) log(b Boundary, action string) { + level := slog.LevelInfo + if b.LastError != "" && b.LastError != "no boundary" && b.LastError != "not listed" { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, "MeshMapper boundary "+action, "component", "meshmapper.zones", "iata", b.IATA, + "source", b.URL, "imported", b.Feature != nil, "checked_at", b.CheckedAt, "next_attempt", b.NextAttempt, "last_error", b.LastError) +} + +// boundaryEndpoint accepts only a published region site root. +func boundaryEndpoint(site string) (string, bool) { + u, err := url.Parse(site) + if err != nil || u.Scheme != "https" || u.User != nil || !zoneSite.MatchString(u.Host) || + (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return "", false + } + return "https://" + u.Host + "/get_geojson.php", true +} + +func decodeZones(body []byte, country string) (map[string]zoneEntry, error) { + var document struct { + Country string `json:"country"` + Zones *[]struct { + Code string `json:"code"` + URL string `json:"url"` + HasBoundary bool `json:"has_boundary"` + } `json:"zones"` + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, err + } + if !strings.EqualFold(document.Country, country) || document.Zones == nil { + return nil, fmt.Errorf("invalid zone list") + } + zones := make(map[string]zoneEntry, len(*document.Zones)) + for _, zone := range *document.Zones { + zones[strings.ToUpper(zone.Code)] = zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary} + } + return zones, nil +} + +// decodeBoundary returns nil when MeshMapper has no usable outline for the region. +func decodeBoundary(body []byte, iata string) (json.RawMessage, error) { + var collection struct { + Type string `json:"type"` + Features []json.RawMessage `json:"features"` + } + if err := json.Unmarshal(body, &collection); err != nil { + return nil, err + } + if collection.Type != "FeatureCollection" || len(collection.Features) != 1 { + return nil, fmt.Errorf("expected one region feature") + } + var feature struct { + Geometry json.RawMessage `json:"geometry"` + Properties struct { + Code string `json:"code"` + } `json:"properties"` + } + if err := json.Unmarshal(collection.Features[0], &feature); err != nil { + return nil, err + } + if !strings.EqualFold(feature.Properties.Code, iata) { + return nil, fmt.Errorf("feature is for another region") + } + if len(feature.Geometry) == 0 || string(feature.Geometry) == "null" { + return nil, nil + } + validated, err := config.ValidateBorder(collection.Features[0]) + if err != nil { + return nil, err + } + // Foreign marking must accept anything the map serves. + if _, err := config.BuildLocalBorders(nil, map[string]json.RawMessage{iata: validated}); err != nil { + return nil, err + } + return validated, nil +} diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go new file mode 100644 index 00000000..af62cc84 --- /dev/null +++ b/internal/meshmapper/zones_test.go @@ -0,0 +1,331 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +const square = `{"type":"Polygon","coordinates":[[[-76,45],[-75,45],[-75,46],[-76,46],[-76,45]]]}` + +func boundaryBody(code, geometry string) string { + return `{"type":"FeatureCollection","generated_at":"2026-09-25T15:00:00Z","features":[{"type":"Feature","geometry":` + geometry + + `,"properties":{"code":"` + code + `","name":"Ottawa, CA","center":[-75.7,45.4],"radius_km":42,"has_boundary":true}}]}` +} + +type zoneMemoryStore struct { + rows map[string]Boundary + pruned []string + fail bool +} + +func (s *zoneMemoryStore) PruneZoneBoundaries(_ context.Context, keep []string) ([]string, error) { + var removed []string + for iata := range s.rows { + if !slices.Contains(keep, iata) { + delete(s.rows, iata) + removed = append(removed, iata) + } + } + s.pruned = removed + return removed, nil +} + +func (s *zoneMemoryStore) ListZoneBoundaries(context.Context) ([]Boundary, error) { + var out []Boundary + for _, b := range s.rows { + out = append(out, b) + } + return out, nil +} + +func (s *zoneMemoryStore) SaveZoneBoundary(_ context.Context, b Boundary) error { + if s.fail { + return errors.New("offline") + } + old := s.rows[b.IATA] + if b.Feature == nil { + b.Feature = old.Feature + } + if b.CheckedAt.IsZero() { + b.CheckedAt, b.ETag = old.CheckedAt, old.ETag + } + s.rows[b.IATA] = b + return nil +} + +type fakeMeshMapper struct { + *httptest.Server + list, boundary string + listStatus, status int + etag, retryAfter string + listCalls, calls int + ifNoneMatch []string +} + +func newFakeMeshMapper(t *testing.T) *fakeMeshMapper { + f := &fakeMeshMapper{listStatus: 200, status: 200, etag: `"v1"`, boundary: boundaryBody("YOW", square)} + f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/get_zones.php": + f.listCalls++ + if r.URL.Query().Get("country") != "CA" { + t.Errorf("country query: %s", r.URL.RawQuery) + } + w.WriteHeader(f.listStatus) + _, _ = w.Write([]byte(f.list)) + case "/get_geojson.php": + f.calls++ + f.ifNoneMatch = append(f.ifNoneMatch, r.Header.Get("If-None-Match")) + w.Header().Set("ETag", f.etag) + w.Header().Set("Retry-After", f.retryAfter) + w.WriteHeader(f.status) + if f.status == 200 { + _, _ = w.Write([]byte(f.boundary)) + } + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + f.list = `{"country":"CA","count":1,"zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}],"groups":[]}` + t.Cleanup(f.Close) + return f +} + +type zoneHarness struct { + z *Zones + store *zoneMemoryStore + changed []string + imported map[string]json.RawMessage +} + +func newZoneHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, enabled bool) *zoneHarness { + t.Helper() + h := &zoneHarness{store: store} + h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, []string{"YOW"}, store) + h.z.listURL = f.URL + "/get_zones.php" + h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", strings.HasPrefix(site, f.URL) } + h.z.OnChange(func(_ context.Context, iata string) { h.changed = append(h.changed, iata) }) + h.z.OnUpdate(func(imported map[string]json.RawMessage) { h.imported = imported }) + if err := h.z.Restore(context.Background()); err != nil { + t.Fatal(err) + } + return h +} + +// tick forces the region due and runs refreshes until the boundary request is made. +func (h *zoneHarness) tick(t *testing.T) Boundary { + t.Helper() + h.z.regions[0].b.NextAttempt = time.Time{} + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + return h.store.rows["YOW"] +} + +func TestZonesImportAndConditionalRefresh(t *testing.T) { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + b := h.tick(t) + if f.listCalls != 1 || f.calls != 1 || b.Feature == nil || b.ETag != `"v1"` || b.LastError != "" || b.URL != f.URL+"/" { + t.Fatalf("import failed: %+v list=%d calls=%d", b, f.listCalls, f.calls) + } + if !slices.Equal(h.changed, []string{"YOW"}) || h.imported["YOW"] == nil { + t.Fatal("listeners not told", h.changed, h.imported) + } + if !strings.Contains(string(b.Feature), `"bbox"`) || !strings.Contains(string(b.Feature), `"code":"YOW"`) { + t.Fatal("feature not validated with bbox and properties", string(b.Feature)) + } + if got := b.NextAttempt.Sub(b.AttemptedAt); got != 24*time.Hour { + t.Fatal("refresh interval", got) + } + + f.status = 304 + b = h.tick(t) + if f.listCalls != 1 || f.ifNoneMatch[1] != `"v1"` || b.Feature == nil || b.LastError != "" || len(h.changed) != 1 { + t.Fatalf("304 mishandled: %+v list=%d changed=%v", b, f.listCalls, h.changed) + } + + f.status, f.etag = 200, `"v2"` + f.boundary = boundaryBody("YOW", `{"type":"Polygon","coordinates":[[[-77,45],[-75,45],[-75,46],[-77,46],[-77,45]]]}`) + b = h.tick(t) + if b.ETag != `"v2"` || len(h.changed) != 2 || !strings.Contains(string(h.imported["YOW"]), "-77") { + t.Fatal("changed boundary not published", b.ETag, h.changed) + } +} + +func TestZonesKeepLastGoodBoundary(t *testing.T) { + for _, tc := range []struct { + name, boundary string + status int + wantErr string + wantRetry time.Duration + }{ + {"null geometry", boundaryBody("YOW", "null"), 200, "no boundary", 24 * time.Hour}, + {"not found", "", 404, "HTTP 404", time.Hour}, + {"unavailable", "", 503, "HTTP 503", time.Hour}, + {"truncated", boundaryBody("YOW", square)[:80], 200, "invalid response", time.Hour}, + {"other region", boundaryBody("YVR", square), 200, "invalid response", time.Hour}, + {"oversized", `{"type":"FeatureCollection","features":[],"pad":"` + strings.Repeat("x", MaxBoundaryBody) + `"}`, 200, "invalid response", time.Hour}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + good := h.tick(t).Feature + f.status, f.boundary = tc.status, tc.boundary + b := h.tick(t) + if b.LastError != tc.wantErr || string(b.Feature) != string(good) || string(h.imported["YOW"]) != string(good) || len(h.changed) != 1 { + t.Fatalf("got %q, feature kept=%v", b.LastError, string(b.Feature) == string(good)) + } + if got := b.NextAttempt.Sub(b.AttemptedAt); got != tc.wantRetry { + t.Fatal("retry", got) + } + }) + } +} + +func TestZonesSkipRegionsWithoutBoundary(t *testing.T) { + for _, tc := range []struct{ name, zones, want string }{ + {"not listed", `[]`, "not listed"}, + {"no boundary", `[{"code":"YOW","url":"URL/","has_boundary":false}]`, "no boundary"}, + {"foreign site", `[{"code":"yow","url":"https://example.com/","has_boundary":true}]`, "invalid site URL"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFakeMeshMapper(t) + f.list = `{"country":"CA","count":1,"zones":` + strings.ReplaceAll(tc.zones, "URL", f.URL) + `,"groups":[]}` + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + b := h.tick(t) + if f.calls != 0 || b.LastError != tc.want || b.Feature != nil || len(h.changed) != 0 { + t.Fatalf("got %+v calls=%d", b, f.calls) + } + if got := b.NextAttempt.Sub(b.AttemptedAt); got != 24*time.Hour { + t.Fatal("authoritative answer retried early", got) + } + }) + } +} + +func TestZonesListFailureBacksOff(t *testing.T) { + f := newFakeMeshMapper(t) + f.listStatus = 500 + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + for range 3 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if f.listCalls != 1 || f.calls != 0 || len(h.store.rows) != 0 { + t.Fatal("failed list retried early", f.listCalls, f.calls) + } + f.listStatus, f.list = 200, `{"country":"US","zones":[]}` + h.z.lists["CA"].nextAttempt, h.z.lists["CA"].fetchedAt = time.Time{}, time.Time{} + _ = h.z.Refresh(context.Background()) + if h.z.lists["CA"].zones != nil { + t.Fatal("list for another country accepted") + } +} + +func TestZonesRateLimitPausesEveryRequest(t *testing.T) { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + f.status, f.retryAfter = 429, "120" + b := h.tick(t) + if b.LastError != "HTTP 429" || b.NextAttempt.Sub(b.AttemptedAt) != time.Hour || h.z.retryAfter.Sub(b.AttemptedAt) != 2*time.Minute { + t.Fatalf("429 not honoured: %+v retryAfter=%v", b, h.z.retryAfter) + } + h.z.regions[0].b.NextAttempt = time.Time{} + _ = h.z.Refresh(context.Background()) + if f.calls != 1 { + t.Fatal("requested during Retry-After") + } + restarted := newZoneHarness(t, f, h.store, true) + if !restarted.z.retryAfter.Equal(b.NextAttempt) { + t.Fatal("persisted 429 not restored") + } +} + +func TestZonesRestoreAndPrune(t *testing.T) { + f := newFakeMeshMapper(t) + store := &zoneMemoryStore{rows: map[string]Boundary{}} + first := newZoneHarness(t, f, store, true) + first.tick(t) + store.rows["OLD"] = Boundary{IATA: "OLD", Feature: json.RawMessage(`{}`)} + + restarted := newZoneHarness(t, f, store, true) + if !slices.Equal(store.pruned, []string{"OLD"}) || !slices.Equal(restarted.changed, []string{"OLD"}) { + t.Fatal("unconfigured IATA not pruned", store.pruned, restarted.changed) + } + if restarted.imported["YOW"] == nil || f.calls != 1 { + t.Fatal("saved boundary not restored without a request") + } + if err := restarted.z.Refresh(context.Background()); err != nil || f.calls != 1 || f.listCalls != 1 { + t.Fatal("refreshed before due", err, f.calls) + } + + disabled := newZoneHarness(t, f, store, false) + if len(store.rows) != 0 || !slices.Equal(disabled.changed, []string{"YOW"}) || len(disabled.z.regions) != 0 { + t.Fatal("disabling did not remove imports") + } +} + +func TestZonesPersistFailureWaitsForNextAttempt(t *testing.T) { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}, fail: true}, true) + h.z.regions[0].b.NextAttempt = time.Time{} + _ = h.z.Refresh(context.Background()) + if err := h.z.Refresh(context.Background()); err == nil { + t.Fatal("persist error hidden") + } + if h.z.regions[0].b.NextAttempt.IsZero() || h.imported["YOW"] != nil || len(h.changed) != 0 { + t.Fatal("uncommitted boundary published") + } +} + +func TestBoundaryEndpoint(t *testing.T) { + for site, want := range map[string]string{ + "https://yow.meshmapper.net/": "https://yow.meshmapper.net/get_geojson.php", + "https://yow.meshmapper.net": "https://yow.meshmapper.net/get_geojson.php", + "http://yow.meshmapper.net/": "", + "https://meshmapper.net.evil.com/": "", + "https://user@yow.meshmapper.net/": "", + "https://yow.meshmapper.net/x/": "", + "https://yow.meshmapper.net/?a=b": "", + "https://yow.meshmapper.net:8443/": "", + "https://yow.meshmapper.net/#fragment": "", + } { + got, ok := boundaryEndpoint(site) + if got != want || ok != (want != "") { + t.Errorf("%s: %q %v", site, got, ok) + } + } +} + +func TestDecodeBoundaryRejectsUnusableOutlines(t *testing.T) { + for name, body := range map[string]string{ + "two features": `{"type":"FeatureCollection","features":[{"type":"Feature","geometry":null,"properties":{"code":"YOW"}},{"type":"Feature","geometry":null,"properties":{"code":"YOW"}}]}`, + "not a polygon": boundaryBody("YOW", `{"type":"Point","coordinates":[-75,45]}`), + "open ring": boundaryBody("YOW", `{"type":"Polygon","coordinates":[[[-76,45],[-75,45],[-75,46],[-76,46]]]}`), + "antimeridian": boundaryBody("YOW", `{"type":"Polygon","coordinates":[[[179,10],[-179,10],[-179,20],[179,20],[179,10]]]}`), + "zero area": boundaryBody("YOW", `{"type":"Polygon","coordinates":[[[-76,45],[-75,45],[-74,45],[-76,45]]]}`), + } { + if _, err := decodeBoundary([]byte(body), "YOW"); err == nil { + t.Errorf("%s accepted", name) + } + } + if feature, err := decodeBoundary([]byte(boundaryBody("yow", square)), "YOW"); err != nil || feature == nil { + t.Fatal("case-insensitive code rejected", err) + } +} From 14354b03dc84fa5318914c8fc86444251f3f23cf Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 16:29:28 -0400 Subject: [PATCH 105/141] fix(telemetry): stop partial and bogus stats inflating bucketed counters Bucketed airtime and receive errors used MAX - MIN per bucket, so one zeroed, replayed or garbage row counted the whole running total (e.g. 47,737 s RX in a 6 h bucket). Buckets are now summed from per-row increments with dips, stale replays and physically impossible jumps rejected. Ingest skips uptime-only stats and a migration deletes the rows already stored. --- .../045_clean_partial_observer_telemetry.sql | 7 + db/observers.go | 32 +-- db/observers_test.go | 37 ++-- db/queries/queries.sql | 18 -- db/sqlc/mock/querier.go | 15 -- db/sqlc/querier.go | 1 - db/sqlc/queries.sql.go | 71 ------- internal/api/telemetry.go | 164 +++++++++++++++ internal/api/telemetry_test.go | 199 ++++++++++++++++++ internal/ingest/status.go | 54 +++-- internal/ingest/status_test.go | 35 +++ 11 files changed, 465 insertions(+), 168 deletions(-) create mode 100644 db/migrations/045_clean_partial_observer_telemetry.sql create mode 100644 internal/api/telemetry.go create mode 100644 internal/api/telemetry_test.go create mode 100644 internal/ingest/status_test.go diff --git a/db/migrations/045_clean_partial_observer_telemetry.sql b/db/migrations/045_clean_partial_observer_telemetry.sql new file mode 100644 index 00000000..b5bdfadf --- /dev/null +++ b/db/migrations/045_clean_partial_observer_telemetry.sql @@ -0,0 +1,7 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Some /status messages carry uptime but no radio stats. Those rows zero the airtime and +-- noise floor, dragging bucket averages down; ingest now skips them (statusStats.usable). +DELETE FROM observer_telemetry +WHERE noise_floor_db = 0 AND airtime_tx_secs = 0 AND airtime_rx_secs = 0; diff --git a/db/observers.go b/db/observers.go index f312aca1..566256c3 100644 --- a/db/observers.go +++ b/db/observers.go @@ -196,37 +196,17 @@ func (s *Store) GetObserverTelemetry(ctx context.Context, observerID uuid.UUID, return &api.ObserverTelemetry{Points: points}, nil } +// GetObserverTelemetryBucketed reads a day before since so the first bucket's counter increments have a baseline. func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]api.ObserverTelemetryPoint, error) { - var sinceTS, untilTS pgtype.Timestamptz - if !since.IsZero() { - sinceTS = pgtype.Timestamptz{Time: since, Valid: true} + from := since + if !from.IsZero() { + from = from.Add(-24 * time.Hour) } - if !until.IsZero() { - untilTS = pgtype.Timestamptz{Time: until, Valid: true} - } - rows, err := s.q.GetObserverTelemetryBucketed(ctx, sqlc.GetObserverTelemetryBucketedParams{ - ObserverID: observerID, - Column2: sinceTS, - Column3: untilTS, - Column4: bucketHours, - }) + raw, err := s.GetObserverTelemetry(ctx, observerID, from, until, 0) if err != nil { return nil, err } - points := make([]api.ObserverTelemetryPoint, 0, len(rows)) - for _, r := range rows { - points = append(points, api.ObserverTelemetryPoint{ - T: r.Bucket.Time.UnixMilli(), - BatteryMV: &r.BatteryVoltageMv, - AirtimeTxSecs: &r.AirtimeTxSecs, - AirtimeRxSecs: &r.AirtimeRxSecs, - NoiseFloorDB: &r.NoiseFloorDb, - UptimeSeconds: &r.UptimeSeconds, - QueueLength: &r.QueueLength, - ReceiveErrors: &r.ReceiveErrors, - }) - } - return points, nil + return api.BucketTelemetry(raw.Points, since, time.Duration(bucketHours)*time.Hour), nil } // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. diff --git a/db/observers_test.go b/db/observers_test.go index e5b98506..20597f8c 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -394,32 +394,39 @@ func TestGetObserverTelemetryBucketed_Mapping(t *testing.T) { mock := mockdb.NewMockQuerier(ctrl) observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") - bucket := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} + since := time.Date(2026, 9, 30, 6, 0, 0, 0, time.UTC) + at := func(h int) pgtype.Timestamptz { + return pgtype.Timestamptz{Time: since.Add(time.Duration(h) * time.Hour), Valid: true} + } + f := func(v float32) *float32 { return &v } + up := func(v int64) *int64 { return &v } + bat := int32(3700) mock.EXPECT(). - GetObserverTelemetryBucketed(gomock.Any(), gomock.Any()). - Return([]sqlc.GetObserverTelemetryBucketedRow{ - { - Bucket: bucket, - BatteryVoltageMv: 3700, - NoiseFloorDb: -90.0, - UptimeSeconds: 3600, - }, - }, nil) + GetObserverTelemetry(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, p sqlc.GetObserverTelemetryParams) ([]sqlc.GetObserverTelemetryRow, error) { + if want := since.Add(-24 * time.Hour); !p.Column2.Time.Equal(want) { + t.Errorf("since = %v, want lookback to %v", p.Column2.Time, want) + } + return []sqlc.GetObserverTelemetryRow{ + {ReportedAt: at(-1), AirtimeTxSecs: f(100), UptimeSeconds: up(10000)}, + {ReportedAt: at(0), AirtimeTxSecs: f(110), UptimeSeconds: up(13600), BatteryVoltageMv: &bat}, + }, nil + }) store := &Store{q: mock} - points, err := store.GetObserverTelemetryBucketed(context.Background(), observerID, time.Time{}, time.Time{}, 1) + points, err := store.GetObserverTelemetryBucketed(context.Background(), observerID, since, time.Time{}, 6) if err != nil { t.Fatalf("unexpected error: %v", err) } if len(points) != 1 { t.Fatalf("expected 1 point, got %d", len(points)) } - if points[0].T != 1700000000000 { - t.Errorf("expected T 1700000000000, got %d", points[0].T) + if points[0].T != since.UnixMilli() { + t.Errorf("expected T %d, got %d", since.UnixMilli(), points[0].T) } - if *points[0].BatteryMV != 3700 { - t.Errorf("expected BatteryMV 3700, got %d", *points[0].BatteryMV) + if *points[0].AirtimeTxSecs != 10 || *points[0].BatteryMV != 3700 { + t.Errorf("tx %v battery %v, want 10 3700", *points[0].AirtimeTxSecs, *points[0].BatteryMV) } } diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 941b340f..ca7584c8 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -246,24 +246,6 @@ WHERE observer_id = $1 AND ($4 = 0 OR id > $4) ORDER BY reported_at ASC; --- name: GetObserverTelemetryBucketed :many -SELECT - (date_trunc('day', reported_at) + - (EXTRACT(HOUR FROM reported_at)::int / $4::int) * ($4::int * interval '1 hour'))::timestamptz AS bucket, - AVG(battery_voltage_mv)::int AS battery_voltage_mv, - GREATEST(MAX(airtime_tx_secs) - MIN(airtime_tx_secs), 0)::real AS airtime_tx_secs, - GREATEST(MAX(airtime_rx_secs) - MIN(airtime_rx_secs), 0)::real AS airtime_rx_secs, - AVG(noise_floor_db)::real AS noise_floor_db, - MAX(uptime_seconds)::bigint AS uptime_seconds, - AVG(queue_length)::int AS queue_length, - GREATEST(MAX(receive_errors) - MIN(receive_errors), 0)::int AS receive_errors -FROM observer_telemetry -WHERE observer_id = $1 - AND ($2::timestamptz IS NULL OR reported_at >= $2) - AND ($3::timestamptz IS NULL OR reported_at <= $3) -GROUP BY bucket -ORDER BY bucket ASC; - -- name: GetObserverActivityRaw :many -- Sub-hour activity buckets straight off idx_observations_observer; no join to packets. -- Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 86beeece..f9914cf0 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -565,21 +565,6 @@ func (mr *MockQuerierMockRecorder) GetObserverTelemetry(ctx, arg any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverTelemetry", reflect.TypeOf((*MockQuerier)(nil).GetObserverTelemetry), ctx, arg) } -// GetObserverTelemetryBucketed mocks base method. -func (m *MockQuerier) GetObserverTelemetryBucketed(ctx context.Context, arg db.GetObserverTelemetryBucketedParams) ([]db.GetObserverTelemetryBucketedRow, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetObserverTelemetryBucketed", ctx, arg) - ret0, _ := ret[0].([]db.GetObserverTelemetryBucketedRow) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetObserverTelemetryBucketed indicates an expected call of GetObserverTelemetryBucketed. -func (mr *MockQuerierMockRecorder) GetObserverTelemetryBucketed(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverTelemetryBucketed", reflect.TypeOf((*MockQuerier)(nil).GetObserverTelemetryBucketed), ctx, arg) -} - // GetPacketByHash mocks base method. func (m *MockQuerier) GetPacketByHash(ctx context.Context, packetHash []byte) (db.GetPacketByHashRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 364aea68..ef53cb36 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -78,7 +78,6 @@ type Querier interface { GetObserverRadio(ctx context.Context, id uuid.UUID) (GetObserverRadioRow, error) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) GetObserverTelemetry(ctx context.Context, arg GetObserverTelemetryParams) ([]GetObserverTelemetryRow, error) - GetObserverTelemetryBucketed(ctx context.Context, arg GetObserverTelemetryBucketedParams) ([]GetObserverTelemetryBucketedRow, error) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPacketByHashRow, error) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) // Return distinct observation IATAs in first-heard order for path resolution, diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index ce7bb12d..9ea06a5b 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1174,77 +1174,6 @@ func (q *Queries) GetObserverTelemetry(ctx context.Context, arg GetObserverTelem return items, nil } -const getObserverTelemetryBucketed = `-- name: GetObserverTelemetryBucketed :many -SELECT - (date_trunc('day', reported_at) + - (EXTRACT(HOUR FROM reported_at)::int / $4::int) * ($4::int * interval '1 hour'))::timestamptz AS bucket, - AVG(battery_voltage_mv)::int AS battery_voltage_mv, - GREATEST(MAX(airtime_tx_secs) - MIN(airtime_tx_secs), 0)::real AS airtime_tx_secs, - GREATEST(MAX(airtime_rx_secs) - MIN(airtime_rx_secs), 0)::real AS airtime_rx_secs, - AVG(noise_floor_db)::real AS noise_floor_db, - MAX(uptime_seconds)::bigint AS uptime_seconds, - AVG(queue_length)::int AS queue_length, - GREATEST(MAX(receive_errors) - MIN(receive_errors), 0)::int AS receive_errors -FROM observer_telemetry -WHERE observer_id = $1 - AND ($2::timestamptz IS NULL OR reported_at >= $2) - AND ($3::timestamptz IS NULL OR reported_at <= $3) -GROUP BY bucket -ORDER BY bucket ASC -` - -type GetObserverTelemetryBucketedParams struct { - ObserverID uuid.UUID `json:"observer_id"` - Column2 pgtype.Timestamptz `json:"column_2"` - Column3 pgtype.Timestamptz `json:"column_3"` - Column4 int32 `json:"column_4"` -} - -type GetObserverTelemetryBucketedRow struct { - Bucket pgtype.Timestamptz `json:"bucket"` - BatteryVoltageMv int32 `json:"battery_voltage_mv"` - AirtimeTxSecs float32 `json:"airtime_tx_secs"` - AirtimeRxSecs float32 `json:"airtime_rx_secs"` - NoiseFloorDb float32 `json:"noise_floor_db"` - UptimeSeconds int64 `json:"uptime_seconds"` - QueueLength int32 `json:"queue_length"` - ReceiveErrors int32 `json:"receive_errors"` -} - -func (q *Queries) GetObserverTelemetryBucketed(ctx context.Context, arg GetObserverTelemetryBucketedParams) ([]GetObserverTelemetryBucketedRow, error) { - rows, err := q.db.Query(ctx, getObserverTelemetryBucketed, - arg.ObserverID, - arg.Column2, - arg.Column3, - arg.Column4, - ) - if err != nil { - return nil, err - } - defer rows.Close() - items := []GetObserverTelemetryBucketedRow{} - for rows.Next() { - var i GetObserverTelemetryBucketedRow - if err := rows.Scan( - &i.Bucket, - &i.BatteryVoltageMv, - &i.AirtimeTxSecs, - &i.AirtimeRxSecs, - &i.NoiseFloorDb, - &i.UptimeSeconds, - &i.QueueLength, - &i.ReceiveErrors, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const getPacketByHash = `-- name: GetPacketByHash :one SELECT p.packet_hash, p.payload_type, p.payload_version, p.route_type, p.transport_codes_present, p.region_code, p.sub_region_code, p.scope_id, p.origin_pubkey, p.raw_payload, p.raw_header, p.parsed_payload, p.decrypted, p.channel_hash, p.trace_tag, p.first_heard_at, p.last_heard_at, ts.name AS scope_name, cm.sender_name AS cm_sender_name, diff --git a/internal/api/telemetry.go b/internal/api/telemetry.go new file mode 100644 index 00000000..94fce69c --- /dev/null +++ b/internal/api/telemetry.go @@ -0,0 +1,164 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "math" + "time" +) + +// Fastest a counter can honestly climb, per second of wall time: radio airtime can't +// outrun the clock, and real receive errors peak well under 1/s. +const ( + airtimeRate = 1.0 + errorRate = 10.0 +) + +// counter turns a since-boot running total into increments, rejecting readings +// that can't be real: dips (partial stats) and jumps faster than rate allows +// (garbage values). Three rejections in a row mean the baseline itself was bad. +// Growth across a gap longer than maxGap can't be placed in a bucket, so it's dropped. +type counter struct { + rate float64 + maxGap int64 + last float64 + lastT int64 + seeded bool + rejects int +} + +func (c *counter) add(v *float64, t int64, rebooted bool) float64 { + if v == nil { + return 0 + } + if !c.seeded || t-c.lastT > c.maxGap { + c.last, c.lastT, c.seeded, c.rejects = *v, t, true, 0 + return 0 + } + d := *v - c.last + if rebooted && d < 0 { + d = *v // restarted with the device; some firmware keeps counting instead + } + // Rows are truncated to the telemetry resolution, so allow twice the gap. + if d < 0 || d > c.rate*2*float64(t-c.lastT)/1000 { + if c.rejects++; c.rejects < 3 { + return 0 + } + d = 0 + } + c.last, c.lastT, c.rejects = *v, t, 0 + return d +} + +type mean struct { + sum float64 + n int +} + +func (m *mean) add(v *float64) { + if v != nil { + m.sum += *v + m.n++ + } +} + +type telemetryBucket struct { + start time.Time + tx, rx, errs float64 + hasTx, hasRx, hasErr bool + battery, noise, qlen mean + uptime *int64 +} + +// BucketTelemetry folds raw telemetry rows (ascending, with any rows before since +// serving only as counter baselines) into fixed UTC buckets. Counters are summed +// increments; battery, noise floor and queue are averaged; uptime is the max. +func BucketTelemetry(points []ObserverTelemetryPoint, since time.Time, size time.Duration) []ObserverTelemetryPoint { + maxGap := size.Milliseconds() + tx := counter{rate: airtimeRate, maxGap: maxGap} + rx := counter{rate: airtimeRate, maxGap: maxGap} + errs := counter{rate: errorRate, maxGap: maxGap} + var last *ObserverTelemetryPoint + var buckets []*telemetryBucket + + for i := range points { + p := &points[i] + rebooted := false + if last != nil && p.UptimeSeconds != nil && last.UptimeSeconds != nil && *p.UptimeSeconds < *last.UptimeSeconds { + if *p.UptimeSeconds > 2*(p.T-last.T)/1000 { + continue // an old snapshot replayed, not a reboot + } + rebooted = true + } + last = p + + dTx := tx.add(f64(p.AirtimeTxSecs), p.T, rebooted) + dRx := rx.add(f64(p.AirtimeRxSecs), p.T, rebooted) + dErr := errs.add(i32f(p.ReceiveErrors), p.T, rebooted) + t := time.UnixMilli(p.T).UTC() + if t.Before(since) { + continue + } + start := t.Truncate(size) + if len(buckets) == 0 || !buckets[len(buckets)-1].start.Equal(start) { + buckets = append(buckets, &telemetryBucket{start: start}) + } + b := buckets[len(buckets)-1] + b.tx += dTx + b.rx += dRx + b.errs += dErr + b.hasTx = b.hasTx || p.AirtimeTxSecs != nil + b.hasRx = b.hasRx || p.AirtimeRxSecs != nil + b.hasErr = b.hasErr || p.ReceiveErrors != nil + b.battery.add(i32f(p.BatteryMV)) + b.noise.add(f64(p.NoiseFloorDB)) + b.qlen.add(i32f(p.QueueLength)) + if p.UptimeSeconds != nil && (b.uptime == nil || *p.UptimeSeconds > *b.uptime) { + b.uptime = p.UptimeSeconds + } + } + + out := make([]ObserverTelemetryPoint, 0, len(buckets)) + for _, b := range buckets { + pt := ObserverTelemetryPoint{T: b.start.UnixMilli(), UptimeSeconds: b.uptime} + if b.hasTx { + pt.AirtimeTxSecs = ptr(float32(b.tx)) + } + if b.hasRx { + pt.AirtimeRxSecs = ptr(float32(b.rx)) + } + if b.hasErr { + pt.ReceiveErrors = ptr(int32(b.errs)) + } + if b.battery.n > 0 { + pt.BatteryMV = ptr(int32(math.Round(b.battery.sum / float64(b.battery.n)))) + } + if b.noise.n > 0 { + pt.NoiseFloorDB = ptr(float32(b.noise.sum / float64(b.noise.n))) + } + if b.qlen.n > 0 { + pt.QueueLength = ptr(int32(math.Round(b.qlen.sum / float64(b.qlen.n)))) + } + out = append(out, pt) + } + return out +} + +func f64(v *float32) *float64 { + if v == nil { + return nil + } + f := float64(*v) + return &f +} + +func i32f(v *int32) *float64 { + if v == nil { + return nil + } + f := float64(*v) + return &f +} + +func ptr[T any](v T) *T { return &v } diff --git a/internal/api/telemetry_test.go b/internal/api/telemetry_test.go new file mode 100644 index 00000000..37e08335 --- /dev/null +++ b/internal/api/telemetry_test.go @@ -0,0 +1,199 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "testing" + "time" +) + +var telemetryBase = time.Date(2026, 9, 30, 0, 0, 0, 0, time.UTC) + +type row struct { + hour int + tx, rx float32 + errs int32 + noise float32 + uptime int64 +} + +func telemetryPoints(rows ...row) []ObserverTelemetryPoint { + pts := make([]ObserverTelemetryPoint, 0, len(rows)) + for _, r := range rows { + tx, rx, errs, noise, uptime := r.tx, r.rx, r.errs, r.noise, r.uptime + pts = append(pts, ObserverTelemetryPoint{ + T: telemetryBase.Add(time.Duration(r.hour) * time.Hour).UnixMilli(), + AirtimeTxSecs: &tx, + AirtimeRxSecs: &rx, + ReceiveErrors: &errs, + NoiseFloorDB: &noise, + UptimeSeconds: &uptime, + }) + } + return pts +} + +func bucketAt(t *testing.T, buckets []ObserverTelemetryPoint, hour int) ObserverTelemetryPoint { + t.Helper() + want := telemetryBase.Add(time.Duration(hour) * time.Hour).UnixMilli() + for _, b := range buckets { + if b.T == want { + return b + } + } + t.Fatalf("no bucket at hour %d in %+v", hour, buckets) + return ObserverTelemetryPoint{} +} + +func checkCounters(t *testing.T, b ObserverTelemetryPoint, tx, rx float32, errs int32) { + t.Helper() + if *b.AirtimeTxSecs != tx || *b.AirtimeRxSecs != rx || *b.ReceiveErrors != errs { + t.Errorf("counters = tx %v rx %v errs %v, want tx %v rx %v errs %v", + *b.AirtimeTxSecs, *b.AirtimeRxSecs, *b.ReceiveErrors, tx, rx, errs) + } +} + +func TestBucketTelemetry_SumsIncrements(t *testing.T) { + pts := telemetryPoints( + row{0, 100, 1000, 50, -90, 10000}, + row{1, 110, 1060, 55, -92, 13600}, + row{2, 125, 1100, 60, -94, 17200}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + if len(b) != 1 { + t.Fatalf("got %d buckets, want 1", len(b)) + } + checkCounters(t, b[0], 25, 100, 10) + if *b[0].NoiseFloorDB != -92 || *b[0].UptimeSeconds != 17200 { + t.Errorf("noise %v uptime %v, want -92 17200", *b[0].NoiseFloorDB, *b[0].UptimeSeconds) + } +} + +// Observer e61a62a7 on 2026-09-30: an uptime-only status stored zeros mid-run. +func TestBucketTelemetry_ZeroedRowIsNotARestart(t *testing.T) { + pts := telemetryPoints( + row{5, 7935, 47484, 93689, -96, 1093083}, + row{6, 0, 0, 0, 0, 1096701}, + row{7, 7951, 47586, 93884, -97, 1100319}, + ) + b := BucketTelemetry(pts, telemetryBase.Add(5*time.Hour), 6*time.Hour) + checkCounters(t, bucketAt(t, b, 6), 16, 102, 195) +} + +func TestBucketTelemetry_SingleCounterDip(t *testing.T) { + pts := telemetryPoints( + row{0, 3108, 19536, 38083, -108, 473828}, + row{1, 3160, 19837, 0, -92, 477447}, + row{2, 3208, 20145, 39211, -94, 481065}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 100, 609, 1128) +} + +func TestBucketTelemetry_RebootCountsSinceBoot(t *testing.T) { + pts := telemetryPoints( + row{0, 50, 500, 10, -90, 10000}, + row{1, 60, 560, 12, -90, 13600}, + row{2, 5, 40, 3, -90, 1200}, + row{3, 9, 70, 4, -90, 4800}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 10+5+4, 60+40+30, 2+3+1) +} + +// Observer 50350675 replays an old snapshot between live rows. +func TestBucketTelemetry_StaleReplaySkipped(t *testing.T) { + pts := telemetryPoints( + row{0, 121, 4788, 0, -120, 225259}, + row{1, 123, 4849, 0, -120, 228860}, + row{2, 88, 3879, 0, -50, 184130}, + row{3, 126, 4972, 0, -120, 236061}, + row{4, 88, 3879, 0, -50, 184130}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 5, 184, 0) + if *b[0].NoiseFloorDB != -120 || *b[0].UptimeSeconds != 236061 { + t.Errorf("stale row leaked into averages: noise %v uptime %v", *b[0].NoiseFloorDB, *b[0].UptimeSeconds) + } +} + +func TestBucketTelemetry_LookbackSeedsBaseline(t *testing.T) { + pts := telemetryPoints( + row{5, 100, 1000, 50, -90, 10000}, + row{6, 110, 1030, 52, -90, 13600}, + row{12, 130, 1090, 58, -90, 35200}, + ) + b := BucketTelemetry(pts, telemetryBase.Add(6*time.Hour), 6*time.Hour) + if len(b) != 2 { + t.Fatalf("got %d buckets, want 2 (lookback row must not get its own)", len(b)) + } + checkCounters(t, bucketAt(t, b, 6), 10, 30, 2) + checkCounters(t, bucketAt(t, b, 12), 20, 60, 6) +} + +// Some firmware keeps counting airtime across a reboot (ed5a20b6). +func TestBucketTelemetry_RebootWithPersistentCounters(t *testing.T) { + pts := telemetryPoints( + row{0, 900, 178000, 10, -90, 500000}, + row{1, 910, 178400, 12, -90, 503600}, + row{2, 915, 178485, 1, -90, 2250}, + row{3, 920, 178589, 3, -90, 5851}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 20, 589, 2+1+2) +} + +// Observer 408159b8 reported one-off garbage between sane readings. +func TestBucketTelemetry_GarbageSpikeRejected(t *testing.T) { + pts := telemetryPoints( + row{0, 96, 164, 144, -89, 145911}, + row{1, 101, 172, 146, -89, 149536}, + row{2, 111, 201329470, 2080374784, -90, 156847}, + row{3, 114, 197, 160, -89, 160464}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 18, 33, 16) +} + +func TestBucketTelemetry_GarbageBaselineRecovers(t *testing.T) { + pts := telemetryPoints( + row{0, 100, 2080374784, 50, -90, 10000}, + row{1, 110, 200, 55, -90, 13600}, + row{2, 120, 210, 60, -90, 17200}, + row{3, 130, 220, 65, -90, 20800}, + row{4, 140, 235, 70, -90, 24400}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, b[0], 40, 15, 20) +} + +func TestBucketTelemetry_GapLongerThanBucketDropped(t *testing.T) { + pts := telemetryPoints( + row{0, 100, 22018, 50, -90, 500000}, + row{1, 110, 22100, 55, -90, 503600}, + row{20, 400, 39295, 900, -90, 572000}, + row{21, 410, 39384, 905, -90, 575600}, + ) + b := BucketTelemetry(pts, telemetryBase, 6*time.Hour) + checkCounters(t, bucketAt(t, b, 18), 10, 89, 5) +} + +func TestBucketTelemetry_AveragesAndNils(t *testing.T) { + b1, b2 := int32(3700), int32(3705) + q := int32(2) + pts := []ObserverTelemetryPoint{ + {T: telemetryBase.UnixMilli(), BatteryMV: &b1, QueueLength: &q}, + {T: telemetryBase.Add(time.Hour).UnixMilli(), BatteryMV: &b2}, + } + b := BucketTelemetry(pts, telemetryBase, 24*time.Hour) + if len(b) != 1 { + t.Fatalf("got %d buckets, want 1", len(b)) + } + if *b[0].BatteryMV != 3703 || *b[0].QueueLength != 2 { + t.Errorf("battery %v queue %v, want 3703 2", *b[0].BatteryMV, *b[0].QueueLength) + } + if b[0].AirtimeTxSecs != nil || b[0].NoiseFloorDB != nil || b[0].UptimeSeconds != nil { + t.Errorf("fields with no data should stay nil: %+v", b[0]) + } +} diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 3c693db6..3e2ded6c 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -49,25 +49,37 @@ type statusEvent struct { LastStatusAt int64 `json:"lastStatusAt"` } +type statusStats struct { + UptimeSeconds int64 `json:"uptime_secs"` + BatteryMV int `json:"battery_mv"` + NoiseFloor float32 `json:"noise_floor"` + QueueLen int `json:"queue_len"` + DebugFlags int `json:"debug_flags"` + TxAirSecs float64 `json:"tx_air_secs"` + RxAirSecs float64 `json:"rx_air_secs"` + RecvErrors int `json:"recv_errors"` +} + +// usable reports whether the stats carry real radio readings. A running observer +// never reports zero uptime, and a live radio never reads a 0 dB noise floor, so +// that with zero airtime marks a status sent before the radio stats were filled in. +func (s statusStats) usable() bool { + if s.UptimeSeconds == 0 { + return false + } + return s.NoiseFloor != 0 || s.TxAirSecs != 0 || s.RxAirSecs != 0 +} + // handleStatus processes a /status message and fans out an observerStatus event. func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) { var envelope struct { - ObserverType string `json:"source"` - SoftwareVersion string `json:"client_version"` - HardwareModel string `json:"model"` - FirmwareVersion string `json:"firmware_version"` - DisplayName string `json:"origin"` - RadioString string `json:"radio"` - Stats struct { - UptimeSeconds int64 `json:"uptime_secs"` - BatteryMV int `json:"battery_mv"` - NoiseFloor float32 `json:"noise_floor"` - QueueLen int `json:"queue_len"` - DebugFlags int `json:"debug_flags"` - TxAirSecs float64 `json:"tx_air_secs"` - RxAirSecs float64 `json:"rx_air_secs"` - RecvErrors int `json:"recv_errors"` - } `json:"stats"` + ObserverType string `json:"source"` + SoftwareVersion string `json:"client_version"` + HardwareModel string `json:"model"` + FirmwareVersion string `json:"firmware_version"` + DisplayName string `json:"origin"` + RadioString string `json:"radio"` + Stats statusStats `json:"stats"` } if err := json.Unmarshal(raw, &envelope); err != nil { w.log.Warn(fmt.Sprintf("malformed status envelope from %s", pubkeyHex), "error", err) @@ -154,12 +166,10 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) w.log.Error(fmt.Sprintf("db: update observer status failed for %s", pubkeyHex), "error", err) return } - // Store a telemetry snapshot at the configured resolution. - // A running observer never reports uptime_secs == 0, so its absence (or a - // missing/renamed "stats" object entirely) means this payload has no usable - // stats — skip the insert rather than writing an all-zero row that would win - // the hourly dedup and pollute the telemetry aggregates. - if envelope.Stats.UptimeSeconds == 0 { + // Store a telemetry snapshot at the configured resolution. Partial stats are + // skipped rather than written as zeros that would win the hourly dedup and + // pollute the telemetry aggregates. + if !envelope.Stats.usable() { w.log.Debug("status has no usable stats; skipping telemetry insert") } else { resolution := w.cfg.TelemetryResolution diff --git a/internal/ingest/status_test.go b/internal/ingest/status_test.go new file mode 100644 index 00000000..d61713b6 --- /dev/null +++ b/internal/ingest/status_test.go @@ -0,0 +1,35 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "encoding/json" + "testing" +) + +func TestStatusStatsUsable(t *testing.T) { + cases := []struct { + name string + raw string + want bool + }{ + {"full stats", `{"uptime_secs":1093083,"noise_floor":-96,"tx_air_secs":7935,"rx_air_secs":47484,"recv_errors":93689}`, true}, + {"missing stats", `{}`, false}, + {"uptime only", `{"uptime_secs":1096701,"noise_floor":0,"tx_air_secs":0,"rx_air_secs":0,"recv_errors":0}`, false}, + {"uptime and recv errors only", `{"uptime_secs":3834495,"recv_errors":325529}`, false}, + {"fresh boot, no airtime yet", `{"uptime_secs":60,"noise_floor":-110,"tx_air_secs":0,"rx_air_secs":0}`, true}, + {"no noise floor reported", `{"uptime_secs":5000,"tx_air_secs":3,"rx_air_secs":40}`, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var s statusStats + if err := json.Unmarshal([]byte(c.raw), &s); err != nil { + t.Fatal(err) + } + if got := s.usable(); got != c.want { + t.Errorf("usable() = %v, want %v", got, c.want) + } + }) + } +} From 5b177d6fc9752da9e925a6f5c19684d06d25cd7f Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 18:29:22 -0400 Subject: [PATCH 106/141] fix(ingest): strip NUL bytes from observer status An observer sending NUL-padded status strings had every status update rejected by Postgres, freezing its status and telemetry. Strip them from the text fields and the stored JSON, and warn with the fields affected. Also drop the timestamp-clamp, empty-timestamp and malformed-IATA logs to debug; they're expected and outnumbered inserted packets several times over. --- internal/ingest/ingest.go | 2 +- internal/ingest/logging_test.go | 13 +++---- internal/ingest/packet.go | 4 +-- internal/ingest/status.go | 62 +++++++++++++++++++++++++++++++++ internal/ingest/status_test.go | 39 +++++++++++++++++++++ 5 files changed, 111 insertions(+), 9 deletions(-) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 523a1fb9..59b9b4e0 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -374,7 +374,7 @@ func (w *Worker) handleMessageContext(parent context.Context, msg mqtt.Message) // iata_codes.iata is CHAR(3); anything else would fail the DB insert // downstream, so reject malformed topic segments here instead. if !isValidIATA(iata) { - w.log.Warn("dropped packet with malformed IATA", "iata", iata, "topic", msg.Topic()) + w.log.Debug("dropped packet with malformed IATA", "iata", iata, "topic", msg.Topic()) return } diff --git a/internal/ingest/logging_test.go b/internal/ingest/logging_test.go index fa8bcc49..3f9d343a 100644 --- a/internal/ingest/logging_test.go +++ b/internal/ingest/logging_test.go @@ -22,14 +22,15 @@ func (m malformedTopicMessage) MessageID() uint16 { return 0 } func (m malformedTopicMessage) Payload() []byte { return nil } func (m malformedTopicMessage) Ack() {} -func TestMalformedIATAWarnNamesSource(t *testing.T) { +func TestMalformedIATALogNamesSource(t *testing.T) { for _, format := range []string{"json", "text"} { for _, iata := range []string{"XX", "XX\r\nforged=true"} { t.Run(format+"/"+strconv.Quote(iata), func(t *testing.T) { var output bytes.Buffer - var handler slog.Handler = slog.NewJSONHandler(&output, nil) + opts := &slog.HandlerOptions{Level: slog.LevelDebug} + var handler slog.Handler = slog.NewJSONHandler(&output, opts) if format == "text" { - handler = slog.NewTextHandler(&output, nil) + handler = slog.NewTextHandler(&output, opts) } previous := slog.Default() slog.SetDefault(slog.New(handler)) @@ -38,7 +39,7 @@ func TestMalformedIATAWarnNamesSource(t *testing.T) { topic := "meshcore/" + iata + "/0011223344556677/packets" worker.handleMessage(malformedTopicMessage{topic: topic}) if bytes.Count(output.Bytes(), []byte("\n")) != 1 || bytes.ContainsRune(output.Bytes(), '\r') { - t.Fatalf("warning did not remain one escaped record: %q", output.String()) + t.Fatalf("log did not remain one escaped record: %q", output.String()) } if format == "json" { var record map[string]any @@ -46,7 +47,7 @@ func TestMalformedIATAWarnNamesSource(t *testing.T) { t.Fatal(err) } if record["iata"] != iata || record["topic"] != topic || record["msg"] != "dropped packet with malformed IATA" { - t.Fatalf("warning lost its source: %v", record) + t.Fatalf("log lost its source: %v", record) } } else { expectedIATA, expectedTopic := iata, topic @@ -54,7 +55,7 @@ func TestMalformedIATAWarnNamesSource(t *testing.T) { expectedIATA, expectedTopic = strconv.Quote(iata), strconv.Quote(topic) } if !strings.Contains(output.String(), "iata="+expectedIATA) || !strings.Contains(output.String(), "topic="+expectedTopic) { - t.Fatalf("warning lost its source: %q", output.String()) + t.Fatalf("log lost its source: %q", output.String()) } } }) diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index ad5336eb..7bc4ccf5 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -732,14 +732,14 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } heardAt, err := parseHeardAt(envelope.Timestamp) if err != nil { - w.log.Warn(fmt.Sprintf("failed to parse timestamp %q", envelope.Timestamp), "error", err) + w.log.Debug(fmt.Sprintf("failed to parse timestamp %q", envelope.Timestamp), "error", err) heardAt = time.Now().UTC() } else { // clamp to server time if offset is suspicious (> 30 min drift) now := time.Now().UTC() diff := heardAt.UTC().Sub(now) if diff > 30*time.Minute || diff < -30*time.Minute { - w.log.Warn(fmt.Sprintf("clamping suspicious timestamp %s (diff %v) for pubkey %s", envelope.Timestamp, diff, pubkeyHex[:8])) + w.log.Debug(fmt.Sprintf("clamping suspicious timestamp %s (diff %v) for pubkey %s", envelope.Timestamp, diff, pubkeyHex[:8])) heardAt = now } } diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 3e2ded6c..9fe6a82b 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -4,10 +4,12 @@ package ingest import ( + "bytes" "context" "encoding/hex" "encoding/json" "fmt" + "slices" "strconv" "strings" "time" @@ -70,6 +72,57 @@ func (s statusStats) usable() bool { return s.NoiseFloor != 0 || s.TxAirSecs != 0 || s.RxAirSecs != 0 } +// stripNULs removes NUL bytes, which Postgres text columns reject, and returns the +// names of the fields that had any. +func stripNULs(fields map[string]*string) []string { + var dirty []string + for name, s := range fields { + if strings.Contains(*s, "\x00") { + *s = strings.ReplaceAll(*s, "\x00", "") + dirty = append(dirty, name) + } + } + slices.Sort(dirty) + return dirty +} + +// stripJSONNULs drops \u0000 from a JSON document's strings, which jsonb rejects. +// Payloads without one are returned untouched. +func stripJSONNULs(raw []byte) ([]byte, bool) { + if !bytes.Contains(raw, []byte(`\u0000`)) { + return raw, false + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return raw, false + } + out, err := json.Marshal(dropNULs(v)) + if err != nil { + return raw, false + } + return out, true +} + +func dropNULs(v any) any { + switch v := v.(type) { + case string: + return strings.ReplaceAll(v, "\x00", "") + case []any: + for i := range v { + v[i] = dropNULs(v[i]) + } + case map[string]any: + out := make(map[string]any, len(v)) + for k, e := range v { + out[strings.ReplaceAll(k, "\x00", "")] = dropNULs(e) + } + return out + } + return v +} + // handleStatus processes a /status message and fans out an observerStatus event. func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) { var envelope struct { @@ -90,6 +143,15 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) w.log.Warn(fmt.Sprintf("invalid pubkey hex in status from %s", pubkeyHex), "error", err) return } + dirty := stripNULs(map[string]*string{ + "source": &envelope.ObserverType, "client_version": &envelope.SoftwareVersion, + "model": &envelope.HardwareModel, "firmware_version": &envelope.FirmwareVersion, + "origin": &envelope.DisplayName, "radio": &envelope.RadioString, + }) + raw, rawDirty := stripJSONNULs(raw) + if len(dirty) > 0 || rawDirty { + w.log.Warn(fmt.Sprintf("stripped NUL bytes from status from %s", pubkeyHex), "fields", dirty, "metadata", rawDirty) + } id, _, err := w.db.UpsertObserver(ctx, pubkey) if err != nil { w.log.Error(fmt.Sprintf("db: upsert observer failed in status from %s", pubkeyHex), "error", err) diff --git a/internal/ingest/status_test.go b/internal/ingest/status_test.go index d61713b6..6d8876cc 100644 --- a/internal/ingest/status_test.go +++ b/internal/ingest/status_test.go @@ -33,3 +33,42 @@ func TestStatusStatsUsable(t *testing.T) { }) } } + +func TestStripNULs(t *testing.T) { + fw := "v1.17.1-d929643\x00\x00\x00" + name := "Canadaverse\x00 RemoteTerm" + model := "Heltec Wireless Paper" + got := stripNULs(map[string]*string{"firmware_version": &fw, "origin": &name, "model": &model}) + if fw != "v1.17.1-d929643" || name != "Canadaverse RemoteTerm" || model != "Heltec Wireless Paper" { + t.Errorf("stripped to %q %q %q", fw, name, model) + } + if len(got) != 2 || got[0] != "firmware_version" || got[1] != "origin" { + t.Errorf("reported fields %v, want [firmware_version origin]", got) + } +} + +func TestStripJSONNULs(t *testing.T) { + clean := []byte(`{"origin":"a","stats":{"uptime_secs":12}}`) + if out, changed := stripJSONNULs(clean); changed || string(out) != string(clean) { + t.Errorf("clean payload rewritten: %s", out) + } + + dirty := []byte(`{"firmware_version":"v1.17\u0000\u0000","extra":{"list":["x\u0000y"],"n":1.50}}`) + out, changed := stripJSONNULs(dirty) + if !changed { + t.Fatal("expected payload to be cleaned") + } + var v struct { + FirmwareVersion string `json:"firmware_version"` + Extra struct { + List []string `json:"list"` + N json.Number `json:"n"` + } `json:"extra"` + } + if err := json.Unmarshal(out, &v); err != nil { + t.Fatal(err) + } + if v.FirmwareVersion != "v1.17" || v.Extra.List[0] != "xy" || v.Extra.N != "1.50" { + t.Errorf("cleaned to %s", out) + } +} From 18f5925876c2c05a4ffe940ea7ca1d591f2bdcc8 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 13:46:33 -0400 Subject: [PATCH 107/141] ci: reserve latest images for stable releases --- .github/workflows/docker-publish.yml | 1 - README.md | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 48f459db..686b3d98 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -41,7 +41,6 @@ jobs: org.opencontainers.image.description=Beacon — MeshCore LoRa mesh observation server org.opencontainers.image.licenses=AGPL-3.0-or-later tags: | - type=raw,value=latest,enable={{is_default_branch}} type=raw,value=dev,enable=${{ github.ref == 'refs/heads/dev' }} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} diff --git a/README.md b/README.md index dfcec34f..6e07bc35 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,23 @@ in PostgreSQL, and streams live events to WebSocket clients. For deployment instructions including the frontend app, see the deployment docs. +### MeshCore Canada: Beacon 1.4.0 rollout + +Beacon/web 1.4.0 is planned to replace CoreScope at **https://live.meshcore.ca**. +**https://dev.meshcore.ca** remains online for development testing, with its own +Beacon database, cache, configuration and application deployment. Production must +not depend on the development API or WebSocket endpoint. + +The server retains its independent version history (v1.6.0 is already released). +Record the matched server revision and approved image digest alongside web 1.4.0; +do not retag or downgrade the server to 1.4.0. Follow the +[1.4.0 release and cutover plan](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/app_documentation/release-140-preparation.md) +for migration checks, data continuity and the owner-controlled CoreScope switch. + +Development builds publish `dev` and revision tags. Only stable semantic-version +tags publish `latest`; pushes to `dev` or `main`, and prerelease tags, do not move +that production alias. Production deployments should pin the approved image digest. + For a bounded private database and saved-config bundle, see [backup export](docs/backup-export.md). A standalone export tool is available; the protected download API is opt-in; browser login and import remain separate follow-ups. From 765408e7fced2dbb2d023ef23eeafc4c3ca11ee7 Mon Sep 17 00:00:00 2001 From: n30nex Date: Wed, 30 Sep 2026 16:53:29 -0400 Subject: [PATCH 108/141] test: drain readiness probes before packet assertions --- internal/ingest/packet_summary_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/ingest/packet_summary_test.go b/internal/ingest/packet_summary_test.go index 887c3960..7f17aba2 100644 --- a/internal/ingest/packet_summary_test.go +++ b/internal/ingest/packet_summary_test.go @@ -6,6 +6,7 @@ package ingest import ( "context" "crypto/ed25519" + "crypto/rand" "encoding/json" "testing" "time" @@ -24,7 +25,20 @@ func waitForSummarySubscriber(t *testing.T, ctx context.Context, h *hub.Hub, cli h.Broadcast(hub.Event{Type: hub.EventObserverStatus}) select { case <-client.Send: - return + // A slow hub can leave more probes queued. Drain through a final + // marker so the next test read receives its packet, not a probe. + marker := json.RawMessage(`"` + rand.Text() + `"`) + h.Broadcast(hub.Event{Type: hub.EventObserverStatus, Payload: marker}) + for { + select { + case event := <-client.Send: + if string(event.Payload) == string(marker) { + return + } + case <-ctx.Done(): + t.Fatal("hub readiness probes did not drain") + } + } case <-ticker.C: case <-ctx.Done(): t.Fatal("hub subscription did not become ready") From da6de40b58bd10e96b345b0573324d87e35c43bb Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 19:31:44 -0400 Subject: [PATCH 109/141] ci: tie latest image tag to main Drop the deployment-specific rollout notes from the README. --- .github/workflows/docker-publish.yml | 3 +++ README.md | 17 ----------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 686b3d98..0cc526c8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -40,7 +40,10 @@ jobs: org.opencontainers.image.title=beacon-server org.opencontainers.image.description=Beacon — MeshCore LoRa mesh observation server org.opencontainers.image.licenses=AGPL-3.0-or-later + flavor: | + latest=false tags: | + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} type=raw,value=dev,enable=${{ github.ref == 'refs/heads/dev' }} type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} diff --git a/README.md b/README.md index 6e07bc35..dfcec34f 100644 --- a/README.md +++ b/README.md @@ -27,23 +27,6 @@ in PostgreSQL, and streams live events to WebSocket clients. For deployment instructions including the frontend app, see the deployment docs. -### MeshCore Canada: Beacon 1.4.0 rollout - -Beacon/web 1.4.0 is planned to replace CoreScope at **https://live.meshcore.ca**. -**https://dev.meshcore.ca** remains online for development testing, with its own -Beacon database, cache, configuration and application deployment. Production must -not depend on the development API or WebSocket endpoint. - -The server retains its independent version history (v1.6.0 is already released). -Record the matched server revision and approved image digest alongside web 1.4.0; -do not retag or downgrade the server to 1.4.0. Follow the -[1.4.0 release and cutover plan](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/app_documentation/release-140-preparation.md) -for migration checks, data continuity and the owner-controlled CoreScope switch. - -Development builds publish `dev` and revision tags. Only stable semantic-version -tags publish `latest`; pushes to `dev` or `main`, and prerelease tags, do not move -that production alias. Production deployments should pin the approved image digest. - For a bounded private database and saved-config bundle, see [backup export](docs/backup-export.md). A standalone export tool is available; the protected download API is opt-in; browser login and import remain separate follow-ups. From 3ac4035942430ca43676e284899d2e1200bb1370 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 21:33:34 -0400 Subject: [PATCH 110/141] refactor(db): flatten migrations into a 2.0.0 baseline 2.0.0 is a fresh deploy, so the 1.x chain becomes one schema file. Startup now refuses a pre-2.0.0 database instead of applying the baseline on top. --- .github/workflows/ci.yml | 2 +- CONTRIBUTING.md | 6 +- db/accounts_integration_test.go | 8 +- db/analytics_retention_integration_test.go | 68 +- db/baseline_integration_test.go | 119 ++ db/channel_cursor_integration_test.go | 6 +- db/channel_index_migration_test.go | 74 - db/migrate.go | 53 +- db/migrations/001_baseline.sql | 1314 +++++++++++++++++ db/migrations/001_initial_schema.sql | 439 ------ db/migrations/002_drop_single_hop_routes.sql | 7 - db/migrations/003_tag_ping_traces.sql | 9 - .../004_clean_non_infra_neighbors.sql | 11 - db/migrations/005_add_neighbor_snr.sql | 5 - .../006_fix_observation_dedup_key.sql | 24 - .../007_dedupe_hashonly_channels.sql | 21 - .../008_packets_last_heard_index.sql | 8 - .../009_clean_zeroed_observer_telemetry.sql | 19 - db/migrations/010_add_node_clock_drift.sql | 16 - db/migrations/011_channel_iatas.sql | 23 - db/migrations/012_trace_iatas.sql | 23 - db/migrations/013_packets_scope_index.sql | 3 - .../014_known_routes_last_seen_index.sql | 3 - .../015_observations_payload_type.sql | 17 - db/migrations/016_mv_payload_breakdown.sql | 17 - db/migrations/017_mv_top_observers.sql | 19 - db/migrations/018_mv_top_talkers.sql | 19 - db/migrations/019_mv_top_advertisers.sql | 22 - db/migrations/020_add_iata_border.sql | 21 - .../021_mv_top_advertisers_route_type.sql | 32 - db/migrations/022_neighbor_scopes.sql | 20 - .../023_channel_messages_cascade.sql | 12 - db/migrations/024_known_routes_pathkey.sql | 43 - .../026_observation_endpoint_snapshots.sql | 9 - .../027_known_routes_iata_last_seen.sql | 9 - db/migrations/028_channels_last_seen_id.sql | 8 - db/migrations/029_drop_channels_last_seen.sql | 6 - db/migrations/030_observation_airtime.sql | 2 - .../031_observation_airtime_backfill.sql | 30 - db/migrations/032_mv_observer_activity.sql | 24 - db/migrations/033_telemetry_airtime_secs.sql | 4 - db/migrations/034_accounts.sql | 12 - db/migrations/035_mv_signal_stats.sql | 33 - db/migrations/036_mv_path_stats.sql | 31 - db/migrations/037_autovacuum_tuning.sql | 14 - db/migrations/038_drop_resolved_endpoints.sql | 6 - db/migrations/039_analytics_retention.sql | 613 -------- db/migrations/041_route_evidence_index.sql | 9 - db/migrations/042_meshmapper_scopes.sql | 17 - db/migrations/043_clear_zero_locations.sql | 6 - db/migrations/044_meshmapper_zones.sql | 15 - .../045_clean_partial_observer_telemetry.sql | 7 - db/observer_metrics_integration_test.go | 22 +- db/paths_integration_test.go | 26 +- db/retention_integration_test.go | 4 +- db/route_evidence_integration_test.go | 95 +- db/routes_integration_test.go | 14 +- db/signal_integration_test.go | 19 +- db/stats_migration_integration_test.go | 33 - internal/config/border.go | 2 +- 60 files changed, 1553 insertions(+), 2000 deletions(-) create mode 100644 db/baseline_integration_test.go delete mode 100644 db/channel_index_migration_test.go create mode 100644 db/migrations/001_baseline.sql delete mode 100644 db/migrations/001_initial_schema.sql delete mode 100644 db/migrations/002_drop_single_hop_routes.sql delete mode 100644 db/migrations/003_tag_ping_traces.sql delete mode 100644 db/migrations/004_clean_non_infra_neighbors.sql delete mode 100644 db/migrations/005_add_neighbor_snr.sql delete mode 100644 db/migrations/006_fix_observation_dedup_key.sql delete mode 100644 db/migrations/007_dedupe_hashonly_channels.sql delete mode 100644 db/migrations/008_packets_last_heard_index.sql delete mode 100644 db/migrations/009_clean_zeroed_observer_telemetry.sql delete mode 100644 db/migrations/010_add_node_clock_drift.sql delete mode 100644 db/migrations/011_channel_iatas.sql delete mode 100644 db/migrations/012_trace_iatas.sql delete mode 100644 db/migrations/013_packets_scope_index.sql delete mode 100644 db/migrations/014_known_routes_last_seen_index.sql delete mode 100644 db/migrations/015_observations_payload_type.sql delete mode 100644 db/migrations/016_mv_payload_breakdown.sql delete mode 100644 db/migrations/017_mv_top_observers.sql delete mode 100644 db/migrations/018_mv_top_talkers.sql delete mode 100644 db/migrations/019_mv_top_advertisers.sql delete mode 100644 db/migrations/020_add_iata_border.sql delete mode 100644 db/migrations/021_mv_top_advertisers_route_type.sql delete mode 100644 db/migrations/022_neighbor_scopes.sql delete mode 100644 db/migrations/023_channel_messages_cascade.sql delete mode 100644 db/migrations/024_known_routes_pathkey.sql delete mode 100644 db/migrations/026_observation_endpoint_snapshots.sql delete mode 100644 db/migrations/027_known_routes_iata_last_seen.sql delete mode 100644 db/migrations/028_channels_last_seen_id.sql delete mode 100644 db/migrations/029_drop_channels_last_seen.sql delete mode 100644 db/migrations/030_observation_airtime.sql delete mode 100644 db/migrations/031_observation_airtime_backfill.sql delete mode 100644 db/migrations/032_mv_observer_activity.sql delete mode 100644 db/migrations/033_telemetry_airtime_secs.sql delete mode 100644 db/migrations/034_accounts.sql delete mode 100644 db/migrations/035_mv_signal_stats.sql delete mode 100644 db/migrations/036_mv_path_stats.sql delete mode 100644 db/migrations/037_autovacuum_tuning.sql delete mode 100644 db/migrations/038_drop_resolved_endpoints.sql delete mode 100644 db/migrations/039_analytics_retention.sql delete mode 100644 db/migrations/041_route_evidence_index.sql delete mode 100644 db/migrations/042_meshmapper_scopes.sql delete mode 100644 db/migrations/043_clear_zero_locations.sql delete mode 100644 db/migrations/044_meshmapper_zones.sql delete mode 100644 db/migrations/045_clean_partial_observer_telemetry.sql delete mode 100644 db/stats_migration_integration_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dc16322..95527aa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketSummaries|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes|NodeLocationReset|RunMigrations[A-Za-z]*|Reconfirm[A-Za-z]*)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7513a94d..f859d2fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,8 +79,8 @@ swag init # if you changed any handler or api type (see below) All schema changes must include a proper migration path: - Add a new migration file to `db/migrations/` following the existing naming - convention (e.g. `002_add_observation_count.sql`). Do not modify existing - migration files — append only via new files. + convention (e.g. `002_add_observation_count.sql`). `001_baseline.sql` is the + 2.0.0 schema; do not modify existing migration files — append only via new files. - Update `db/queries/queries.sql` with any new or modified queries - Re-run `sqlc generate` to regenerate `db/sqlc/` - Update the store layer in `db/` to expose the new functionality @@ -239,7 +239,7 @@ Scopes are optional but helpful for larger codebases. Common scopes: `api`, ``` cmd/beacon/ — main entry point, wiring, startup db/ — store layer: sqlc-generated code + thin mapping layer - migrations/ — SQL schema (single file, append only) + migrations/ — SQL schema (2.0.0 baseline + numbered files, append only) queries/ — SQL queries (input to sqlc) sqlc/ — generated Go code (do not edit by hand) internal/ diff --git a/db/accounts_integration_test.go b/db/accounts_integration_test.go index 02700506..a5426270 100644 --- a/db/accounts_integration_test.go +++ b/db/accounts_integration_test.go @@ -50,15 +50,9 @@ func TestAccountsPostgres(t *testing.T) { t.Fatal(err) } defer pool.Close() - sql, err := migrationFiles.ReadFile("migrations/034_accounts.sql") - if err != nil { + if err := RunMigrations(ctx, pool); err != nil { t.Fatal(err) } - for i := 0; i < 2; i++ { - if err := applyMigration(ctx, pool, string(sql)); err != nil { - t.Fatal(err) - } - } store := New(pool, 0, 0) items, err := store.ListAccounts(ctx) if err != nil || items == nil || len(items) != 0 { diff --git a/db/analytics_retention_integration_test.go b/db/analytics_retention_integration_test.go index 2bfbdfe6..4306b5a5 100644 --- a/db/analytics_retention_integration_test.go +++ b/db/analytics_retention_integration_test.go @@ -16,13 +16,20 @@ import ( var retainedViews = []string{"mv_hourly_iata_stats", "mv_payload_breakdown_by_iata", "mv_top_observers_by_iata", "mv_top_talkers_by_iata", "mv_top_advertisers_by_iata", "mv_observer_activity_hourly", "mv_signal_stats_hourly", "mv_path_stats_hourly"} -func analyticsSnapshot(t *testing.T, ctx context.Context, tx pgx.Tx) map[string]string { +func refreshRetainedViews(t *testing.T, ctx context.Context, tx pgx.Tx) { t.Helper() - result := map[string]string{} for _, view := range retainedViews { if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW "+view); err != nil { t.Fatal(err) } + } +} + +func analyticsSnapshot(t *testing.T, ctx context.Context, tx pgx.Tx) map[string]string { + t.Helper() + refreshRetainedViews(t, ctx, tx) + result := map[string]string{} + for _, view := range retainedViews { var rows string if err := tx.QueryRow(ctx, "SELECT COALESCE(jsonb_agg(r ORDER BY r::text),'[]')::text FROM (SELECT to_jsonb(v) r FROM "+view+" v) s").Scan(&rows); err != nil { t.Fatal(err) @@ -34,9 +41,8 @@ func analyticsSnapshot(t *testing.T, ctx context.Context, tx pgx.Tx) map[string] func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { ctx, tx := retentionTx(t) - analyticsTables(t, ctx, tx) - applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") - if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash,last_heard_at) VALUES ('\x01',NOW()-interval '5 days'); INSERT INTO observers(id) VALUES ('00000000-0000-0000-0000-000000000001')`); err != nil { + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) VALUES ('\x01',4,0,1,'\x00','\x00',NOW()-interval '5 days',NOW()-interval '5 days'); INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01')`); err != nil { t.Fatal(err) } var schema string @@ -70,7 +76,7 @@ func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { } defer ingest.Rollback(context.Background()) // An in-flight FK insert holds KEY SHARE on the packet until it commits. - if _, err = ingest.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\x01','00000000-0000-0000-0000-000000000001','YVR',NOW()-interval '5 days')`); err != nil { + if _, err = ingest.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES ('\x01','00000000-0000-0000-0000-000000000001','YVR',NOW()-interval '5 days',0,1,0)`); err != nil { t.Fatal(err) } q := sqlc.New(conn) @@ -114,33 +120,11 @@ func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { } } -// Minimal real tables keep this regression runnable against an empty CI database. -func analyticsTables(t *testing.T, ctx context.Context, tx pgx.Tx) { - t.Helper() - isolateStatsSchema(t, ctx, tx) - _, err := tx.Exec(ctx, ` -CREATE TABLE packets (packet_hash bytea PRIMARY KEY, payload_type smallint, payload_version smallint, - route_type smallint, raw_payload bytea, raw_header bytea, origin_pubkey bytea, - first_heard_at timestamptz, last_heard_at timestamptz); -CREATE INDEX ON packets(last_heard_at); -CREATE TABLE packet_observations (id bigserial PRIMARY KEY, packet_hash bytea REFERENCES packets ON DELETE CASCADE, - observer_id uuid NOT NULL, iata char(3) NOT NULL, heard_at timestamptz NOT NULL, - path_length_byte smallint, hash_size smallint, hop_count smallint, path_bytes bytea, - snr real, rssi smallint, airtime_ms real, payload_type smallint, UNIQUE(packet_hash,observer_id)); -CREATE TABLE observers (id uuid PRIMARY KEY, public_key bytea, display_name text, observer_type text); -CREATE TABLE nodes (id uuid PRIMARY KEY, public_key bytea UNIQUE, node_type smallint, name text); -CREATE TABLE channel_messages (id bigserial PRIMARY KEY, channel_id integer, packet_hash bytea UNIQUE REFERENCES packets ON DELETE CASCADE, - sender_name text, content text, sent_at timestamptz); -`) - if err != nil { - t.Fatal(err) - } -} - func TestAnalyticsRetentionPostgres(t *testing.T) { ctx, tx := retentionTx(t) - analyticsTables(t, ctx, tx) + applyBaseline(t, ctx, tx) _, err := tx.Exec(ctx, ` + INSERT INTO channels (id,channel_hash) VALUES (1,'\x01'); INSERT INTO observers (id,public_key,display_name,observer_type) SELECT ('00000000-0000-0000-0000-00000000000'||i)::uuid,int4send(i),CASE WHEN i=1 THEN NULL ELSE 'Observer '||i END,CASE WHEN i=1 THEN NULL ELSE 'test' END FROM generate_series(1,3) i; INSERT INTO nodes (id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2,'Repeater'); INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) @@ -152,22 +136,17 @@ func TestAnalyticsRetentionPostgres(t *testing.T) { FROM packets CROSS JOIN (VALUES ('YVR',1),('YVR',2),('YYZ',3)) v(iata,n); INSERT INTO channel_messages (channel_id,packet_hash,sender_name,content,sent_at) SELECT 1,packet_hash,'Sender','body that must expire',first_heard_at FROM packets; - CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS SELECT iata,date_trunc('hour',heard_at) AS hour,count(*) AS observation_count,count(DISTINCT packet_hash) AS unique_packets,count(DISTINCT observer_id) AS active_observers FROM packet_observations GROUP BY 1,2; `) if err != nil { t.Fatal(err) } - for _, migration := range []string{"016_mv_payload_breakdown.sql", "017_mv_top_observers.sql", "018_mv_top_talkers.sql", "019_mv_top_advertisers.sql", "021_mv_top_advertisers_route_type.sql", "032_mv_observer_activity.sql", "035_mv_signal_stats.sql", "036_mv_path_stats.sql"} { - applyStatsMigration(t, ctx, tx, migration) - } - applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") for _, name := range []string{"idx_mv_signal_stats_hourly_hour", "idx_mv_path_stats_hourly_hour"} { var definition string if err := tx.QueryRow(ctx, "SELECT pg_get_indexdef(to_regclass($1))", name).Scan(&definition); err != nil || !strings.Contains(definition, "(hour)") { t.Fatalf("missing unfiltered time index %s: %s %v", name, definition, err) } } - // The signal/path stores must read the 039 view definitions, not just 035/036. + refreshRetainedViews(t, ctx, tx) store := &Store{q: sqlc.New(tx)} hour := time.Now().UTC().Truncate(time.Hour) signal, err := store.GetSignalStats(ctx, hour.Add(-6*24*time.Hour), hour, nil) @@ -176,14 +155,14 @@ func TestAnalyticsRetentionPostgres(t *testing.T) { } // 12 observations; only the n=1 rows carry a signal (snr 0, rssi -100). if signal.Receptions != 12 || signal.SNR.Samples != 4 || signal.RSSI.Samples != 4 { - t.Fatalf("signal stats on 039 views: %+v", signal) + t.Fatalf("signal stats: %+v", signal) } paths, err := store.GetPathStats(ctx, hour.Add(-6*24*time.Hour), hour, nil) if err != nil { t.Fatal(err) } if paths.Receptions != 12 { - t.Fatalf("path stats on 039 views: %+v", paths) + t.Fatalf("path stats: %+v", paths) } before := analyticsSnapshot(t, ctx, tx) q := sqlc.New(tx) @@ -216,18 +195,17 @@ func TestAnalyticsRetentionPostgres(t *testing.T) { t.Errorf("%s lost or double-counted history after raw expiry: before=%s after=%s", view, before[view], after[view]) } } - // Retry and migration journal retry preserve the exact summaries. + // A retry preserves the exact summaries. if n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1000}); err != nil || n != 0 { t.Fatalf("retry: %d %v", n, err) } - applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") for view, rows := range analyticsSnapshot(t, ctx, tx) { if rows != before[view] { t.Errorf("retry changed %s", view) } } // Later receptions on a still-live packet must join its old bucket before expiry. - if _, err := tx.Exec(ctx, `INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000004'); INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,payload_type) + if _, err := tx.Exec(ctx, `INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000004','\x04'); INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,payload_type) SELECT packet_hash,'00000000-0000-0000-0000-000000000004','YVR',first_heard_at,0,1,0,'NaN',-80,NULL FROM packets; UPDATE packets SET last_heard_at=first_heard_at`); err != nil { t.Fatal(err) @@ -282,13 +260,13 @@ func TestAnalyticsRetentionPostgres(t *testing.T) { } // More than one populated cohort, with shared hourly keys across the boundary. if _, err := tx.Exec(ctx, ` -INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000001'); +INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); INSERT INTO nodes (id,public_key,node_type) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2); -INSERT INTO packets (packet_hash,payload_type,route_type,origin_pubkey,first_heard_at,last_heard_at,raw_payload) -SELECT int4send(i),4,1,'\x02',date_trunc('hour',NOW())-interval '5 days',date_trunc('hour',NOW())-interval '5 days',decode(repeat('ab',1024),'hex') FROM generate_series(10000,11000) i; +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,origin_pubkey,first_heard_at,last_heard_at,raw_payload,raw_header) +SELECT int4send(i),4,0,1,'\x02',date_trunc('hour',NOW())-interval '5 days',date_trunc('hour',NOW())-interval '5 days',decode(repeat('ab',1024),'hex'),'\x00' FROM generate_series(10000,11000) i; INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YVR',first_heard_at,0,1,0,4,0,-100 FROM packets; -INSERT INTO channel_messages (packet_hash,sender_name,sent_at) SELECT packet_hash,'Sender',first_heard_at FROM packets; +INSERT INTO channel_messages (channel_id,packet_hash,sender_name,sent_at) SELECT 1,packet_hash,'Sender',first_heard_at FROM packets; `); err != nil { t.Fatal(err) } diff --git a/db/baseline_integration_test.go b/db/baseline_integration_test.go new file mode 100644 index 00000000..e879c493 --- /dev/null +++ b/db/baseline_integration_test.go @@ -0,0 +1,119 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Views cannot reference temporary tables. Keep fixtures and real migration DDL +// in a unique schema inside the caller's rolled-back transaction instead. +func isolateStatsSchema(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + schema := pgx.Identifier{"stats_test_" + uuid.NewString()}.Sanitize() + if _, err := tx.Exec(ctx, "CREATE SCHEMA "+schema+"; SET LOCAL search_path TO "+schema); err != nil { + t.Fatal(err) + } +} + +// applyBaseline builds the real schema in an isolated schema, plus the IATAs fixtures use. +func applyBaseline(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + isolateStatsSchema(t, ctx, tx) + ddl, err := migrationFiles.ReadFile("migrations/" + baselineMigration) + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, string(ddl)); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, "INSERT INTO iata_codes (iata) VALUES ('YVR'),('YYJ'),('YYZ'),('YOW')"); err != nil { + t.Fatal(err) + } +} + +// schemaPool returns a pool whose connections all use a fresh, dropped-on-cleanup schema. +func schemaPool(t *testing.T) (context.Context, *pgxpool.Pool) { + t.Helper() + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + setup, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + schema := "migrate_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + quoted := pgx.Identifier{schema}.Sanitize() + if _, err := setup.Exec(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := setup.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error(err) + } + setup.Close(context.Background()) + }) + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + t.Fatal(err) + } + cfg.ConnConfig.RuntimeParams["search_path"] = schema + cfg.MaxConns = 4 + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + return ctx, pool +} + +func TestRunMigrationsBaselinePostgres(t *testing.T) { + ctx, pool := schemaPool(t) + for range 2 { + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + } + rows, err := pool.Query(ctx, "SELECT filename FROM schema_migrations ORDER BY filename") + if err != nil { + t.Fatal(err) + } + ledger, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil || len(ledger) != 1 || ledger[0] != baselineMigration { + t.Fatalf("ledger %v, %v", ledger, err) + } + // Stats refresh uses CONCURRENTLY, which needs populated views. + if _, err := pool.Exec(ctx, "REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hourly_iata_stats"); err != nil { + t.Fatal(err) + } +} + +func TestRunMigrationsRefusesPreBaselinePostgres(t *testing.T) { + for name, setup := range map[string]string{ + "1.x ledger": "CREATE TABLE schema_migrations (filename text PRIMARY KEY); INSERT INTO schema_migrations VALUES ('001_initial_schema.sql')", + "pre-ledger 1.x": "CREATE TABLE packets (packet_hash bytea PRIMARY KEY)", + } { + t.Run(name, func(t *testing.T) { + ctx, pool := schemaPool(t) + if _, err := pool.Exec(ctx, setup); err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); !errors.Is(err, errPreBaseline) { + t.Fatalf("got %v, want errPreBaseline", err) + } + }) + } +} diff --git a/db/channel_cursor_integration_test.go b/db/channel_cursor_integration_test.go index 8f55844a..d748f2aa 100644 --- a/db/channel_cursor_integration_test.go +++ b/db/channel_cursor_integration_test.go @@ -56,10 +56,8 @@ func TestChannelCursorPostgres(t *testing.T) { t.Fatal(err) } } - exec(`CREATE TEMP TABLE channels (LIKE public.channels INCLUDING ALL) ON COMMIT DROP; -CREATE TEMP TABLE channel_iatas (LIKE public.channel_iatas INCLUDING ALL) ON COMMIT DROP; -CREATE INDEX ON channels(last_seen DESC,id DESC); -INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) VALUES + applyBaseline(t, ctx, tx) + exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) VALUES (1,'\xaa','\x01','2026-09-08 12:00:00+00'),(2,'\xaa','\x02','2026-09-08 12:00:00+00'), (3,'\xaa','\x03','2026-09-08 12:00:00+00'),(4,'\xbb','\x04','2026-09-08 11:59:59+00'), (5,'\xcc','\x05','2026-09-08 12:00:00.000321+00'), diff --git a/db/channel_index_migration_test.go b/db/channel_index_migration_test.go deleted file mode 100644 index 4e0fb02b..00000000 --- a/db/channel_index_migration_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later - -package db - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/jackc/pgx/v5" -) - -func TestChannelIndexMigrationsPostgres(t *testing.T) { - dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") - if dsn == "" { - t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") - } - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - conn, err := pgx.Connect(ctx, dsn) - if err != nil { - t.Fatal(err) - } - defer conn.Close(context.Background()) - exec := func(query string) { - t.Helper() - if _, err := conn.Exec(ctx, query); err != nil { - t.Fatal(err) - } - } - schema := pgx.Identifier{fmt.Sprintf("channel_index_test_%d", time.Now().UnixNano())}.Sanitize() - exec("CREATE SCHEMA " + schema) - defer func() { - cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if _, err := conn.Exec(cleanup, "DROP SCHEMA "+schema+" CASCADE"); err != nil { - t.Error(err) - } - }() - exec("SET search_path TO " + schema) - exec(`CREATE TABLE channels (id bigint PRIMARY KEY, last_seen timestamptz NOT NULL); -CREATE INDEX idx_channels_last_seen ON channels(last_seen DESC); -INSERT INTO channels VALUES (1, '2026-09-09 12:00:00+00');`) - migrate := func(name string) { - t.Helper() - query, err := migrationFiles.ReadFile("migrations/" + name) - if err != nil { - t.Fatal(err) - } - exec(string(query)) // Concurrent DDL must remain outside an explicit transaction. - } - checkReplacement := func() { - t.Helper() - var valid bool - if err := conn.QueryRow(ctx, `SELECT indisvalid AND indisready AND indnkeyatts=2 -FROM pg_index WHERE indexrelid='idx_channels_last_seen_id'::regclass`).Scan(&valid); err != nil || !valid { - t.Fatalf("replacement index valid=%v, err=%v", valid, err) - } - } - migrate("028_channels_last_seen_id.sql") - checkReplacement() - for range 2 { // Retry after the drop succeeds but before its migration record is saved. - migrate("029_drop_channels_last_seen.sql") - var preserved bool - if err := conn.QueryRow(ctx, `SELECT to_regclass('idx_channels_last_seen') IS NULL -AND (SELECT count(*) FROM channels)=1`).Scan(&preserved); err != nil || !preserved { - t.Fatalf("old index removed and row preserved=%v, err=%v", preserved, err) - } - checkReplacement() - } -} diff --git a/db/migrate.go b/db/migrate.go index 7aa67255..560c02e6 100644 --- a/db/migrate.go +++ b/db/migrate.go @@ -82,6 +82,28 @@ func applyMigration(ctx context.Context, db execQuerier, sql string) error { return err } +const baselineMigration = "001_baseline.sql" + +var errPreBaseline = errors.New("database schema predates Beacon 2.0.0; 2.0.0 needs a fresh database") + +// checkBaseline refuses databases built by the 1.x migration chain, which the +// 2.0.0 baseline replaced; applying it on top would collide with existing tables. +func checkBaseline(ctx context.Context, db execQuerier) error { + var ledger, hasBaseline, hasPackets bool + err := db.QueryRow(ctx, ` + SELECT EXISTS(SELECT 1 FROM schema_migrations), + EXISTS(SELECT 1 FROM schema_migrations WHERE filename = $1), + to_regclass('packets') IS NOT NULL`, baselineMigration, + ).Scan(&ledger, &hasBaseline, &hasPackets) + if err != nil { + return fmt.Errorf("failed to check migration baseline: %w", err) + } + if (ledger && !hasBaseline) || (!ledger && hasPackets) { + return errPreBaseline + } + return nil +} + func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { _, err := pool.Exec(ctx, ` CREATE TABLE IF NOT EXISTS schema_migrations ( @@ -93,35 +115,8 @@ func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { return fmt.Errorf("failed to create schema_migrations: %w", err) } - // After creating schema_migrations table, check if we need to bootstrap - var count int - err = pool.QueryRow(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&count) - if err != nil { - return fmt.Errorf("failed to check migrations count: %w", err) - } - if count == 0 { - // Check if db is already initialized by looking for a known table - var exists bool - err = pool.QueryRow(ctx, ` - SELECT EXISTS( - SELECT 1 FROM information_schema.tables - WHERE table_name = 'packets' - ) - `).Scan(&exists) - if err != nil { - return fmt.Errorf("failed to check existing schema: %w", err) - } - if exists { - // Mark 001 as already applied - if _, err := pool.Exec( - ctx, - "INSERT INTO schema_migrations (filename) VALUES ($1)", - "001_initial_schema.sql", - ); err != nil { - return fmt.Errorf("failed to bootstrap migrations: %w", err) - } - slog.Info("bootstrapped existing schema as 001_initial_schema.sql", "component", "db") - } + if err := checkBaseline(ctx, pool); err != nil { + return err } entries, err := fs.ReadDir(migrationFiles, "migrations") diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql new file mode 100644 index 00000000..b858627c --- /dev/null +++ b/db/migrations/001_baseline.sql @@ -0,0 +1,1314 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- 2.0.0 schema baseline: pg_dump of a fresh database migrated through the 1.x +-- chain (001_initial_schema..045). Add new numbered migrations after this file. + +CREATE FUNCTION archive_delete_packets(cutoff timestamp with time zone, batch_size integer) RETURNS bigint + LANGUAGE plpgsql + AS $$ +DECLARE hashes bytea[]; deleted bigint; +BEGIN + -- Serialize cleanup runners before overlapping aggregate upserts. + PERFORM pg_advisory_xact_lock(hashtext('beacon.archive_delete_packets')); + IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; + SELECT array_agg(p.packet_hash) INTO hashes FROM ( + SELECT ep.packet_hash FROM packets ep + WHERE ep.last_heard_at < cutoff + ORDER BY ep.last_heard_at, ep.packet_hash + LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED + ) p; + + WITH expired_observations AS MATERIALIZED ( + SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) + ), +archived_hourly_iata_stats AS ( + INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) + SELECT iata, hour, observation_count, unique_packets FROM ( +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,hour) DO UPDATE SET + observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, + unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets +), +archived_payload_breakdown_by_iata AS ( + INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) + SELECT iata, payload_type, bucket, count FROM ( +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET + count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count +), +archived_top_observers_by_iata AS ( + INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM expired_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET + observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, + display_name = EXCLUDED.display_name, + observer_type = EXCLUDED.observer_type +), +archived_top_talkers_by_iata AS ( + INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) + SELECT iata, sender_name, bucket, message_count, last_sent FROM ( +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN expired_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') + ) batch + ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET + message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, + last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) +), +archived_top_advertisers_by_iata AS ( + INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN expired_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,node_id,bucket) DO UPDATE SET + advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, + flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, + direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, + last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), + name = EXCLUDED.name, + node_type = EXCLUDED.node_type +), +archived_observer_activity_hourly AS ( + INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET + observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, + airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, + airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, + snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, + snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, + snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), + rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, + rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n +), +archived_signal_stats_hourly AS ( + INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) + ) batch + ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET + receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, + snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, + snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, + rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, + rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum +), +archived_path_stats_hourly AS ( + INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) + SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries + ) batch + ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET + receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions +) + DELETE FROM packets WHERE packet_hash = ANY(hashes); + GET DIAGNOSTICS deleted = ROW_COUNT; + + -- Once per completed cleanup, including when there were no raw packets to delete. + IF deleted < batch_size THEN + DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + END IF; + RETURN deleted; +END $$; + +CREATE TABLE accounts ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + name text NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + deactivated_at timestamp with time zone, + CONSTRAINT accounts_name_check CHECK ((((char_length(name) >= 1) AND (char_length(name) <= 128)) AND (name = btrim(name)))) +); + +CREATE TABLE analytics_hourly_iata_stats ( + iata character(3) NOT NULL, + hour timestamp with time zone NOT NULL, + observation_count bigint DEFAULT 0 NOT NULL, + unique_packets bigint DEFAULT 0 NOT NULL +); + +CREATE TABLE packet_observations ( + id bigint NOT NULL, + packet_hash bytea NOT NULL, + observer_id uuid NOT NULL, + iata character(3) NOT NULL, + heard_at timestamp with time zone NOT NULL, + path_length_byte smallint NOT NULL, + hash_size smallint NOT NULL, + hop_count smallint NOT NULL, + path_bytes bytea, + rssi smallint, + snr real, + propagation_time_ms integer, + radio_freq_mhz real, + spread_factor smallint, + bandwidth_khz real, + coding_rate smallint, + source_broker text, + payload_type smallint, + airtime_ms real +) +WITH (autovacuum_vacuum_scale_factor='0.05', autovacuum_vacuum_cost_limit='1000'); + +CREATE VIEW analytics_live_hourly_iata_stats AS +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC'); + +CREATE VIEW analytics_live_observer_activity_hourly AS +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); + +CREATE VIEW analytics_live_path_stats_hourly AS +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; + +CREATE VIEW analytics_live_payload_breakdown_by_iata AS +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC'); + +CREATE VIEW analytics_live_signal_stats_hourly AS +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); + +CREATE TABLE nodes ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + public_key bytea NOT NULL, + node_type smallint NOT NULL, + name text, + latitude double precision, + longitude double precision, + location_source text, + last_advert_at timestamp with time zone, + supports_multibyte_paths boolean DEFAULT false NOT NULL, + supports_multibyte_traces boolean DEFAULT false NOT NULL, + default_scope_id integer, + min_firmware_version text GENERATED ALWAYS AS ( +CASE + WHEN supports_multibyte_paths THEN '1.14.0+'::text + WHEN supports_multibyte_traces THEN '1.11.0+'::text + ELSE NULL::text +END) STORED, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + radio_freq_mhz real, + radio_sf smallint, + radio_bw_khz real, + metadata jsonb, + device_clock_drift_seconds integer +) +WITH (autovacuum_vacuum_scale_factor='0.05'); + +CREATE TABLE packets ( + packet_hash bytea NOT NULL, + payload_type smallint NOT NULL, + payload_version smallint NOT NULL, + route_type smallint NOT NULL, + transport_codes_present boolean DEFAULT false, + region_code integer, + sub_region_code integer, + scope_id integer, + origin_pubkey bytea, + raw_payload bytea NOT NULL, + raw_header bytea NOT NULL, + parsed_payload jsonb, + decrypted boolean DEFAULT false, + channel_hash bytea, + trace_tag bytea, + first_heard_at timestamp with time zone NOT NULL, + last_heard_at timestamp with time zone NOT NULL +) +WITH (autovacuum_vacuum_scale_factor='0.05', autovacuum_vacuum_cost_limit='1000'); + +CREATE VIEW analytics_live_top_advertisers_by_iata AS +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN packet_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE TABLE observers ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + public_key bytea NOT NULL, + display_name text, + observer_type text, + software_version text, + hardware_model text, + firmware_version text, + firmware_build text, + radio_freq_mhz real, + radio_sf smallint, + radio_bw_khz real, + radio_cr smallint, + battery_level real, + uptime_seconds bigint, + status_metadata jsonb, + last_status_at timestamp with time zone, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + observation_count bigint DEFAULT 0, + metadata jsonb, + region_scope text +); + +CREATE VIEW analytics_live_top_observers_by_iata AS +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE TABLE channel_messages ( + id bigint NOT NULL, + channel_id integer NOT NULL, + packet_hash bytea NOT NULL, + sender_name text, + sender_pubkey bytea, + content text, + sent_at timestamp with time zone NOT NULL +); + +CREATE VIEW analytics_live_top_talkers_by_iata AS +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN packet_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC'); + +CREATE TABLE analytics_observer_activity_hourly ( + observer_id uuid NOT NULL, + payload_type smallint NOT NULL, + bucket timestamp with time zone NOT NULL, + observations bigint DEFAULT 0 NOT NULL, + airtime_ms real, + airtime_n bigint DEFAULT 0 NOT NULL, + snr_sum real, + snr_n bigint DEFAULT 0 NOT NULL, + snr_min real, + rssi_sum bigint, + rssi_n bigint DEFAULT 0 NOT NULL +); + +CREATE TABLE analytics_path_stats_hourly ( + iata character(3) NOT NULL, + hour timestamp with time zone NOT NULL, + category integer NOT NULL, + hash_bytes integer NOT NULL, + entries integer NOT NULL, + receptions bigint DEFAULT 0 NOT NULL +); + +CREATE TABLE analytics_payload_breakdown_by_iata ( + iata character(3) NOT NULL, + payload_type smallint NOT NULL, + bucket timestamp with time zone NOT NULL, + count bigint DEFAULT 0 NOT NULL +); + +CREATE TABLE analytics_signal_stats_hourly ( + iata character(3) NOT NULL, + hour timestamp with time zone NOT NULL, + kind integer NOT NULL, + snr_bin integer NOT NULL, + rssi_bin integer NOT NULL, + receptions bigint DEFAULT 0 NOT NULL, + snr_samples bigint DEFAULT 0 NOT NULL, + snr_sum double precision, + rssi_samples bigint DEFAULT 0 NOT NULL, + rssi_sum double precision +); + +CREATE TABLE analytics_top_advertisers_by_iata ( + iata character(3) NOT NULL, + node_id uuid NOT NULL, + bucket timestamp with time zone NOT NULL, + advert_count bigint DEFAULT 0 NOT NULL, + flood_advert_count bigint DEFAULT 0 NOT NULL, + direct_advert_count bigint DEFAULT 0 NOT NULL, + last_heard timestamp with time zone, + name text, + node_type smallint +); + +CREATE TABLE analytics_top_observers_by_iata ( + iata character(3) NOT NULL, + observer_id uuid NOT NULL, + bucket timestamp with time zone NOT NULL, + observation_count bigint DEFAULT 0 NOT NULL, + display_name text, + observer_type text +); + +CREATE TABLE analytics_top_talkers_by_iata ( + iata character(3) NOT NULL, + sender_name text NOT NULL, + bucket timestamp with time zone NOT NULL, + message_count bigint DEFAULT 0 NOT NULL, + last_sent timestamp with time zone +); + +CREATE TABLE channel_iatas ( + channel_hash bytea NOT NULL, + iata character(3) NOT NULL, + last_heard timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE channel_keys ( + channel_id integer NOT NULL, + key_bytes bytea NOT NULL, + key_fingerprint bytea NOT NULL, + added_at timestamp with time zone DEFAULT now() NOT NULL, + added_by text +); + +CREATE SEQUENCE channel_messages_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE channel_messages_id_seq OWNED BY channel_messages.id; + +CREATE TABLE channels ( + id integer NOT NULL, + channel_hash bytea NOT NULL, + key_fingerprint bytea, + name text, + hashtag text, + is_hashtag boolean DEFAULT false, + is_public boolean DEFAULT false, + key_known boolean DEFAULT false, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + message_count bigint DEFAULT 0 +); + +CREATE SEQUENCE channels_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE channels_id_seq OWNED BY channels.id; + +CREATE TABLE iata_codes ( + iata character(3) NOT NULL, + display_name text, + approx_lat double precision, + approx_lng double precision, + added_at timestamp with time zone DEFAULT now() NOT NULL, + border jsonb +); + +CREATE TABLE known_routes ( + id bigint NOT NULL, + path_key bytea NOT NULL, + node_ids uuid[] NOT NULL, + hash_prefix bytea[] NOT NULL, + iata character(3) NOT NULL, + hop_count integer NOT NULL, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + observation_count bigint DEFAULT 1 NOT NULL, + last_reconfirmed_at timestamp with time zone DEFAULT now() NOT NULL +) +WITH (autovacuum_vacuum_scale_factor='0.02', autovacuum_analyze_scale_factor='0.02', autovacuum_vacuum_cost_limit='1000'); + +ALTER TABLE known_routes ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( + SEQUENCE NAME known_routes_new_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1 +); + +CREATE TABLE meshmapper_scope_catalogues ( + iata text NOT NULL, + url text NOT NULL, + payload jsonb, + etag text, + checked_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + +CREATE TABLE meshmapper_zone_boundaries ( + iata text NOT NULL, + url text NOT NULL, + feature jsonb, + etag text, + checked_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + +CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS +WITH combined AS ( + SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats + UNION ALL + SELECT iata, hour, observation_count, unique_packets FROM analytics_hourly_iata_stats + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observers_by_hour AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, observer_id + FROM packet_observations WHERE heard_at > NOW() - INTERVAL '30 days' + UNION + SELECT iata, bucket AS hour, observer_id FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observer_counts AS (SELECT iata, hour, COUNT(*) AS n FROM observers_by_hour GROUP BY iata,hour) +SELECT combined.iata, combined.hour, SUM(observation_count)::bigint AS observation_count, SUM(unique_packets)::bigint AS unique_packets, MAX(o.n)::bigint AS active_observers +FROM combined JOIN observer_counts o USING (iata,hour) GROUP BY combined.iata,combined.hour; + +CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS +WITH combined AS ( + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly + UNION ALL + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n +FROM combined GROUP BY observer_id,payload_type,bucket; + +CREATE MATERIALIZED VIEW mv_path_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_live_path_stats_hourly + UNION ALL + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_path_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, category, hash_bytes, entries, SUM(receptions)::bigint AS receptions +FROM combined GROUP BY iata,hour,category,hash_bytes,entries; + +CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS +WITH combined AS ( + SELECT iata, payload_type, bucket, count FROM analytics_live_payload_breakdown_by_iata + UNION ALL + SELECT iata, payload_type, bucket, count FROM analytics_payload_breakdown_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, payload_type, bucket, SUM(count)::bigint AS count +FROM combined GROUP BY iata,payload_type,bucket; + +CREATE TABLE node_iatas ( + node_id uuid NOT NULL, + iata character(3) NOT NULL, + first_heard timestamp with time zone DEFAULT now() NOT NULL, + last_heard timestamp with time zone DEFAULT now() NOT NULL, + observation_count bigint DEFAULT 0 +) +WITH (autovacuum_vacuum_scale_factor='0.05'); + +CREATE MATERIALIZED VIEW mv_radio_presets AS +SELECT + concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text AS preset, + (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) AS iata, + 'observer' AS source_type, + COUNT(*) AS count +FROM observers o +WHERE o.radio_freq_mhz IS NOT NULL + AND o.radio_sf IS NOT NULL + AND o.radio_bw_khz IS NOT NULL +GROUP BY concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text, + (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) +HAVING (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) IS NOT NULL + +UNION ALL + +SELECT + concat(n.radio_freq_mhz, ',', n.radio_bw_khz, ',', n.radio_sf) AS preset, + ni.iata, + 'node' AS source_type, + COUNT(*) AS count +FROM nodes n +JOIN node_iatas ni ON ni.node_id = n.id +WHERE n.radio_freq_mhz IS NOT NULL + AND n.radio_sf IS NOT NULL + AND n.radio_bw_khz IS NOT NULL +GROUP BY concat(n.radio_freq_mhz, ',', n.radio_bw_khz, ',', n.radio_sf), ni.iata; + +CREATE MATERIALIZED VIEW mv_signal_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_live_signal_stats_hourly + UNION ALL + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_signal_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, kind, snr_bin, rssi_bin, SUM(receptions)::bigint AS receptions, SUM(snr_samples)::bigint AS snr_samples, SUM(snr_sum)::double precision AS snr_sum, SUM(rssi_samples)::bigint AS rssi_samples, SUM(rssi_sum)::double precision AS rssi_sum +FROM combined GROUP BY iata,hour,kind,snr_bin,rssi_bin; + +CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS +WITH combined AS ( + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_live_top_advertisers_by_iata + UNION ALL + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_top_advertisers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, node_id, bucket, SUM(advert_count)::bigint AS advert_count, SUM(flood_advert_count)::bigint AS flood_advert_count, SUM(direct_advert_count)::bigint AS direct_advert_count, MAX(last_heard) AS last_heard, MAX(name) AS name, MAX(node_type) AS node_type +FROM combined GROUP BY iata,node_id,bucket) +SELECT t.iata, t.node_id, COALESCE(n.name,t.name) AS name, COALESCE(n.node_type,t.node_type) AS node_type, + t.bucket, t.advert_count, t.flood_advert_count, t.direct_advert_count, t.last_heard +FROM totals t LEFT JOIN nodes n ON n.id=t.node_id; + +CREATE MATERIALIZED VIEW mv_top_nodes_by_iata AS +SELECT + ni.iata, + ni.node_id, + n.name, + n.node_type, + ni.observation_count, + ni.last_heard +FROM node_iatas ni +JOIN nodes n ON n.id = ni.node_id +WHERE ni.last_heard > NOW() - INTERVAL '7 days'; + +CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS +WITH combined AS ( + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_live_top_observers_by_iata + UNION ALL + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, observer_id, bucket, SUM(observation_count)::bigint AS observation_count, MAX(display_name) AS display_name, MAX(observer_type) AS observer_type +FROM combined GROUP BY iata,observer_id,bucket) +SELECT t.iata, t.observer_id, COALESCE(o.display_name,t.display_name) AS display_name, + COALESCE(o.observer_type,t.observer_type) AS observer_type, t.bucket, t.observation_count +FROM totals t LEFT JOIN observers o ON o.id=t.observer_id; + +CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS +WITH combined AS ( + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_live_top_talkers_by_iata + UNION ALL + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_top_talkers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, sender_name, bucket, SUM(message_count)::bigint AS message_count, MAX(last_sent) AS last_sent +FROM combined GROUP BY iata,sender_name,bucket; + +CREATE TABLE node_neighbors ( + node_id uuid NOT NULL, + neighbor_id uuid NOT NULL, + iata character(3) NOT NULL, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + observation_count bigint DEFAULT 1 NOT NULL, + snr real, + region_scope text +); + +CREATE TABLE node_short_ids ( + node_id uuid NOT NULL, + iata character(3) NOT NULL, + prefix_4 bytea NOT NULL, + prefix_1 bytea GENERATED ALWAYS AS (SUBSTRING(prefix_4 FROM 1 FOR 1)) STORED, + prefix_2 bytea GENERATED ALWAYS AS (SUBSTRING(prefix_4 FROM 1 FOR 2)) STORED, + prefix_3 bytea GENERATED ALWAYS AS (SUBSTRING(prefix_4 FROM 1 FOR 3)) STORED +) +WITH (autovacuum_vacuum_scale_factor='0.05'); + +CREATE TABLE observer_brokers ( + observer_id uuid NOT NULL, + broker_name text NOT NULL, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL, + last_packet_at timestamp with time zone, + auth_ok boolean DEFAULT true +); + +CREATE TABLE observer_locations ( + observer_id uuid NOT NULL, + iata character(3), + latitude double precision, + longitude double precision, + reported_at timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE observer_owners ( + observer_id uuid NOT NULL, + owner_node_id uuid, + owner_pubkey bytea, + contact_name text, + contact_email text, + notes text, + source text, + added_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE observer_scopes ( + observer_id uuid NOT NULL, + scope_id integer NOT NULL, + first_seen timestamp with time zone DEFAULT now() NOT NULL, + last_seen timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE observer_telemetry ( + id bigint NOT NULL, + observer_id uuid NOT NULL, + reported_at timestamp with time zone NOT NULL, + battery_voltage_mv integer, + airtime_tx_secs real, + airtime_rx_secs real, + noise_floor_db real, + uptime_seconds bigint, + queue_length integer, + debug_flags integer, + receive_errors integer +); + +CREATE SEQUENCE observer_telemetry_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE observer_telemetry_id_seq OWNED BY observer_telemetry.id; + +CREATE SEQUENCE packet_observations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE packet_observations_id_seq OWNED BY packet_observations.id; + +CREATE TABLE region_iatas ( + region_id integer NOT NULL, + iata character(3) NOT NULL, + added_at timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE regions ( + id integer NOT NULL, + slug text NOT NULL, + name text NOT NULL, + description text, + display_order integer DEFAULT 0, + center_lat double precision, + center_lng double precision, + zoom_level integer DEFAULT 8, + created_at timestamp with time zone DEFAULT now() NOT NULL, + updated_at timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE SEQUENCE regions_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE regions_id_seq OWNED BY regions.id; + +CREATE TABLE trace_iatas ( + trace_tag bytea NOT NULL, + iata character(3) NOT NULL, + last_heard timestamp with time zone DEFAULT now() NOT NULL +); + +CREATE TABLE transport_scopes ( + id integer NOT NULL, + name text NOT NULL, + display_name text, + transport_key bytea NOT NULL, + key_fingerprint bytea NOT NULL, + created_at timestamp with time zone DEFAULT now() NOT NULL, + imported_only boolean DEFAULT false NOT NULL +); + +CREATE SEQUENCE transport_scopes_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE transport_scopes_id_seq OWNED BY transport_scopes.id; + +ALTER TABLE ONLY channel_messages ALTER COLUMN id SET DEFAULT nextval('channel_messages_id_seq'::regclass); + +ALTER TABLE ONLY channels ALTER COLUMN id SET DEFAULT nextval('channels_id_seq'::regclass); + +ALTER TABLE ONLY observer_telemetry ALTER COLUMN id SET DEFAULT nextval('observer_telemetry_id_seq'::regclass); + +ALTER TABLE ONLY packet_observations ALTER COLUMN id SET DEFAULT nextval('packet_observations_id_seq'::regclass); + +ALTER TABLE ONLY regions ALTER COLUMN id SET DEFAULT nextval('regions_id_seq'::regclass); + +ALTER TABLE ONLY transport_scopes ALTER COLUMN id SET DEFAULT nextval('transport_scopes_id_seq'::regclass); + +ALTER TABLE ONLY accounts + ADD CONSTRAINT accounts_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY analytics_hourly_iata_stats + ADD CONSTRAINT analytics_hourly_iata_stats_pkey PRIMARY KEY (iata, hour); + +ALTER TABLE ONLY analytics_observer_activity_hourly + ADD CONSTRAINT analytics_observer_activity_hourly_pkey PRIMARY KEY (observer_id, payload_type, bucket); + +ALTER TABLE ONLY analytics_path_stats_hourly + ADD CONSTRAINT analytics_path_stats_hourly_pkey PRIMARY KEY (iata, hour, category, hash_bytes, entries); + +ALTER TABLE ONLY analytics_payload_breakdown_by_iata + ADD CONSTRAINT analytics_payload_breakdown_by_iata_pkey PRIMARY KEY (iata, payload_type, bucket); + +ALTER TABLE ONLY analytics_signal_stats_hourly + ADD CONSTRAINT analytics_signal_stats_hourly_pkey PRIMARY KEY (iata, hour, kind, snr_bin, rssi_bin); + +ALTER TABLE ONLY analytics_top_advertisers_by_iata + ADD CONSTRAINT analytics_top_advertisers_by_iata_pkey PRIMARY KEY (iata, node_id, bucket); + +ALTER TABLE ONLY analytics_top_observers_by_iata + ADD CONSTRAINT analytics_top_observers_by_iata_pkey PRIMARY KEY (iata, observer_id, bucket); + +ALTER TABLE ONLY analytics_top_talkers_by_iata + ADD CONSTRAINT analytics_top_talkers_by_iata_pkey PRIMARY KEY (iata, sender_name, bucket); + +ALTER TABLE ONLY channel_iatas + ADD CONSTRAINT channel_iatas_pkey PRIMARY KEY (channel_hash, iata); + +ALTER TABLE ONLY channel_keys + ADD CONSTRAINT channel_keys_pkey PRIMARY KEY (channel_id); + +ALTER TABLE ONLY channel_messages + ADD CONSTRAINT channel_messages_packet_hash_key UNIQUE (packet_hash); + +ALTER TABLE ONLY channel_messages + ADD CONSTRAINT channel_messages_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY channels + ADD CONSTRAINT channels_channel_hash_key_fingerprint_key UNIQUE (channel_hash, key_fingerprint); + +ALTER TABLE ONLY channels + ADD CONSTRAINT channels_hashtag_key UNIQUE (hashtag); + +ALTER TABLE ONLY channels + ADD CONSTRAINT channels_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY iata_codes + ADD CONSTRAINT iata_codes_pkey PRIMARY KEY (iata); + +ALTER TABLE ONLY known_routes + ADD CONSTRAINT known_routes_pkey PRIMARY KEY (iata, path_key); + +ALTER TABLE ONLY meshmapper_scope_catalogues + ADD CONSTRAINT meshmapper_scope_catalogues_pkey PRIMARY KEY (iata, url); + +ALTER TABLE ONLY meshmapper_zone_boundaries + ADD CONSTRAINT meshmapper_zone_boundaries_pkey PRIMARY KEY (iata); + +ALTER TABLE ONLY node_iatas + ADD CONSTRAINT node_iatas_pkey PRIMARY KEY (node_id, iata); + +ALTER TABLE ONLY node_neighbors + ADD CONSTRAINT node_neighbors_pkey PRIMARY KEY (node_id, neighbor_id, iata); + +ALTER TABLE ONLY node_short_ids + ADD CONSTRAINT node_short_ids_pkey PRIMARY KEY (node_id, iata); + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_public_key_key UNIQUE (public_key); + +ALTER TABLE ONLY observer_brokers + ADD CONSTRAINT observer_brokers_pkey PRIMARY KEY (observer_id, broker_name); + +ALTER TABLE ONLY observer_locations + ADD CONSTRAINT observer_locations_pkey PRIMARY KEY (observer_id, reported_at); + +ALTER TABLE ONLY observer_owners + ADD CONSTRAINT observer_owners_pkey PRIMARY KEY (observer_id); + +ALTER TABLE ONLY observer_scopes + ADD CONSTRAINT observer_scopes_pkey PRIMARY KEY (observer_id, scope_id); + +ALTER TABLE ONLY observer_telemetry + ADD CONSTRAINT observer_telemetry_observer_id_reported_at_key UNIQUE (observer_id, reported_at); + +ALTER TABLE ONLY observer_telemetry + ADD CONSTRAINT observer_telemetry_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY observers + ADD CONSTRAINT observers_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY observers + ADD CONSTRAINT observers_public_key_key UNIQUE (public_key); + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_packet_hash_observer_id_key UNIQUE (packet_hash, observer_id); + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY packets + ADD CONSTRAINT packets_pkey PRIMARY KEY (packet_hash); + +ALTER TABLE ONLY region_iatas + ADD CONSTRAINT region_iatas_pkey PRIMARY KEY (region_id, iata); + +ALTER TABLE ONLY regions + ADD CONSTRAINT regions_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY regions + ADD CONSTRAINT regions_slug_key UNIQUE (slug); + +ALTER TABLE ONLY trace_iatas + ADD CONSTRAINT trace_iatas_pkey PRIMARY KEY (trace_tag, iata); + +ALTER TABLE ONLY transport_scopes + ADD CONSTRAINT transport_scopes_name_key UNIQUE (name); + +ALTER TABLE ONLY transport_scopes + ADD CONSTRAINT transport_scopes_pkey PRIMARY KEY (id); + +CREATE UNIQUE INDEX accounts_name_uidx ON accounts USING btree (name) WHERE (deactivated_at IS NULL); + +CREATE INDEX idx_analytics_hourly_iata_stats_time ON analytics_hourly_iata_stats USING btree (hour); + +CREATE UNIQUE INDEX idx_analytics_hourly_iata_stats_view ON mv_hourly_iata_stats USING btree (iata, hour); + +CREATE INDEX idx_analytics_observer_activity_hourly_time ON analytics_observer_activity_hourly USING btree (bucket); + +CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly USING btree (observer_id, payload_type, bucket); + +CREATE INDEX idx_analytics_path_stats_hourly_time ON analytics_path_stats_hourly USING btree (hour); + +CREATE UNIQUE INDEX idx_analytics_path_stats_hourly_view ON mv_path_stats_hourly USING btree (iata, hour, category, hash_bytes, entries); + +CREATE INDEX idx_analytics_payload_breakdown_by_iata_time ON analytics_payload_breakdown_by_iata USING btree (bucket); + +CREATE UNIQUE INDEX idx_analytics_payload_breakdown_by_iata_view ON mv_payload_breakdown_by_iata USING btree (iata, payload_type, bucket); + +CREATE INDEX idx_analytics_signal_stats_hourly_time ON analytics_signal_stats_hourly USING btree (hour); + +CREATE UNIQUE INDEX idx_analytics_signal_stats_hourly_view ON mv_signal_stats_hourly USING btree (iata, hour, kind, snr_bin, rssi_bin); + +CREATE INDEX idx_analytics_top_advertisers_by_iata_time ON analytics_top_advertisers_by_iata USING btree (bucket); + +CREATE UNIQUE INDEX idx_analytics_top_advertisers_by_iata_view ON mv_top_advertisers_by_iata USING btree (iata, node_id, bucket); + +CREATE INDEX idx_analytics_top_observers_by_iata_time ON analytics_top_observers_by_iata USING btree (bucket); + +CREATE UNIQUE INDEX idx_analytics_top_observers_by_iata_view ON mv_top_observers_by_iata USING btree (iata, observer_id, bucket); + +CREATE INDEX idx_analytics_top_talkers_by_iata_time ON analytics_top_talkers_by_iata USING btree (bucket); + +CREATE UNIQUE INDEX idx_analytics_top_talkers_by_iata_view ON mv_top_talkers_by_iata USING btree (iata, sender_name, bucket); + +CREATE INDEX idx_channel_iatas_iata ON channel_iatas USING btree (iata); + +CREATE INDEX idx_channel_messages_channel ON channel_messages USING btree (channel_id, sent_at DESC); + +CREATE INDEX idx_channel_messages_sent_brin ON channel_messages USING brin (sent_at); + +CREATE INDEX idx_channels_hash ON channels USING btree (channel_hash); + +CREATE UNIQUE INDEX idx_channels_hash_no_key ON channels USING btree (channel_hash) WHERE (key_fingerprint IS NULL); + +CREATE INDEX idx_channels_hashtag ON channels USING btree (hashtag) WHERE (hashtag IS NOT NULL); + +CREATE INDEX idx_channels_last_seen_id ON channels USING btree (last_seen DESC, id DESC); + +CREATE INDEX idx_known_routes_hop_count ON known_routes USING btree (iata, hop_count); + +CREATE INDEX idx_known_routes_iata_last_seen ON known_routes USING btree (iata, last_seen DESC); + +CREATE INDEX idx_known_routes_last_seen ON known_routes USING btree (last_seen DESC); + +CREATE INDEX idx_known_routes_reconfirm ON known_routes USING btree (last_reconfirmed_at); + +CREATE INDEX idx_mv_path_stats_hourly_hour ON mv_path_stats_hourly USING btree (hour); + +CREATE UNIQUE INDEX idx_mv_radio_presets ON mv_radio_presets USING btree (preset, iata, source_type); + +CREATE INDEX idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly USING btree (hour); + +CREATE UNIQUE INDEX idx_mv_top_nodes ON mv_top_nodes_by_iata USING btree (iata, node_id); + +CREATE INDEX idx_node_iatas_iata ON node_iatas USING btree (iata, last_heard DESC); + +CREATE INDEX idx_node_neighbors_neighbor ON node_neighbors USING btree (neighbor_id, iata); + +CREATE INDEX idx_node_neighbors_node ON node_neighbors USING btree (node_id, iata); + +CREATE INDEX idx_nodes_location ON nodes USING btree (latitude, longitude) WHERE ((latitude IS NOT NULL) AND (longitude IS NOT NULL)); + +CREATE INDEX idx_nodes_min_firmware ON nodes USING btree (min_firmware_version) WHERE (min_firmware_version IS NOT NULL); + +CREATE INDEX idx_nodes_multibyte_paths ON nodes USING btree (supports_multibyte_paths) WHERE supports_multibyte_paths; + +CREATE INDEX idx_nodes_multibyte_traces ON nodes USING btree (supports_multibyte_traces) WHERE supports_multibyte_traces; + +CREATE INDEX idx_nodes_pubkey ON nodes USING btree (public_key); + +CREATE INDEX idx_nodes_type_last_seen ON nodes USING btree (node_type, last_seen DESC); + +CREATE INDEX idx_observations_heard_brin ON packet_observations USING brin (heard_at); + +CREATE INDEX idx_observations_iata_heard ON packet_observations USING btree (iata, heard_at DESC); + +CREATE INDEX idx_observations_observer ON packet_observations USING btree (observer_id, heard_at DESC); + +CREATE INDEX idx_observations_packet ON packet_observations USING btree (packet_hash); + +CREATE INDEX idx_observations_route_evidence ON packet_observations USING btree (iata, hash_size, decode(md5(path_bytes), 'hex'::text), heard_at DESC, id DESC) WHERE ((path_bytes IS NOT NULL) AND (payload_type IS NOT NULL) AND (payload_type <> 9) AND (hop_count >= 2)); + +CREATE INDEX idx_observer_locations_recent ON observer_locations USING btree (observer_id, reported_at DESC); + +CREATE INDEX idx_observer_owners_node ON observer_owners USING btree (owner_node_id) WHERE (owner_node_id IS NOT NULL); + +CREATE INDEX idx_observers_last_seen ON observers USING btree (last_seen DESC); + +CREATE INDEX idx_observers_pubkey ON observers USING btree (public_key); + +CREATE INDEX idx_observers_type ON observers USING btree (observer_type) WHERE (observer_type IS NOT NULL); + +CREATE INDEX idx_packets_channel ON packets USING btree (channel_hash, first_heard_at DESC) WHERE (channel_hash IS NOT NULL); + +CREATE INDEX idx_packets_first_heard_brin ON packets USING brin (first_heard_at); + +CREATE INDEX idx_packets_last_heard ON packets USING btree (last_heard_at DESC); + +CREATE INDEX idx_packets_origin ON packets USING btree (origin_pubkey, first_heard_at DESC) WHERE (origin_pubkey IS NOT NULL); + +CREATE INDEX idx_packets_payload_type ON packets USING btree (payload_type, first_heard_at DESC); + +CREATE INDEX idx_packets_route_type ON packets USING btree (route_type, first_heard_at DESC); + +CREATE INDEX idx_packets_scope ON packets USING btree (scope_id) WHERE (scope_id IS NOT NULL); + +CREATE INDEX idx_packets_trace_tag ON packets USING btree (trace_tag) WHERE (trace_tag IS NOT NULL); + +CREATE INDEX idx_region_iatas_iata ON region_iatas USING btree (iata); + +CREATE INDEX idx_short_ids_p1 ON node_short_ids USING btree (iata, prefix_1); + +CREATE INDEX idx_short_ids_p2 ON node_short_ids USING btree (iata, prefix_2); + +CREATE INDEX idx_short_ids_p3 ON node_short_ids USING btree (iata, prefix_3); + +CREATE INDEX idx_short_ids_p4 ON node_short_ids USING btree (iata, prefix_4); + +CREATE INDEX idx_telemetry_observer_recent ON observer_telemetry USING btree (observer_id, reported_at DESC); + +CREATE INDEX idx_telemetry_reported_brin ON observer_telemetry USING brin (reported_at); + +CREATE INDEX idx_trace_iatas_iata ON trace_iatas USING btree (iata); + +CREATE INDEX idx_transport_scopes_fingerprint ON transport_scopes USING btree (key_fingerprint); + +ALTER TABLE ONLY channel_iatas + ADD CONSTRAINT channel_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY channel_keys + ADD CONSTRAINT channel_keys_channel_id_fkey FOREIGN KEY (channel_id) REFERENCES channels(id) ON DELETE CASCADE; + +ALTER TABLE ONLY channel_messages + ADD CONSTRAINT channel_messages_channel_id_fkey FOREIGN KEY (channel_id) REFERENCES channels(id); + +ALTER TABLE ONLY channel_messages + ADD CONSTRAINT channel_messages_packet_hash_fkey FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; + +ALTER TABLE ONLY known_routes + ADD CONSTRAINT known_routes_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY node_iatas + ADD CONSTRAINT node_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY node_iatas + ADD CONSTRAINT node_iatas_node_id_fkey FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY node_neighbors + ADD CONSTRAINT node_neighbors_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY node_neighbors + ADD CONSTRAINT node_neighbors_neighbor_id_fkey FOREIGN KEY (neighbor_id) REFERENCES nodes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY node_neighbors + ADD CONSTRAINT node_neighbors_node_id_fkey FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY node_short_ids + ADD CONSTRAINT node_short_ids_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY node_short_ids + ADD CONSTRAINT node_short_ids_node_id_fkey FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY nodes + ADD CONSTRAINT nodes_default_scope_id_fkey FOREIGN KEY (default_scope_id) REFERENCES transport_scopes(id); + +ALTER TABLE ONLY observer_brokers + ADD CONSTRAINT observer_brokers_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_locations + ADD CONSTRAINT observer_locations_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata); + +ALTER TABLE ONLY observer_locations + ADD CONSTRAINT observer_locations_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_owners + ADD CONSTRAINT observer_owners_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_owners + ADD CONSTRAINT observer_owners_owner_node_id_fkey FOREIGN KEY (owner_node_id) REFERENCES nodes(id); + +ALTER TABLE ONLY observer_scopes + ADD CONSTRAINT observer_scopes_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_scopes + ADD CONSTRAINT observer_scopes_scope_id_fkey FOREIGN KEY (scope_id) REFERENCES transport_scopes(id) ON DELETE CASCADE; + +ALTER TABLE ONLY observer_telemetry + ADD CONSTRAINT observer_telemetry_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id) ON DELETE CASCADE; + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata); + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_observer_id_fkey FOREIGN KEY (observer_id) REFERENCES observers(id); + +ALTER TABLE ONLY packet_observations + ADD CONSTRAINT packet_observations_packet_hash_fkey FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; + +ALTER TABLE ONLY packets + ADD CONSTRAINT packets_scope_id_fkey FOREIGN KEY (scope_id) REFERENCES transport_scopes(id); + +ALTER TABLE ONLY region_iatas + ADD CONSTRAINT region_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + +ALTER TABLE ONLY region_iatas + ADD CONSTRAINT region_iatas_region_id_fkey FOREIGN KEY (region_id) REFERENCES regions(id) ON DELETE CASCADE; + +ALTER TABLE ONLY trace_iatas + ADD CONSTRAINT trace_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; + + +-- + +-- diff --git a/db/migrations/001_initial_schema.sql b/db/migrations/001_initial_schema.sql deleted file mode 100644 index 973990f9..00000000 --- a/db/migrations/001_initial_schema.sql +++ /dev/null @@ -1,439 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: agpl - --- ============================================================ --- Beacon schema migration --- ============================================================ - --- ============================================================ --- IATA CODES --- ============================================================ - -CREATE TABLE iata_codes ( - iata CHAR(3) PRIMARY KEY, - display_name TEXT, - approx_lat DOUBLE PRECISION, - approx_lng DOUBLE PRECISION, - added_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - --- ============================================================ --- REGIONS --- ============================================================ - -CREATE TABLE regions ( - id SERIAL PRIMARY KEY, - slug TEXT UNIQUE NOT NULL, - name TEXT NOT NULL, - description TEXT, - display_order INT DEFAULT 0, - center_lat DOUBLE PRECISION, - center_lng DOUBLE PRECISION, - zoom_level INT DEFAULT 8, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE TABLE region_iatas ( - region_id INT NOT NULL REFERENCES regions(id) ON DELETE CASCADE, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - added_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (region_id, iata) -); - -CREATE INDEX idx_region_iatas_iata ON region_iatas(iata); - --- ============================================================ --- TRANSPORT SCOPES --- ============================================================ - -CREATE TABLE transport_scopes ( - id SERIAL PRIMARY KEY, - name TEXT NOT NULL UNIQUE, -- normalized key string e.g. "#bc" - display_name TEXT, -- optional friendly name - transport_key BYTEA NOT NULL, -- 16-byte derived key - key_fingerprint BYTEA NOT NULL, -- SHA256(transport_key)[:8] for fast lookup - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_transport_scopes_fingerprint ON transport_scopes(key_fingerprint); - --- ============================================================ --- NODES (must come before observers due to observer_owners FK) --- ============================================================ - -CREATE TABLE nodes ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - public_key BYTEA UNIQUE NOT NULL, - node_type SMALLINT NOT NULL, - name TEXT, - latitude DOUBLE PRECISION, - longitude DOUBLE PRECISION, - location_source TEXT, - last_advert_at TIMESTAMPTZ, - supports_multibyte_paths BOOLEAN NOT NULL DEFAULT FALSE, - supports_multibyte_traces BOOLEAN NOT NULL DEFAULT FALSE, - default_scope_id INT REFERENCES transport_scopes(id), - min_firmware_version TEXT GENERATED ALWAYS AS ( - CASE - WHEN supports_multibyte_paths THEN '1.14.0+' - WHEN supports_multibyte_traces THEN '1.11.0+' - ELSE NULL - END - ) STORED, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - radio_freq_mhz REAL, - radio_sf SMALLINT, - radio_bw_khz REAL, - metadata JSONB -); - -CREATE INDEX idx_nodes_type_last_seen ON nodes(node_type, last_seen DESC); -CREATE INDEX idx_nodes_location ON nodes(latitude, longitude) - WHERE latitude IS NOT NULL AND longitude IS NOT NULL; -CREATE INDEX idx_nodes_pubkey ON nodes(public_key); -CREATE INDEX idx_nodes_multibyte_paths ON nodes(supports_multibyte_paths) WHERE supports_multibyte_paths; -CREATE INDEX idx_nodes_multibyte_traces ON nodes(supports_multibyte_traces) WHERE supports_multibyte_traces; -CREATE INDEX idx_nodes_min_firmware ON nodes(min_firmware_version) WHERE min_firmware_version IS NOT NULL; - --- ============================================================ --- OBSERVERS --- ============================================================ - -CREATE TABLE observers ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - public_key BYTEA UNIQUE NOT NULL, - display_name TEXT, - observer_type TEXT, - software_version TEXT, - hardware_model TEXT, - firmware_version TEXT, - firmware_build TEXT, - radio_freq_mhz REAL, - radio_sf SMALLINT, - radio_bw_khz REAL, - radio_cr SMALLINT, - battery_level REAL, - uptime_seconds BIGINT, - status_metadata JSONB, - last_status_at TIMESTAMPTZ, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - observation_count BIGINT DEFAULT 0, - metadata JSONB -); - -CREATE INDEX idx_observers_last_seen ON observers(last_seen DESC); -CREATE INDEX idx_observers_pubkey ON observers(public_key); -CREATE INDEX idx_observers_type ON observers(observer_type) WHERE observer_type IS NOT NULL; - -CREATE TABLE observer_brokers ( - observer_id UUID NOT NULL REFERENCES observers(id) ON DELETE CASCADE, - broker_name TEXT NOT NULL, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_packet_at TIMESTAMPTZ, - auth_ok BOOLEAN DEFAULT TRUE, - PRIMARY KEY (observer_id, broker_name) -); - -CREATE TABLE observer_locations ( - observer_id UUID NOT NULL REFERENCES observers(id) ON DELETE CASCADE, - iata CHAR(3) REFERENCES iata_codes(iata), - latitude DOUBLE PRECISION, - longitude DOUBLE PRECISION, - reported_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (observer_id, reported_at) -); - -CREATE TABLE observer_scopes ( - observer_id UUID NOT NULL REFERENCES observers(id) ON DELETE CASCADE, - scope_id INT NOT NULL REFERENCES transport_scopes(id) ON DELETE CASCADE, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (observer_id, scope_id) -); - -CREATE INDEX idx_observer_locations_recent ON observer_locations(observer_id, reported_at DESC); - -CREATE TABLE observer_telemetry ( - id BIGSERIAL PRIMARY KEY, - observer_id UUID NOT NULL REFERENCES observers(id) ON DELETE CASCADE, - reported_at TIMESTAMPTZ NOT NULL, - battery_voltage_mv INT, - airtime_tx_pct REAL, - airtime_rx_pct REAL, - noise_floor_db REAL, - uptime_seconds BIGINT, - queue_length INT, - debug_flags INT, - receive_errors INT, - UNIQUE (observer_id, reported_at) -); - -CREATE INDEX idx_telemetry_reported_brin ON observer_telemetry USING BRIN (reported_at); -CREATE INDEX idx_telemetry_observer_recent ON observer_telemetry(observer_id, reported_at DESC); - -CREATE TABLE observer_owners ( - observer_id UUID PRIMARY KEY REFERENCES observers(id) ON DELETE CASCADE, - owner_node_id UUID REFERENCES nodes(id), - owner_pubkey BYTEA, - contact_name TEXT, - contact_email TEXT, - notes TEXT, - source TEXT, - added_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() -); - -CREATE INDEX idx_observer_owners_node ON observer_owners(owner_node_id) WHERE owner_node_id IS NOT NULL; - --- ============================================================ --- NODE IATAS AND SHORT IDS --- ============================================================ - -CREATE TABLE node_iatas ( - node_id UUID NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - first_heard TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_heard TIMESTAMPTZ NOT NULL DEFAULT NOW(), - observation_count BIGINT DEFAULT 0, - PRIMARY KEY (node_id, iata) -); - -CREATE INDEX idx_node_iatas_iata ON node_iatas(iata, last_heard DESC); - -CREATE TABLE node_short_ids ( - node_id UUID NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - prefix_4 BYTEA NOT NULL, - prefix_1 BYTEA GENERATED ALWAYS AS (substring(prefix_4 from 1 for 1)) STORED, - prefix_2 BYTEA GENERATED ALWAYS AS (substring(prefix_4 from 1 for 2)) STORED, - prefix_3 BYTEA GENERATED ALWAYS AS (substring(prefix_4 from 1 for 3)) STORED, - PRIMARY KEY (node_id, iata) -); - -CREATE INDEX idx_short_ids_p1 ON node_short_ids(iata, prefix_1); -CREATE INDEX idx_short_ids_p2 ON node_short_ids(iata, prefix_2); -CREATE INDEX idx_short_ids_p3 ON node_short_ids(iata, prefix_3); -CREATE INDEX idx_short_ids_p4 ON node_short_ids(iata, prefix_4); - --- ============================================================ --- PACKETS --- ============================================================ - -CREATE TABLE packets ( - packet_hash BYTEA PRIMARY KEY, - payload_type SMALLINT NOT NULL, - payload_version SMALLINT NOT NULL, - route_type SMALLINT NOT NULL, - transport_codes_present BOOLEAN DEFAULT FALSE, - region_code INT, - sub_region_code INT, - scope_id INT REFERENCES transport_scopes(id), - origin_pubkey BYTEA, - raw_payload BYTEA NOT NULL, - raw_header BYTEA NOT NULL, - parsed_payload JSONB, - decrypted BOOLEAN DEFAULT FALSE, - channel_hash BYTEA, - trace_tag BYTEA, - first_heard_at TIMESTAMPTZ NOT NULL, - last_heard_at TIMESTAMPTZ NOT NULL -); - -CREATE INDEX idx_packets_first_heard_brin ON packets USING BRIN (first_heard_at); -CREATE INDEX idx_packets_payload_type ON packets(payload_type, first_heard_at DESC); -CREATE INDEX idx_packets_route_type ON packets(route_type, first_heard_at DESC); -CREATE INDEX idx_packets_origin ON packets(origin_pubkey, first_heard_at DESC) - WHERE origin_pubkey IS NOT NULL; -CREATE INDEX idx_packets_channel ON packets(channel_hash, first_heard_at DESC) - WHERE channel_hash IS NOT NULL; - CREATE INDEX idx_packets_trace_tag ON packets(trace_tag) WHERE trace_tag IS NOT NULL; - --- ============================================================ --- PACKET OBSERVATIONS --- ============================================================ - -CREATE TABLE packet_observations ( - id BIGSERIAL PRIMARY KEY, - packet_hash BYTEA NOT NULL REFERENCES packets(packet_hash) ON DELETE CASCADE, - observer_id UUID NOT NULL REFERENCES observers(id), - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata), - heard_at TIMESTAMPTZ NOT NULL, - path_length_byte SMALLINT NOT NULL, - hash_size SMALLINT NOT NULL, - hop_count SMALLINT NOT NULL, - path_bytes BYTEA, - rssi SMALLINT, - snr REAL, - propagation_time_ms INT, - radio_freq_mhz REAL, - spread_factor SMALLINT, - bandwidth_khz REAL, - coding_rate SMALLINT, - source_broker TEXT, - UNIQUE (packet_hash, observer_id, heard_at) -); - -CREATE INDEX idx_observations_heard_brin ON packet_observations USING BRIN (heard_at); -CREATE INDEX idx_observations_iata_heard ON packet_observations(iata, heard_at DESC); -CREATE INDEX idx_observations_observer ON packet_observations(observer_id, heard_at DESC); -CREATE INDEX idx_observations_packet ON packet_observations(packet_hash); - --- ============================================================ --- CHANNELS AND CHAT MESSAGES --- ============================================================ - -CREATE TABLE channels ( - id SERIAL PRIMARY KEY, - channel_hash BYTEA NOT NULL, - -- key_fingerprint is the first 8 bytes of SHA256(key_bytes). - -- Combined with channel_hash it uniquely identifies a channel, - -- since a 1-byte hash has high collision probability. - -- NULL when the key is unknown (hash-only record). - key_fingerprint BYTEA, - name TEXT, - hashtag TEXT UNIQUE, -- e.g. "meshcore"; only set for hashtag-derived channels - is_hashtag BOOLEAN DEFAULT FALSE, - is_public BOOLEAN DEFAULT FALSE, - key_known BOOLEAN DEFAULT FALSE, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - message_count BIGINT DEFAULT 0, - -- Unique on (hash, fingerprint): allows multiple channels with the same - -- hash as long as they have different keys. NULL fingerprint is allowed - -- once per hash (the unknown-key record). - UNIQUE (channel_hash, key_fingerprint) -); - -CREATE UNIQUE INDEX idx_channels_hash_no_key -ON channels (channel_hash) -WHERE key_fingerprint IS NULL; - -CREATE INDEX idx_channels_last_seen ON channels(last_seen DESC); -CREATE INDEX idx_channels_hash ON channels(channel_hash); -CREATE INDEX idx_channels_hashtag ON channels(hashtag) WHERE hashtag IS NOT NULL; - -CREATE TABLE channel_keys ( - channel_id INT PRIMARY KEY REFERENCES channels(id) ON DELETE CASCADE, - key_bytes BYTEA NOT NULL, - key_fingerprint BYTEA NOT NULL, -- first 8 bytes of SHA256(key_bytes) - added_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - added_by TEXT -); - -CREATE TABLE channel_messages ( - id BIGSERIAL PRIMARY KEY, - channel_id INT NOT NULL REFERENCES channels(id), - packet_hash BYTEA NOT NULL REFERENCES packets(packet_hash), - sender_name TEXT, - sender_pubkey BYTEA, - content TEXT, - sent_at TIMESTAMPTZ NOT NULL, - UNIQUE (packet_hash) -); - -CREATE INDEX idx_channel_messages_channel ON channel_messages(channel_id, sent_at DESC); -CREATE INDEX idx_channel_messages_sent_brin ON channel_messages USING BRIN (sent_at); - --- ============================================================ --- ROUTES --- ============================================================ - -CREATE TABLE known_routes ( - id BIGSERIAL PRIMARY KEY, - node_ids UUID[] NOT NULL, -- resolved node UUIDs in hop order - hash_prefix BYTEA[] NOT NULL, -- raw hash bytes in hop order for prefix matching - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - hop_count INT NOT NULL, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - observation_count BIGINT NOt NULL DEFAULT 1, - UNIQUE (node_ids, iata) -); - -CREATE INDEX idx_known_routes_iata ON known_routes(iata); -CREATE INDEX idx_known_routes_hop_count ON known_routes(iata, hop_count); - --- ============================================================ --- NEIGHBORS --- ============================================================ - -CREATE TABLE node_neighbors ( - node_id UUID NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, - neighbor_id UUID NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - observation_count BIGINT NOT NULL DEFAULT 1, - PRIMARY KEY (node_id, neighbor_id, iata) -); - -CREATE INDEX idx_node_neighbors_node ON node_neighbors(node_id, iata); -CREATE INDEX idx_node_neighbors_neighbor ON node_neighbors(neighbor_id, iata); - --- ============================================================ --- MATERIALIZED VIEWS --- ============================================================ - -CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS -SELECT - iata, - date_trunc('hour', heard_at)::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets, - COUNT(DISTINCT observer_id) AS active_observers -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '7 days' -GROUP BY iata, date_trunc('hour', heard_at); - -CREATE MATERIALIZED VIEW mv_top_nodes_by_iata AS -SELECT - ni.iata, - ni.node_id, - n.name, - n.node_type, - ni.observation_count, - ni.last_heard -FROM node_iatas ni -JOIN nodes n ON n.id = ni.node_id -WHERE ni.last_heard > NOW() - INTERVAL '7 days'; - -CREATE UNIQUE INDEX idx_mv_top_nodes - ON mv_top_nodes_by_iata(iata, node_id); - -CREATE UNIQUE INDEX idx_mv_hourly_iata_stats - ON mv_hourly_iata_stats(iata, hour); - -CREATE MATERIALIZED VIEW mv_radio_presets AS -SELECT - concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text AS preset, - (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) AS iata, - 'observer' AS source_type, - COUNT(*) AS count -FROM observers o -WHERE o.radio_freq_mhz IS NOT NULL - AND o.radio_sf IS NOT NULL - AND o.radio_bw_khz IS NOT NULL -GROUP BY concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text, - (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) -HAVING (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) IS NOT NULL - -UNION ALL - -SELECT - concat(n.radio_freq_mhz, ',', n.radio_bw_khz, ',', n.radio_sf) AS preset, - ni.iata, - 'node' AS source_type, - COUNT(*) AS count -FROM nodes n -JOIN node_iatas ni ON ni.node_id = n.id -WHERE n.radio_freq_mhz IS NOT NULL - AND n.radio_sf IS NOT NULL - AND n.radio_bw_khz IS NOT NULL -GROUP BY concat(n.radio_freq_mhz, ',', n.radio_bw_khz, ',', n.radio_sf), ni.iata; - -CREATE UNIQUE INDEX idx_mv_radio_presets - ON mv_radio_presets(preset, iata, source_type); diff --git a/db/migrations/002_drop_single_hop_routes.sql b/db/migrations/002_drop_single_hop_routes.sql deleted file mode 100644 index 472962e5..00000000 --- a/db/migrations/002_drop_single_hop_routes.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: agpl - --- 002_drop_single_hop_routes.sql --- routes were being stored as long as the path was --- greater than 0. -DELETE FROM known_routes WHERE hop_count < 2; diff --git a/db/migrations/003_tag_ping_traces.sql b/db/migrations/003_tag_ping_traces.sql deleted file mode 100644 index 6ebfb458..00000000 --- a/db/migrations/003_tag_ping_traces.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: agpl - --- 003_tag_ping_traces.sql -UPDATE packets -SET parsed_payload = jsonb_set(parsed_payload, '{type}', '"PING"') -WHERE trace_tag IS NOT NULL - AND jsonb_array_length(parsed_payload->'pathHashes') = 1 - AND parsed_payload->>'type' = 'TRACE'; diff --git a/db/migrations/004_clean_non_infra_neighbors.sql b/db/migrations/004_clean_non_infra_neighbors.sql deleted file mode 100644 index 6cebb97b..00000000 --- a/db/migrations/004_clean_non_infra_neighbors.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Remove neighbor records where either side is not infrastructure (repeater or room). --- These are dirty rows from historical prefix collisions resolving to non-forwarding nodes. -DELETE FROM node_neighbors nn -USING nodes n -WHERE nn.neighbor_id = n.id - AND n.node_type NOT IN (2, 3); - -DELETE FROM node_neighbors nn -USING nodes n -WHERE nn.node_id = n.id - AND n.node_type NOT IN (2, 3); diff --git a/db/migrations/005_add_neighbor_snr.sql b/db/migrations/005_add_neighbor_snr.sql deleted file mode 100644 index 4440b081..00000000 --- a/db/migrations/005_add_neighbor_snr.sql +++ /dev/null @@ -1,5 +0,0 @@ --- Adds an optional SNR reading to node_neighbors. Most rows (advert --- path-hop-derived topology) will leave this null; it's populated only --- when the neighbor relationship comes with an actual signal reading --- (e.g. observer-detected via DISCOVER_RESP). -ALTER TABLE node_neighbors ADD COLUMN snr REAL; diff --git a/db/migrations/006_fix_observation_dedup_key.sql b/db/migrations/006_fix_observation_dedup_key.sql deleted file mode 100644 index aa3cf7a8..00000000 --- a/db/migrations/006_fix_observation_dedup_key.sql +++ /dev/null @@ -1,24 +0,0 @@ --- 006_fix_observation_dedup_key.sql --- --- The previous unique constraint on (packet_hash, observer_id, heard_at) was --- too permissive: heard_at is stamped at ingest time (time.Now()), so MQTT --- redeliveries of the same packet arrive with slightly different timestamps --- and bypass dedup, producing duplicate observations for the same observer. --- --- An observer either heard a packet or they didn't. Drop heard_at from the --- constraint so that (packet_hash, observer_id) is the dedup key. - --- Remove duplicate observations, keeping the earliest (lowest id) per (packet_hash, observer_id). -DELETE FROM packet_observations -WHERE id NOT IN ( - SELECT MIN(id) - FROM packet_observations - GROUP BY packet_hash, observer_id -); - -ALTER TABLE packet_observations - DROP CONSTRAINT packet_observations_packet_hash_observer_id_heard_at_key; - -ALTER TABLE packet_observations - ADD CONSTRAINT packet_observations_packet_hash_observer_id_key - UNIQUE (packet_hash, observer_id); diff --git a/db/migrations/007_dedupe_hashonly_channels.sql b/db/migrations/007_dedupe_hashonly_channels.sql deleted file mode 100644 index e59e1eb7..00000000 --- a/db/migrations/007_dedupe_hashonly_channels.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Cleans up the ghost "unknown key" channel rows created by the --- side_effects.go bug where UpsertChannelHashOnly was called --- unconditionally on every GRP_TXT packet, even when the key was --- known and the message decrypted successfully. --- --- Safe because hash-only rows (key_fingerprint IS NULL) never have --- channel_messages attached to them -- InsertChannelMessage is only --- ever called against a keyed channel row. We only delete a hash-only --- row when a keyed row already exists for the same channel_hash; --- hashes with no known key at all (genuinely unknown) are left as-is. - -DELETE FROM channels c -WHERE c.key_fingerprint IS NULL - AND EXISTS ( - SELECT 1 FROM channels k - WHERE k.channel_hash = c.channel_hash - AND k.key_fingerprint IS NOT NULL - ); diff --git a/db/migrations/008_packets_last_heard_index.sql b/db/migrations/008_packets_last_heard_index.sql deleted file mode 100644 index f8a4cec0..00000000 --- a/db/migrations/008_packets_last_heard_index.sql +++ /dev/null @@ -1,8 +0,0 @@ --- 008_packets_last_heard_index.sql --- --- The packet list orders by last_heard_at but there was no index on it, so --- every page seq-scanned and sorted all packets (~2.8s per page on a 1.5M --- row table; 31ms with the index). Affordable to maintain now that presence --- coalescing keeps this column from being rewritten on every observation. - -CREATE INDEX idx_packets_last_heard ON packets(last_heard_at DESC); diff --git a/db/migrations/009_clean_zeroed_observer_telemetry.sql b/db/migrations/009_clean_zeroed_observer_telemetry.sql deleted file mode 100644 index 3289dc5a..00000000 --- a/db/migrations/009_clean_zeroed_observer_telemetry.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Cleans up observer_telemetry rows written from /status messages whose --- "stats" object was missing or renamed. status.go's json.Unmarshal --- silently tolerated the missing object, leaving the stats struct at Go --- zero-values, and the row was inserted unconditionally from there -- --- uptime_secs=0, battery_mv=0, noise_floor=0, tx/rx_air_secs=0. With the --- hourly ON CONFLICT (observer_id, reported_at) DO NOTHING dedup, --- whichever message landed first per hour won, so these zero rows --- regularly became the hour's telemetry and corrupted the 24h/7d/30d --- aggregates (AVG(noise_floor_db), AVG(battery_voltage_mv), MAX-MIN --- airtime). --- --- Safe because a running observer never reports uptime_seconds == 0; --- this is the same signal now used by status.go to skip the insert going --- forward (see internal/ingest/status.go handleStatus). - -DELETE FROM observer_telemetry WHERE uptime_seconds = 0; diff --git a/db/migrations/010_add_node_clock_drift.sql b/db/migrations/010_add_node_clock_drift.sql deleted file mode 100644 index b79a56e8..00000000 --- a/db/migrations/010_add_node_clock_drift.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Stores the device-reported clock drift measured from each ADVERT: the signed --- advert body already carries the device's own wall-clock timestamp --- (advert.Timestamp, epoch seconds), which the server previously decoded and --- then discarded before it reached the nodes table. This column stores --- (device timestamp - server receive time) in seconds, computed at the same --- moment nodes.last_advert_at is set to NOW(), so last_advert_at doubles as --- the "when was this drift measured" timestamp with no separate column --- needed. NULL until a node's first successfully-verified advert is upserted. --- --- +ve = device clock ahead of the server; -ve = behind. Surfaced on the node --- API for repeaters/room servers only (nodeType 2/3) -- see internal/api/nodes.go. - -ALTER TABLE nodes ADD COLUMN device_clock_drift_seconds INTEGER; diff --git a/db/migrations/011_channel_iatas.sql b/db/migrations/011_channel_iatas.sql deleted file mode 100644 index a89885d6..00000000 --- a/db/migrations/011_channel_iatas.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Per-IATA channel activity so the IATA filter skips the ~7s EXISTS over packets. --- Keyed by raw hash (channels can share one), so no FK to channels. - -CREATE TABLE channel_iatas ( - channel_hash BYTEA NOT NULL, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - last_heard TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (channel_hash, iata) -); - -CREATE INDEX idx_channel_iatas_iata ON channel_iatas(iata); - --- Seed from retained packets; parallelism off so the join spills to disk, not /dev/shm. -SET max_parallel_workers_per_gather = 0; - -INSERT INTO channel_iatas (channel_hash, iata, last_heard) -SELECT p.channel_hash, po.iata, MAX(po.heard_at) -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -WHERE p.channel_hash IS NOT NULL -GROUP BY p.channel_hash, po.iata; - -RESET max_parallel_workers_per_gather; diff --git a/db/migrations/012_trace_iatas.sql b/db/migrations/012_trace_iatas.sql deleted file mode 100644 index ffc9840b..00000000 --- a/db/migrations/012_trace_iatas.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Per-IATA trace activity so the trace filter stops joining every trace packet --- to observations (the hash join was overrunning /dev/shm). - -CREATE TABLE trace_iatas ( - trace_tag BYTEA NOT NULL, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - last_heard TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (trace_tag, iata) -); - -CREATE INDEX idx_trace_iatas_iata ON trace_iatas(iata); - --- Seed from retained packets; parallelism off so the join spills to disk, not /dev/shm. -SET max_parallel_workers_per_gather = 0; - -INSERT INTO trace_iatas (trace_tag, iata, last_heard) -SELECT p.trace_tag, po.iata, MAX(po.heard_at) -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -WHERE p.trace_tag IS NOT NULL -GROUP BY p.trace_tag, po.iata; - -RESET max_parallel_workers_per_gather; diff --git a/db/migrations/013_packets_scope_index.sql b/db/migrations/013_packets_scope_index.sql deleted file mode 100644 index 73029cf9..00000000 --- a/db/migrations/013_packets_scope_index.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Scope stats count off scope_id, which had no index. - -CREATE INDEX idx_packets_scope ON packets(scope_id) WHERE scope_id IS NOT NULL; diff --git a/db/migrations/014_known_routes_last_seen_index.sql b/db/migrations/014_known_routes_last_seen_index.sql deleted file mode 100644 index faa58081..00000000 --- a/db/migrations/014_known_routes_last_seen_index.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Routes list orders by last_seen; index it (was seq-scanning ~150k rows per page). - -CREATE INDEX idx_known_routes_last_seen ON known_routes(last_seen DESC); diff --git a/db/migrations/015_observations_payload_type.sql b/db/migrations/015_observations_payload_type.sql deleted file mode 100644 index e3bb79bf..00000000 --- a/db/migrations/015_observations_payload_type.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Copy payload_type onto the observation (like source_broker) so the breakdown --- drops its join back to packets, which was overrunning /dev/shm and 500ing. - -ALTER TABLE packet_observations ADD COLUMN payload_type SMALLINT; - --- Backfill the last 7 days only (all the breakdown reads); parallelism off so --- the join spills to disk, not /dev/shm. -SET max_parallel_workers_per_gather = 0; - -UPDATE packet_observations po -SET payload_type = p.payload_type -FROM packets p -WHERE p.packet_hash = po.packet_hash - AND po.heard_at > NOW() - INTERVAL '7 days' - AND po.payload_type IS NULL; - -RESET max_parallel_workers_per_gather; diff --git a/db/migrations/016_mv_payload_breakdown.sql b/db/migrations/016_mv_payload_breakdown.sql deleted file mode 100644 index bce48d9d..00000000 --- a/db/migrations/016_mv_payload_breakdown.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Precomputed payload breakdown, bucketed by hour so the stats endpoint can --- serve any window (24h/7d/30d) by summing the buckets in range instead of --- scanning a month of observations per request. - -CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS -SELECT - iata, - payload_type, - date_trunc('hour', heard_at)::timestamptz AS bucket, - COUNT(*) AS count -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at); - -CREATE UNIQUE INDEX idx_mv_payload_breakdown - ON mv_payload_breakdown_by_iata(iata, payload_type, bucket); diff --git a/db/migrations/017_mv_top_observers.sql b/db/migrations/017_mv_top_observers.sql deleted file mode 100644 index 9cc0c1f2..00000000 --- a/db/migrations/017_mv_top_observers.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Precomputed observer activity, bucketed by hour so top-observers can be --- served for any window (24h/7d/30d) by summing the buckets in range. Was a --- ~2s per-request scan of a week of observations. - -CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at)::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM packet_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at); - -CREATE UNIQUE INDEX idx_mv_top_observers - ON mv_top_observers_by_iata(iata, observer_id, bucket); diff --git a/db/migrations/018_mv_top_talkers.sql b/db/migrations/018_mv_top_talkers.sql deleted file mode 100644 index 1b2132cf..00000000 --- a/db/migrations/018_mv_top_talkers.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Hourly-bucketed talker counts so top-talkers serves any window from a small --- table instead of the live channel_messages/observations join (millions of rows --- per request). Per-IATA counts; cross-IATA senders are approximate all-regions. - -CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at)::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN packet_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at); - -CREATE UNIQUE INDEX idx_mv_top_talkers - ON mv_top_talkers_by_iata(iata, sender_name, bucket); diff --git a/db/migrations/019_mv_top_advertisers.sql b/db/migrations/019_mv_top_advertisers.sql deleted file mode 100644 index 47488c1c..00000000 --- a/db/migrations/019_mv_top_advertisers.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Hourly-bucketed advert counts (distinct ADVERT packets per node = adverts sent, --- not heard) so top-advertisers serves any window from a small table, not the live --- scan. Per-IATA counts; cross-IATA adverts are approximate all-regions. - -CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at)::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at); - -CREATE UNIQUE INDEX idx_mv_top_advertisers - ON mv_top_advertisers_by_iata(iata, node_id, bucket); diff --git a/db/migrations/020_add_iata_border.sql b/db/migrations/020_add_iata_border.sql deleted file mode 100644 index bc728402..00000000 --- a/db/migrations/020_add_iata_border.sql +++ /dev/null @@ -1,21 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Optional GeoJSON border for the region border map feature. Stores a full --- GeoJSON Feature (type/bbox/properties/geometry), not a bare geometry, so --- properties can be added later without a schema change. Geometry is always --- Polygon or MultiPolygon; bbox is computed and stored at write time (see --- internal/config/border.go) so reads never need to recompute it. --- --- NULL for the vast majority of IATAs, matching display_name/approx_lat/ --- approx_lng's existing "optional, mostly unset" pattern. Written via the --- same config-file-driven seeding path as those columns (internal/config/ --- seed.go), not a runtime HTTP write endpoint -- this codebase has none. --- --- Deliberately excluded from GET /api/v1/iatas (border polygons are large -- --- thousands of vertices, MultiPolygon for archipelago coastlines -- and --- would bloat every list response for the ~all IATAs that don't have one). --- Served only via GET /api/v1/iatas/{iata}/border, fetched lazily one at a --- time on client-side region selection. - -ALTER TABLE iata_codes ADD COLUMN border JSONB; diff --git a/db/migrations/021_mv_top_advertisers_route_type.sql b/db/migrations/021_mv_top_advertisers_route_type.sql deleted file mode 100644 index a2ac09b6..00000000 --- a/db/migrations/021_mv_top_advertisers_route_type.sql +++ /dev/null @@ -1,32 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Splits mv_top_advertisers_by_iata's advert_count into flood- and direct-routed counts, --- alongside the existing combined total, so /stats/top-advertisers can report how a node is --- actually being heard rather than only a single blended number. --- --- Flood = route_type 0 (transport_flood) or 1 (flood) -- broadcast, no prior path needed. --- Direct = route_type 2 (direct) or 3 (transport_direct) -- routed along a known path. - -DROP MATERIALIZED VIEW mv_top_advertisers_by_iata; - -CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at)::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at); - -CREATE UNIQUE INDEX idx_mv_top_advertisers - ON mv_top_advertisers_by_iata(iata, node_id, bucket); diff --git a/db/migrations/022_neighbor_scopes.sql b/db/migrations/022_neighbor_scopes.sql deleted file mode 100644 index 2a533826..00000000 --- a/db/migrations/022_neighbor_scopes.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Supports the observer /neighbors MQTT topic: a periodic report listing the --- OTA-configured region scope of the observer itself ("self") and of each --- zero-hop neighbor the observer was able to query. --- --- NOTE: these are plaintext OTA "region scope" strings (comma-separated, --- e.g. "*" or "US,CA") read from device config over the air. They are --- unrelated to the transport_scopes / observer_scopes / default_scope_id --- machinery elsewhere in this schema, which matches encrypted transport --- packets to named routing scopes by key fingerprint. Deliberately named --- differently (region_scope, not scope) to avoid confusion between the two. - -ALTER TABLE observers ADD COLUMN region_scope TEXT; - --- Per-neighbor-edge, not per-node: the OTA scope query is answered by --- whichever node reports it, in the context of a specific observer's --- zero-hop neighbor list, so it lives alongside snr on node_neighbors. -ALTER TABLE node_neighbors ADD COLUMN region_scope TEXT; diff --git a/db/migrations/023_channel_messages_cascade.sql b/db/migrations/023_channel_messages_cascade.sql deleted file mode 100644 index 582a8f0c..00000000 --- a/db/migrations/023_channel_messages_cascade.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- The packet_hash FK had no ON DELETE action, so packets carrying a channel --- message could never age out and DeleteOldPackets aborted. Match packet_observations. - -ALTER TABLE channel_messages - DROP CONSTRAINT IF EXISTS channel_messages_packet_hash_fkey; - -ALTER TABLE channel_messages - ADD CONSTRAINT channel_messages_packet_hash_fkey - FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; diff --git a/db/migrations/024_known_routes_pathkey.sql b/db/migrations/024_known_routes_pathkey.sql deleted file mode 100644 index ee121a41..00000000 --- a/db/migrations/024_known_routes_pathkey.sql +++ /dev/null @@ -1,43 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- known_routes grew unbounded (28 GB / 32.8M rows in 24 days on prod) and its --- UNIQUE(node_ids, iata) key indexed whole UUID arrays. Rebuild keyed on a --- 16-byte md5 of node_ids and add reconfirm bookkeeping. No rows are pruned --- here: retention is deployment config (routes.retention/grace), so the first --- cleanup tick after startup enforces it; a schema migration must not bake in --- one deployment's policy. id stays (the API serializes it) but loses its --- index; uniqueness lives on (iata, path_key). - -CREATE TABLE known_routes_new ( - id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, - path_key BYTEA NOT NULL, - node_ids UUID[] NOT NULL, - hash_prefix BYTEA[] NOT NULL, - iata CHAR(3) NOT NULL REFERENCES iata_codes(iata) ON DELETE CASCADE, - hop_count INT NOT NULL, - first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - last_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(), - observation_count BIGINT NOT NULL DEFAULT 1, - last_reconfirmed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - PRIMARY KEY (iata, path_key) -); - -INSERT INTO known_routes_new - (id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count) -SELECT id, - decode(md5(array_to_string(node_ids, ',')), 'hex'), - node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count -FROM known_routes; - -SELECT setval(pg_get_serial_sequence('known_routes_new', 'id'), - (SELECT COALESCE(MAX(id), 1) FROM known_routes_new)); - -DROP TABLE known_routes; -ALTER TABLE known_routes_new RENAME TO known_routes; -ALTER TABLE known_routes RENAME CONSTRAINT known_routes_new_pkey TO known_routes_pkey; -ALTER TABLE known_routes RENAME CONSTRAINT known_routes_new_iata_fkey TO known_routes_iata_fkey; - -CREATE INDEX idx_known_routes_hop_count ON known_routes(iata, hop_count); -CREATE INDEX idx_known_routes_last_seen ON known_routes(last_seen DESC); -CREATE INDEX idx_known_routes_reconfirm ON known_routes(last_reconfirmed_at); diff --git a/db/migrations/026_observation_endpoint_snapshots.sql b/db/migrations/026_observation_endpoint_snapshots.sql deleted file mode 100644 index 77a163b2..00000000 --- a/db/migrations/026_observation_endpoint_snapshots.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Preserve the endpoint resolution already computed for the live observation. --- SQL NULL means no endpoint nodes were captured; empty legacy captures also --- permit detail-time lookup once the node is known (for example, first adverts). --- No default/backfill: current node metadata cannot reconstruct past names. -ALTER TABLE packet_observations - ADD COLUMN IF NOT EXISTS resolved_endpoints JSONB; diff --git a/db/migrations/027_known_routes_iata_last_seen.sql b/db/migrations/027_known_routes_iata_last_seen.sql deleted file mode 100644 index b0bae7b2..00000000 --- a/db/migrations/027_known_routes_iata_last_seen.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- RunMigrations executes each file outside an explicit transaction. Keep this --- as one statement so other subscribers can keep writing during the build. --- If interrupted, check pg_index.indisvalid and remove this named index before --- retrying; do not silently accept an invalid index with IF NOT EXISTS. -CREATE INDEX CONCURRENTLY idx_known_routes_iata_last_seen -ON known_routes (iata, last_seen DESC); diff --git a/db/migrations/028_channels_last_seen_id.sql b/db/migrations/028_channels_last_seen_id.sql deleted file mode 100644 index 6c295090..00000000 --- a/db/migrations/028_channels_last_seen_id.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- RunMigrations executes this single statement outside an explicit transaction. --- If interrupted, check pg_index.indisvalid and remove this named index before --- retrying; do not silently accept an invalid index with IF NOT EXISTS. -CREATE INDEX CONCURRENTLY idx_channels_last_seen_id -ON channels (last_seen DESC, id DESC); diff --git a/db/migrations/029_drop_channels_last_seen.sql b/db/migrations/029_drop_channels_last_seen.sql deleted file mode 100644 index fadcd428..00000000 --- a/db/migrations/029_drop_channels_last_seen.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Migration 028 replaces this index with (last_seen DESC, id DESC). --- Keep this separate for concurrent execution; IF EXISTS allows a completed drop to retry. -DROP INDEX CONCURRENTLY IF EXISTS idx_channels_last_seen; diff --git a/db/migrations/030_observation_airtime.sql b/db/migrations/030_observation_airtime.sql deleted file mode 100644 index 1a7c29eb..00000000 --- a/db/migrations/030_observation_airtime.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Time-on-air per observation so per-observer activity never joins back to packets. -ALTER TABLE packet_observations ADD COLUMN airtime_ms REAL; diff --git a/db/migrations/031_observation_airtime_backfill.sql b/db/migrations/031_observation_airtime_backfill.sql deleted file mode 100644 index 0a74f7d6..00000000 --- a/db/migrations/031_observation_airtime_backfill.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Separate from 030 so the ALTER's ACCESS EXCLUSIVE lock is released before this backfill runs. - --- Backfill only: mirrors internal/lora.TimeOnAirMs (RadioLib getTimeOnAir, CRC on, explicit header). -CREATE FUNCTION lora_airtime_ms_backfill(frame_len int, sf int, bw real, cr int) RETURNS real -LANGUAGE sql IMMUTABLE AS $$ - SELECT ( - (CASE WHEN sf <= 8 THEN 32 ELSE 16 END) + 4.25 - + 8 + GREATEST(ceil((8*frame_len - 4*sf + 44)::double precision - / (4*(sf - 2*(CASE WHEN power(2, sf)/bw >= 16 THEN 1 ELSE 0 END)))) * cr, 0) - ) * (power(2, sf)/bw) -$$; - --- Last 7 days like 015; parallelism off so the join spills to disk, not /dev/shm. -SET max_parallel_workers_per_gather = 0; - -UPDATE packet_observations po -SET airtime_ms = lora_airtime_ms_backfill( - 1 + CASE WHEN p.transport_codes_present THEN 4 ELSE 0 END + 1 - + COALESCE(octet_length(po.path_bytes), 0) + octet_length(p.raw_payload), - po.spread_factor, po.bandwidth_khz, po.coding_rate) -FROM packets p -WHERE p.packet_hash = po.packet_hash - AND po.heard_at > NOW() - INTERVAL '7 days' - AND po.airtime_ms IS NULL - AND po.spread_factor BETWEEN 7 AND 12 - AND po.bandwidth_khz > 0 - AND po.coding_rate BETWEEN 5 AND 8; - -RESET max_parallel_workers_per_gather; -DROP FUNCTION lora_airtime_ms_backfill(int, int, real, int); diff --git a/db/migrations/032_mv_observer_activity.sql b/db/migrations/032_mv_observer_activity.sql deleted file mode 100644 index 36407593..00000000 --- a/db/migrations/032_mv_observer_activity.sql +++ /dev/null @@ -1,24 +0,0 @@ --- Hourly per-observer rollup so 7d/30d activity sums a few thousand rows instead of scanning --- a month of observations. payload_type is a key so the breakdown comes from the same rows. --- sqlc types the cast aggregates as non-null even though airtime_ms/snr_*/rssi_sum are NULL for --- empty groups, so readers must gate on the matching *_n counts. -CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS -SELECT - observer_id, - payload_type, - date_trunc('hour', heard_at)::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY observer_id, payload_type, date_trunc('hour', heard_at); - -CREATE UNIQUE INDEX idx_mv_observer_activity_hourly - ON mv_observer_activity_hourly(observer_id, payload_type, bucket); diff --git a/db/migrations/033_telemetry_airtime_secs.sql b/db/migrations/033_telemetry_airtime_secs.sql deleted file mode 100644 index 3ca4d364..00000000 --- a/db/migrations/033_telemetry_airtime_secs.sql +++ /dev/null @@ -1,4 +0,0 @@ --- These always held seconds of radio time from the observer's tx_air_secs/rx_air_secs --- status fields, never a percentage; name them for what they are. -ALTER TABLE observer_telemetry RENAME COLUMN airtime_tx_pct TO airtime_tx_secs; -ALTER TABLE observer_telemetry RENAME COLUMN airtime_rx_pct TO airtime_rx_secs; diff --git a/db/migrations/034_accounts.sql b/db/migrations/034_accounts.sql deleted file mode 100644 index 16b57fc5..00000000 --- a/db/migrations/034_accounts.sql +++ /dev/null @@ -1,12 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - -CREATE TABLE IF NOT EXISTS accounts ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name TEXT NOT NULL CHECK (char_length(name) BETWEEN 1 AND 128 AND name = btrim(name)), - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - deactivated_at TIMESTAMPTZ -); - -CREATE UNIQUE INDEX IF NOT EXISTS accounts_name_uidx - ON accounts (name) WHERE deactivated_at IS NULL; diff --git a/db/migrations/035_mv_signal_stats.sql b/db/migrations/035_mv_signal_stats.sql deleted file mode 100644 index 14b3dd3f..00000000 --- a/db/migrations/035_mv_signal_stats.sql +++ /dev/null @@ -1,33 +0,0 @@ --- Reception-IATA rollups include legacy NULL payload types and exclude non-finite --- SNR. mv_observer_activity_hourly has a different grain/filter, so reusing its --- means would disagree with these histograms and the reception coverage counts. -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_signal_stats_hourly AS -WITH samples AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM packet_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - AND heard_at < date_trunc('hour', NOW(), 'UTC') -), binned AS ( - SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples -) -SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(sum(snr), 0)::double precision AS snr_sum, - count(rssi)::bigint AS rssi_samples, - COALESCE(sum(rssi), 0)::double precision AS rssi_sum -FROM binned -GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_signal_stats_hourly - ON mv_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin); -CREATE INDEX IF NOT EXISTS idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly (hour); diff --git a/db/migrations/036_mv_path_stats.sql b/db/migrations/036_mv_path_stats.sql deleted file mode 100644 index 7612255c..00000000 --- a/db/migrations/036_mv_path_stats.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Classify once per background refresh, preserving the decoder's header rules. --- No observation rows or deduplication constraints are changed. -CREATE MATERIALIZED VIEW IF NOT EXISTS mv_path_stats_hourly AS -WITH classified AS ( - SELECT iata, heard_at, hash_size, hop_count, - CASE WHEN payload_type = 9 THEN 2 - -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). - WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 - OR NOT (path_length_byte BETWEEN 0 AND 191 - AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 - AND hash_size = (path_length_byte >> 6) + 1 - AND hop_count = (path_length_byte & 63) - AND hash_size::integer * hop_count::integer <= 64 - AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) - THEN 3 - WHEN hop_count = 0 THEN 1 - ELSE 0 END::integer AS category - FROM packet_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - AND heard_at < date_trunc('hour', NOW(), 'UTC') -), buckets AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, - CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, - CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries - FROM classified -) -SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions -FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; -CREATE UNIQUE INDEX IF NOT EXISTS idx_mv_path_stats_hourly - ON mv_path_stats_hourly (iata, hour, category, hash_bytes, entries); -CREATE INDEX IF NOT EXISTS idx_mv_path_stats_hourly_hour ON mv_path_stats_hourly (hour); diff --git a/db/migrations/037_autovacuum_tuning.sql b/db/migrations/037_autovacuum_tuning.sql deleted file mode 100644 index f8db967d..00000000 --- a/db/migrations/037_autovacuum_tuning.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Default 20% trigger never fired on known_routes, bloating its indexes. -ALTER TABLE known_routes SET ( - autovacuum_vacuum_scale_factor = 0.02, - autovacuum_analyze_scale_factor = 0.02, - autovacuum_vacuum_cost_limit = 1000 -); -ALTER TABLE packets SET (autovacuum_vacuum_scale_factor = 0.05, autovacuum_vacuum_cost_limit = 1000); -ALTER TABLE packet_observations SET (autovacuum_vacuum_scale_factor = 0.05, autovacuum_vacuum_cost_limit = 1000); -ALTER TABLE nodes SET (autovacuum_vacuum_scale_factor = 0.05); -ALTER TABLE node_iatas SET (autovacuum_vacuum_scale_factor = 0.05); -ALTER TABLE node_short_ids SET (autovacuum_vacuum_scale_factor = 0.05); diff --git a/db/migrations/038_drop_resolved_endpoints.sql b/db/migrations/038_drop_resolved_endpoints.sql deleted file mode 100644 index e69190ec..00000000 --- a/db/migrations/038_drop_resolved_endpoints.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Endpoints now resolve at read time like path hops. The per-observation --- snapshots were ~60% of packet_observations on prod. -ALTER TABLE packet_observations DROP COLUMN IF EXISTS resolved_endpoints; diff --git a/db/migrations/039_analytics_retention.sql b/db/migrations/039_analytics_retention.sql deleted file mode 100644 index 069204ad..00000000 --- a/db/migrations/039_analytics_retention.sql +++ /dev/null @@ -1,613 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Archive only aggregates of deleted packet cohorts, never packet/message bodies --- or raw paths. Live and archived cohorts are disjoint in every refresh snapshot. --- Thirty days matches the longest analytics window; raw retention is independent. - - -CREATE TABLE IF NOT EXISTS analytics_hourly_iata_stats ( - iata char(3), - hour timestamptz, - observation_count bigint NOT NULL DEFAULT 0, - unique_packets bigint NOT NULL DEFAULT 0, - PRIMARY KEY (iata,hour) -); - -CREATE INDEX IF NOT EXISTS idx_analytics_hourly_iata_stats_time ON analytics_hourly_iata_stats (hour); - -CREATE TABLE IF NOT EXISTS analytics_payload_breakdown_by_iata ( - iata char(3), - payload_type smallint, - bucket timestamptz, - count bigint NOT NULL DEFAULT 0, - PRIMARY KEY (iata,payload_type,bucket) -); - -CREATE INDEX IF NOT EXISTS idx_analytics_payload_breakdown_by_iata_time ON analytics_payload_breakdown_by_iata (bucket); - -CREATE TABLE IF NOT EXISTS analytics_top_observers_by_iata ( - iata char(3), - observer_id uuid, - bucket timestamptz, - observation_count bigint NOT NULL DEFAULT 0, - display_name text, - observer_type text, - PRIMARY KEY (iata,observer_id,bucket) -); - -CREATE INDEX IF NOT EXISTS idx_analytics_top_observers_by_iata_time ON analytics_top_observers_by_iata (bucket); - -CREATE TABLE IF NOT EXISTS analytics_top_talkers_by_iata ( - iata char(3), - sender_name text, - bucket timestamptz, - message_count bigint NOT NULL DEFAULT 0, - last_sent timestamptz, - PRIMARY KEY (iata,sender_name,bucket) -); - -CREATE INDEX IF NOT EXISTS idx_analytics_top_talkers_by_iata_time ON analytics_top_talkers_by_iata (bucket); - -CREATE TABLE IF NOT EXISTS analytics_top_advertisers_by_iata ( - iata char(3), - node_id uuid, - bucket timestamptz, - advert_count bigint NOT NULL DEFAULT 0, - flood_advert_count bigint NOT NULL DEFAULT 0, - direct_advert_count bigint NOT NULL DEFAULT 0, - last_heard timestamptz, - name text, - node_type smallint, - PRIMARY KEY (iata,node_id,bucket) -); - -CREATE INDEX IF NOT EXISTS idx_analytics_top_advertisers_by_iata_time ON analytics_top_advertisers_by_iata (bucket); - -CREATE TABLE IF NOT EXISTS analytics_observer_activity_hourly ( - observer_id uuid, - payload_type smallint, - bucket timestamptz, - observations bigint NOT NULL DEFAULT 0, - airtime_ms real, - airtime_n bigint NOT NULL DEFAULT 0, - snr_sum real, - snr_n bigint NOT NULL DEFAULT 0, - snr_min real, - rssi_sum bigint, - rssi_n bigint NOT NULL DEFAULT 0, - PRIMARY KEY (observer_id,payload_type,bucket) -); - -CREATE INDEX IF NOT EXISTS idx_analytics_observer_activity_hourly_time ON analytics_observer_activity_hourly (bucket); - -CREATE TABLE IF NOT EXISTS analytics_signal_stats_hourly ( - iata char(3), - hour timestamptz, - kind integer, - snr_bin integer, - rssi_bin integer, - receptions bigint NOT NULL DEFAULT 0, - snr_samples bigint NOT NULL DEFAULT 0, - snr_sum double precision, - rssi_samples bigint NOT NULL DEFAULT 0, - rssi_sum double precision, - PRIMARY KEY (iata,hour,kind,snr_bin,rssi_bin) -); - -CREATE INDEX IF NOT EXISTS idx_analytics_signal_stats_hourly_time ON analytics_signal_stats_hourly (hour); - -CREATE TABLE IF NOT EXISTS analytics_path_stats_hourly ( - iata char(3), - hour timestamptz, - category integer, - hash_bytes integer, - entries integer, - receptions bigint NOT NULL DEFAULT 0, - PRIMARY KEY (iata,hour,category,hash_bytes,entries) -); - -CREATE INDEX IF NOT EXISTS idx_analytics_path_stats_hourly_time ON analytics_path_stats_hourly (hour); - -DROP MATERIALIZED VIEW IF EXISTS mv_hourly_iata_stats; - -CREATE OR REPLACE VIEW analytics_live_hourly_iata_stats AS -SELECT - iata, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' -GROUP BY iata, date_trunc('hour', heard_at, 'UTC'); - -CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS -WITH combined AS ( - SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats - UNION ALL - SELECT iata, hour, observation_count, unique_packets FROM analytics_hourly_iata_stats - WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), observers_by_hour AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, observer_id - FROM packet_observations WHERE heard_at > NOW() - INTERVAL '30 days' - UNION - SELECT iata, bucket AS hour, observer_id FROM analytics_top_observers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), observer_counts AS (SELECT iata, hour, COUNT(*) AS n FROM observers_by_hour GROUP BY iata,hour) -SELECT combined.iata, combined.hour, SUM(observation_count)::bigint AS observation_count, SUM(unique_packets)::bigint AS unique_packets, MAX(o.n)::bigint AS active_observers -FROM combined JOIN observer_counts o USING (iata,hour) GROUP BY combined.iata,combined.hour; - -CREATE UNIQUE INDEX idx_analytics_hourly_iata_stats_view ON mv_hourly_iata_stats (iata,hour); - - -DROP MATERIALIZED VIEW IF EXISTS mv_payload_breakdown_by_iata; - -CREATE OR REPLACE VIEW analytics_live_payload_breakdown_by_iata AS -SELECT - iata, - payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS count -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC'); - -CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS -WITH combined AS ( - SELECT iata, payload_type, bucket, count FROM analytics_live_payload_breakdown_by_iata - UNION ALL - SELECT iata, payload_type, bucket, count FROM analytics_payload_breakdown_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT iata, payload_type, bucket, SUM(count)::bigint AS count -FROM combined GROUP BY iata,payload_type,bucket; - -CREATE UNIQUE INDEX idx_analytics_payload_breakdown_by_iata_view ON mv_payload_breakdown_by_iata (iata,payload_type,bucket); - - -DROP MATERIALIZED VIEW IF EXISTS mv_top_observers_by_iata; - -CREATE OR REPLACE VIEW analytics_live_top_observers_by_iata AS -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM packet_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC'); - -CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS -WITH combined AS ( - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_live_top_observers_by_iata - UNION ALL - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_top_observers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), totals AS (SELECT iata, observer_id, bucket, SUM(observation_count)::bigint AS observation_count, MAX(display_name) AS display_name, MAX(observer_type) AS observer_type -FROM combined GROUP BY iata,observer_id,bucket) -SELECT t.iata, t.observer_id, COALESCE(o.display_name,t.display_name) AS display_name, - COALESCE(o.observer_type,t.observer_type) AS observer_type, t.bucket, t.observation_count -FROM totals t LEFT JOIN observers o ON o.id=t.observer_id; - -CREATE UNIQUE INDEX idx_analytics_top_observers_by_iata_view ON mv_top_observers_by_iata (iata,observer_id,bucket); - - -DROP MATERIALIZED VIEW IF EXISTS mv_top_talkers_by_iata; - -CREATE OR REPLACE VIEW analytics_live_top_talkers_by_iata AS -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN packet_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC'); - -CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS -WITH combined AS ( - SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_live_top_talkers_by_iata - UNION ALL - SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_top_talkers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT iata, sender_name, bucket, SUM(message_count)::bigint AS message_count, MAX(last_sent) AS last_sent -FROM combined GROUP BY iata,sender_name,bucket; - -CREATE UNIQUE INDEX idx_analytics_top_talkers_by_iata_view ON mv_top_talkers_by_iata (iata,sender_name,bucket); - - -DROP MATERIALIZED VIEW IF EXISTS mv_top_advertisers_by_iata; - -CREATE OR REPLACE VIEW analytics_live_top_advertisers_by_iata AS -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC'); - -CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS -WITH combined AS ( - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_live_top_advertisers_by_iata - UNION ALL - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_top_advertisers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), totals AS (SELECT iata, node_id, bucket, SUM(advert_count)::bigint AS advert_count, SUM(flood_advert_count)::bigint AS flood_advert_count, SUM(direct_advert_count)::bigint AS direct_advert_count, MAX(last_heard) AS last_heard, MAX(name) AS name, MAX(node_type) AS node_type -FROM combined GROUP BY iata,node_id,bucket) -SELECT t.iata, t.node_id, COALESCE(n.name,t.name) AS name, COALESCE(n.node_type,t.node_type) AS node_type, - t.bucket, t.advert_count, t.flood_advert_count, t.direct_advert_count, t.last_heard -FROM totals t LEFT JOIN nodes n ON n.id=t.node_id; - -CREATE UNIQUE INDEX idx_analytics_top_advertisers_by_iata_view ON mv_top_advertisers_by_iata (iata,node_id,bucket); - - -DROP MATERIALIZED VIEW IF EXISTS mv_observer_activity_hourly; -CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); - -CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS -WITH combined AS ( - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly - UNION ALL - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM combined GROUP BY observer_id,payload_type,bucket; - -CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); - - - -DROP MATERIALIZED VIEW IF EXISTS mv_signal_stats_hourly; - -CREATE OR REPLACE VIEW analytics_live_signal_stats_hourly AS -WITH samples AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM packet_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - AND heard_at < date_trunc('hour', NOW(), 'UTC') -), binned AS ( - SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples -) -SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(sum(snr), 0)::double precision AS snr_sum, - count(rssi)::bigint AS rssi_samples, - COALESCE(sum(rssi), 0)::double precision AS rssi_sum -FROM binned -GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); - -CREATE MATERIALIZED VIEW mv_signal_stats_hourly AS -WITH combined AS ( - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_live_signal_stats_hourly - UNION ALL - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_signal_stats_hourly - WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND hour < date_trunc('hour', NOW(), 'UTC') -) -SELECT iata, hour, kind, snr_bin, rssi_bin, SUM(receptions)::bigint AS receptions, SUM(snr_samples)::bigint AS snr_samples, SUM(snr_sum)::double precision AS snr_sum, SUM(rssi_samples)::bigint AS rssi_samples, SUM(rssi_sum)::double precision AS rssi_sum -FROM combined GROUP BY iata,hour,kind,snr_bin,rssi_bin; - -CREATE UNIQUE INDEX idx_analytics_signal_stats_hourly_view ON mv_signal_stats_hourly (iata,hour,kind,snr_bin,rssi_bin); -CREATE INDEX idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly (hour); - - -DROP MATERIALIZED VIEW IF EXISTS mv_path_stats_hourly; - -CREATE OR REPLACE VIEW analytics_live_path_stats_hourly AS -WITH classified AS ( - SELECT iata, heard_at, hash_size, hop_count, - CASE WHEN payload_type = 9 THEN 2 - -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). - WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 - OR NOT (path_length_byte BETWEEN 0 AND 191 - AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 - AND hash_size = (path_length_byte >> 6) + 1 - AND hop_count = (path_length_byte & 63) - AND hash_size::integer * hop_count::integer <= 64 - AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) - THEN 3 - WHEN hop_count = 0 THEN 1 - ELSE 0 END::integer AS category - FROM packet_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - AND heard_at < date_trunc('hour', NOW(), 'UTC') -), buckets AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, - CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, - CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries - FROM classified -) -SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions -FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; - -CREATE MATERIALIZED VIEW mv_path_stats_hourly AS -WITH combined AS ( - SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_live_path_stats_hourly - UNION ALL - SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_path_stats_hourly - WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND hour < date_trunc('hour', NOW(), 'UTC') -) -SELECT iata, hour, category, hash_bytes, entries, SUM(receptions)::bigint AS receptions -FROM combined GROUP BY iata,hour,category,hash_bytes,entries; - -CREATE UNIQUE INDEX idx_analytics_path_stats_hourly_view ON mv_path_stats_hourly (iata,hour,category,hash_bytes,entries); -CREATE INDEX idx_mv_path_stats_hourly_hour ON mv_path_stats_hourly (hour); - - --- VOLATILE gives the archive statement a new READ COMMITTED snapshot after --- acquiring parent locks. FK inserts cannot race between that snapshot and the --- cascade. SKIP LOCKED leaves packets being ingested for a later cleanup tick. -CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) -RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ -DECLARE hashes bytea[]; deleted bigint; -BEGIN - -- Serialize cleanup runners before overlapping aggregate upserts. - PERFORM pg_advisory_xact_lock(hashtext('beacon.archive_delete_packets')); - IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; - SELECT array_agg(p.packet_hash) INTO hashes FROM ( - SELECT ep.packet_hash FROM packets ep - WHERE ep.last_heard_at < cutoff - ORDER BY ep.last_heard_at, ep.packet_hash - LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED - ) p; - - WITH expired_observations AS MATERIALIZED ( - SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) - ), -archived_hourly_iata_stats AS ( - INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) - SELECT iata, hour, observation_count, unique_packets FROM ( -SELECT - iata, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY iata, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,hour) DO UPDATE SET - observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, - unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets -), -archived_payload_breakdown_by_iata AS ( - INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) - SELECT iata, payload_type, bucket, count FROM ( -SELECT - iata, - payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS count -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET - count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count -), -archived_top_observers_by_iata AS ( - INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM expired_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET - observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, - display_name = EXCLUDED.display_name, - observer_type = EXCLUDED.observer_type -), -archived_top_talkers_by_iata AS ( - INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) - SELECT iata, sender_name, bucket, message_count, last_sent FROM ( -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN expired_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') - ) batch - ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET - message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, - last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) -), -archived_top_advertisers_by_iata AS ( - INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN expired_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,node_id,bucket) DO UPDATE SET - advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, - flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, - direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, - last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), - name = EXCLUDED.name, - node_type = EXCLUDED.node_type -), -archived_observer_activity_hourly AS ( - INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET - observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, - airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, - airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, - snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, - snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, - snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), - rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, - rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n -), -archived_signal_stats_hourly AS ( - INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( -WITH samples AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), binned AS ( - SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples -) -SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(sum(snr), 0)::double precision AS snr_sum, - count(rssi)::bigint AS rssi_samples, - COALESCE(sum(rssi), 0)::double precision AS rssi_sum -FROM binned -GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) - ) batch - ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET - receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, - snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, - snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, - rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, - rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum -), -archived_path_stats_hourly AS ( - INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) - SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( -WITH classified AS ( - SELECT iata, heard_at, hash_size, hop_count, - CASE WHEN payload_type = 9 THEN 2 - -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). - WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 - OR NOT (path_length_byte BETWEEN 0 AND 191 - AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 - AND hash_size = (path_length_byte >> 6) + 1 - AND hop_count = (path_length_byte & 63) - AND hash_size::integer * hop_count::integer <= 64 - AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) - THEN 3 - WHEN hop_count = 0 THEN 1 - ELSE 0 END::integer AS category - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), buckets AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, - CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, - CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries - FROM classified -) -SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions -FROM buckets GROUP BY iata, hour, category, hash_bytes, entries - ) batch - ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET - receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions -) - DELETE FROM packets WHERE packet_hash = ANY(hashes); - GET DIAGNOSTICS deleted = ROW_COUNT; - - -- Once per completed cleanup, including when there were no raw packets to delete. - IF deleted < batch_size THEN - DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - END IF; - RETURN deleted; -END $$; diff --git a/db/migrations/041_route_evidence_index.sql b/db/migrations/041_route_evidence_index.sql deleted file mode 100644 index 71088cc5..00000000 --- a/db/migrations/041_route_evidence_index.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- One statement outside a transaction. RunMigrations recovers interrupted builds; --- avoid IF NOT EXISTS, which would silently accept an invalid index. --- A compact digest narrows candidates; queries still compare the complete bytes. -CREATE INDEX CONCURRENTLY idx_observations_route_evidence -ON packet_observations (iata, hash_size, (decode(md5(path_bytes), 'hex')), heard_at DESC, id DESC) -WHERE path_bytes IS NOT NULL AND payload_type IS NOT NULL AND payload_type <> 9 AND hop_count >= 2; diff --git a/db/migrations/042_meshmapper_scopes.sql b/db/migrations/042_meshmapper_scopes.sql deleted file mode 100644 index 43b48b6c..00000000 --- a/db/migrations/042_meshmapper_scopes.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Preserve historical identities while distinguishing import-only matching keys. -ALTER TABLE transport_scopes ADD COLUMN IF NOT EXISTS imported_only BOOLEAN NOT NULL DEFAULT FALSE; - -CREATE TABLE IF NOT EXISTS meshmapper_scope_catalogues ( - iata TEXT NOT NULL, - url TEXT NOT NULL, - payload JSONB, - etag TEXT, - checked_at TIMESTAMPTZ, - attempted_at TIMESTAMPTZ NOT NULL, - next_attempt TIMESTAMPTZ NOT NULL, - last_error TEXT NOT NULL DEFAULT '', - PRIMARY KEY (iata, url) -); diff --git a/db/migrations/043_clear_zero_locations.sql b/db/migrations/043_clear_zero_locations.sql deleted file mode 100644 index b4a83b41..00000000 --- a/db/migrations/043_clear_zero_locations.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- An explicit 0/0 advert now clears the stored position; drop old 0/0 rows and stale sources. -UPDATE nodes SET latitude = NULL, longitude = NULL, location_source = NULL -WHERE location_source = 'advert' AND (latitude IS NULL OR (latitude = 0 AND longitude = 0)); diff --git a/db/migrations/044_meshmapper_zones.sql b/db/migrations/044_meshmapper_zones.sql deleted file mode 100644 index 0103225f..00000000 --- a/db/migrations/044_meshmapper_zones.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Imported MeshMapper region boundaries. Kept apart from iata_codes.border so --- startup seeding of manual borderFile borders never undoes the override. -CREATE TABLE IF NOT EXISTS meshmapper_zone_boundaries ( - iata TEXT PRIMARY KEY, - url TEXT NOT NULL, - feature JSONB, - etag TEXT, - checked_at TIMESTAMPTZ, - attempted_at TIMESTAMPTZ NOT NULL, - next_attempt TIMESTAMPTZ NOT NULL, - last_error TEXT NOT NULL DEFAULT '' -); diff --git a/db/migrations/045_clean_partial_observer_telemetry.sql b/db/migrations/045_clean_partial_observer_telemetry.sql deleted file mode 100644 index b5bdfadf..00000000 --- a/db/migrations/045_clean_partial_observer_telemetry.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Some /status messages carry uptime but no radio stats. Those rows zero the airtime and --- noise floor, dragging bucket averages down; ingest now skips them (statusStats.usable). -DELETE FROM observer_telemetry -WHERE noise_floor_db = 0 AND airtime_tx_secs = 0 AND airtime_rx_secs = 0; diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 91dbeca1..64c82df8 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -7,35 +7,21 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" - "strings" "testing" "time" ) func TestObserverMetricsPostgres(t *testing.T) { ctx, tx := retentionTx(t) - analyticsTables(t, ctx, tx) - // Use the real observer DDL without depending on a preinstalled public schema. - initial, err := migrationFiles.ReadFile("migrations/001_initial_schema.sql") - if err != nil { - t.Fatal(err) - } - ddl := string(initial) - start := strings.Index(ddl, "CREATE TABLE observers (") - end := strings.Index(ddl[start:], "\n);") + start + 3 - if _, err := tx.Exec(ctx, "DROP TABLE observers; "+ddl[start:end]+" ALTER TABLE observers ADD COLUMN region_scope TEXT;"); err != nil { - t.Fatal(err) - } + applyBaseline(t, ctx, tx) if _, err := tx.Exec(ctx, ` - CREATE TABLE observer_brokers (observer_id uuid,broker_name text,last_seen timestamptz,last_packet_at timestamptz,PRIMARY KEY(observer_id,broker_name)); INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); - INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; + INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,last_heard_at,first_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash IN (int4send(1),int4send(2)) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; `); err != nil { t.Fatal(err) } - applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") store := &Store{q: sqlc.New(tx)} id := uuid.MustParse("00000000-0000-0000-0000-000000000001") if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { @@ -59,9 +45,9 @@ func TestObserverMetricsPostgres(t *testing.T) { if activity.WindowEnd != until.UnixMilli() || activity.WindowStart != until.Add(-7*24*time.Hour).UnixMilli() || activity.Source != "hourly" { t.Fatalf("window: %+v", activity) } - if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,last_heard_at) VALUES ('\xaa',NOW()); + if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) VALUES ('\xaa',4,0,1,'\x00','\x00',NOW(),NOW()); INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,'NaN'::real,-100); - INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW()) ON CONFLICT DO NOTHING;`); err != nil { + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW(),0,1,0) ON CONFLICT DO NOTHING;`); err != nil { t.Fatal(err) } raw, err := store.GetObserverActivity(ctx, id, time.Hour, 15*time.Minute, until) diff --git a/db/paths_integration_test.go b/db/paths_integration_test.go index e10e7935..76a96628 100644 --- a/db/paths_integration_test.go +++ b/db/paths_integration_test.go @@ -36,11 +36,11 @@ func TestPathsPostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) - isolateStatsSchema(t, ctx, tx) + applyBaseline(t, ctx, tx) _, err = tx.Exec(ctx, ` SET LOCAL TIME ZONE 'America/Vancouver'; -CREATE TABLE packet_observations(heard_at timestamptz NOT NULL,iata char(3) NOT NULL,payload_type smallint,path_length_byte smallint NOT NULL,hash_size smallint NOT NULL,hop_count smallint NOT NULL,path_bytes bytea); -INSERT INTO packet_observations +CREATE TEMP TABLE fixture(heard_at timestamptz NOT NULL,iata char(3) NOT NULL,payload_type smallint,path_length_byte smallint NOT NULL,hash_size smallint NOT NULL,hop_count smallint NOT NULL,path_bytes bytea); +INSERT INTO fixture SELECT CASE id WHEN 1 THEN '2026-01-01 00:30+00'::timestamptz WHEN 17 THEN '2026-01-01 04:00+00'::timestamptz WHEN 18 THEN '2026-01-01 04:30+00'::timestamptz WHEN 20 THEN '2026-01-01 03:00+00'::timestamptz ELSE '2026-01-01 00:30+00'::timestamptz+(id/10)*interval '2 hour'+(id%10)*interval '1 minute' END, iata,payload,raw,width,hops,CASE WHEN size IS NULL THEN NULL ELSE decode(repeat('ab',size),'hex') END @@ -52,7 +52,13 @@ FROM (VALUES (13,'YYZ',5,0,1,0,0),(14,'YVR',16,1,1,1,1),(15,'YVR',4,63,1,63,63), (16,'YVR',4,149,3,21,63),(17,'YVR',4,1,1,1,1),(18,'YVR',4,1,1,1,1), (19,'YYJ',9,3,1,3,3),(20,'YVR',4,64,2,0,0) -) v(id,iata,payload,raw,width,hops,size);`) +) v(id,iata,payload,raw,width,hops,size); +ALTER TABLE fixture ADD COLUMN n serial; +INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT int4send(n),4,0,1,'\x00','\x00',heard_at,heard_at FROM fixture; +INSERT INTO packet_observations (packet_hash,observer_id,heard_at,iata,payload_type,path_length_byte,hash_size,hop_count,path_bytes) +SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_type,path_length_byte,hash_size,hop_count,path_bytes FROM fixture;`) if err != nil { t.Fatal(err) } @@ -60,8 +66,9 @@ FROM (VALUES if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { t.Fatal(err) } - applyStatsMigration(t, ctx, tx, "036_mv_path_stats.sql") - applyStatsMigration(t, ctx, tx, "036_mv_path_stats.sql") // interrupted journal retry + if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW mv_path_stats_hourly"); err != nil { + t.Fatal(err) + } store := &Store{q: sqlc.New(tx)} until := since.Add(4 * time.Hour) for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { @@ -116,7 +123,12 @@ FROM (VALUES if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { t.Fatal(err) } - if _, err = tx.Exec(ctx, `INSERT INTO packet_observations SELECT $1::timestamptz+interval '1 hour','YVR',4,i,(i>>6)+1,i&63,decode(repeat('ab',((i>>6)+1)*(i&63)),'hex') FROM generate_series(0,255)i;`, since); err != nil { + if _, err = tx.Exec(ctx, `INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT int4send(1000+i),4,0,1,'\x00','\x00',$1,$1 FROM generate_series(0,255)i`, since); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `INSERT INTO packet_observations (packet_hash,observer_id,heard_at,iata,payload_type,path_length_byte,hash_size,hop_count,path_bytes) +SELECT int4send(1000+i),'00000000-0000-0000-0000-000000000001',$1::timestamptz+interval '1 hour','YVR',4,i,(i>>6)+1,i&63,decode(repeat('ab',((i>>6)+1)*(i&63)),'hex') FROM generate_series(0,255)i;`, since); err != nil { t.Fatal(err) } stale, err := store.GetPathStats(ctx, since, until, nil) diff --git a/db/retention_integration_test.go b/db/retention_integration_test.go index d6307176..cff7dae0 100644 --- a/db/retention_integration_test.go +++ b/db/retention_integration_test.go @@ -46,9 +46,9 @@ func countRows(t *testing.T, ctx context.Context, tx pgx.Tx, sql string) int { // More than one batch of old packets; observations go with them. func TestDeleteOldPacketsBatchesPostgres(t *testing.T) { ctx, tx := retentionTx(t) - analyticsTables(t, ctx, tx) - applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + applyBaseline(t, ctx, tx) _, err := tx.Exec(ctx, ` +INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x01'); INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) SELECT int4send(i), 4, 0, 1, '\x00', '\x00', CASE WHEN i <= 2500 THEN '2026-01-01'::timestamptz ELSE '2026-02-01'::timestamptz END, diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index f4a3e177..138c80ba 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -7,8 +7,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "fmt" - "os" "strings" "testing" "time" @@ -35,30 +33,15 @@ func (c *evidenceCapture) Query(ctx context.Context, sql string, args ...any) (p func TestRouteEvidencePostgres(t *testing.T) { ctx, tx := retentionTx(t) - analyticsTables(t, ctx, tx) - ddl, err := migrationFiles.ReadFile("migrations/024_known_routes_pathkey.sql") + applyBaseline(t, ctx, tx) + _, err := tx.Exec(ctx, ` + INSERT INTO observers(id,public_key,display_name) VALUES ('00000000-0000-0000-0000-000000000001','\x01','One'),('00000000-0000-0000-0000-000000000002','\x02','Two'); + INSERT INTO nodes(id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa',2,'A'),('00000000-0000-0000-0000-000000000002','\xbb',2,'B'); + INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) + SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(1,10) i;`) if err != nil { t.Fatal(err) } - text := string(ddl) - start := strings.Index(text, "CREATE TABLE known_routes_new (") - end := start + strings.Index(text[start:], "\n);") + 3 - table := strings.ReplaceAll(text[start:end], "known_routes_new", "known_routes") - _, err = tx.Exec(ctx, `CREATE TABLE iata_codes (iata char(3) PRIMARY KEY); INSERT INTO iata_codes VALUES ('YOW'),('YVR'); - ALTER TABLE nodes ADD COLUMN latitude double precision, ADD COLUMN longitude double precision;`+table+` - INSERT INTO observers(id,display_name) VALUES ('00000000-0000-0000-0000-000000000001','One'),('00000000-0000-0000-0000-000000000002','Two'); - INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa','A'),('00000000-0000-0000-0000-000000000002','\xbb','B'); - INSERT INTO packets(packet_hash) SELECT int4send(i) FROM generate_series(1,10) i;`) - if err != nil { - t.Fatal(err) - } - index, err := migrationFiles.ReadFile("migrations/041_route_evidence_index.sql") - if err != nil { - t.Fatal(err) - } - if _, err := tx.Exec(ctx, strings.Replace(string(index), "CREATE INDEX CONCURRENTLY", "CREATE INDEX", 1)); err != nil { - t.Fatal(err) - } capture := &evidenceCapture{Tx: tx} store := &Store{q: sqlc.New(capture)} ids := []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000001"), uuid.MustParse("00000000-0000-0000-0000-000000000002")} @@ -69,17 +52,20 @@ func TestRouteEvidencePostgres(t *testing.T) { since := time.Now().UTC().Truncate(time.Millisecond).Add(-time.Hour) until := since.Add(time.Hour) at := since.Add(time.Minute + 123456*time.Microsecond) - _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,snr) + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,snr,path_length_byte) SELECT i,int4send(i), '00000000-0000-0000-0000-000000000001',CASE WHEN i=5 THEN 'YVR' ELSE 'YOW' END, CASE WHEN i=9 THEN $2::timestamptz WHEN i=1 THEN $1::timestamptz-interval '2 microseconds' ELSE $1::timestamptz END, CASE WHEN i=4 THEN 2 ELSE 1 END,CASE WHEN i=4 THEN 1 WHEN i=10 THEN 3 ELSE 2 END, CASE WHEN i=6 THEN '\xabbb'::bytea WHEN i=10 THEN '\xaabbcc'::bytea ELSE '\xaabb'::bytea END, - CASE WHEN i=7 THEN 9 WHEN i=8 THEN NULL ELSE 4 END,CASE WHEN i=1 THEN 'NaN'::real ELSE 0 END FROM generate_series(1,10) i; + CASE WHEN i=7 THEN 9 WHEN i=8 THEN NULL ELSE 4 END,CASE WHEN i=1 THEN 'NaN'::real ELSE 0 END,0 FROM generate_series(1,10) i; `, at, until) if err != nil { t.Fatal(err) } - if _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) VALUES (11,int4send(1),'00000000-0000-0000-0000-000000000002','YOW',$1,1,2,'\xaabb',4)`, at.Add(-time.Microsecond)); err != nil { + if _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,path_length_byte) VALUES (11,int4send(1),'00000000-0000-0000-0000-000000000002','YOW',$1,1,2,'\xaabb',4,0)`, at.Add(-time.Microsecond)); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `UPDATE packet_observations SET path_length_byte=((hash_size-1)<<6)|hop_count`); err != nil { t.Fatal(err) } query := api.RouteEvidenceQuery{Since: since, Until: until, Limit: 2} @@ -122,12 +108,13 @@ func TestRouteEvidencePostgres(t *testing.T) { // Many unrelated paths must not turn this into a raw-table or route-ID scan. started := time.Now() - _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash) SELECT int4send(i) FROM generate_series(100,200099) i;`) + _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) + SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(100,200099) i;`) if err != nil { t.Fatal(err) } - _, err = tx.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) - SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',$1,1,2, + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,path_bytes,payload_type) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',$1,2,1,2, CASE WHEN get_byte(packet_hash,3)=0 THEN '\xaabb'::bytea ELSE packet_hash END,4 FROM packets;`, at) if err != nil { t.Fatal(err) @@ -163,52 +150,18 @@ func TestRouteEvidencePostgres(t *testing.T) { } func TestRouteEvidenceIndexPostgres(t *testing.T) { - dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") - if dsn == "" { - t.Skip("set BEACON_TEST_POSTGRES_DSN") - } - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - conn, err := pgx.Connect(ctx, dsn) - if err != nil { - t.Fatal(err) - } - defer conn.Close(context.Background()) - schema := fmt.Sprintf("route_evidence_%d", time.Now().UnixNano()) - ident := pgx.Identifier{schema}.Sanitize() - if _, err = conn.Exec(ctx, "CREATE SCHEMA "+ident+"; SET search_path TO "+ident); err != nil { - t.Fatal(err) - } - defer func() { _, _ = conn.Exec(context.Background(), "DROP SCHEMA "+ident+" CASCADE") }() - if _, err = conn.Exec(ctx, `CREATE TABLE packet_observations (id bigint,iata char(3),hash_size smallint,path_bytes bytea,heard_at timestamptz,payload_type smallint,hop_count smallint)`); err != nil { - t.Fatal(err) - } - migration, err := migrationFiles.ReadFile("migrations/041_route_evidence_index.sql") - if err != nil { - t.Fatal(err) - } - for i := 0; i < 2; i++ { - if err = applyMigration(ctx, conn, string(migration)); err != nil { - t.Fatalf("migration/retry: %v", err) - } - } - if _, err = conn.Exec(ctx, `UPDATE pg_index SET indisvalid=false WHERE indexrelid='idx_observations_route_evidence'::regclass`); err != nil { - t.Fatal(err) - } - if err = applyMigration(ctx, conn, string(migration)); err != nil { - t.Fatal(err) - } - var valid bool - if err = conn.QueryRow(ctx, `SELECT indisvalid FROM pg_index WHERE indexrelid='idx_observations_route_evidence'::regclass`).Scan(&valid); err != nil || !valid { - t.Fatalf("invalid index after recovery: %v", err) - } - if _, err = conn.Exec(ctx, `INSERT INTO packet_observations(id,iata,hash_size,path_bytes,heard_at,payload_type,hop_count) -SELECT n,'YOW',1,'\xaabb',NOW(),4,n FROM generate_series(0,2) n; + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT int4send(n),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(0,2) n; +INSERT INTO packet_observations(packet_hash,observer_id,iata,path_length_byte,hash_size,path_bytes,heard_at,payload_type,hop_count) +SELECT int4send(n),'00000000-0000-0000-0000-000000000001','YOW',n,1,'\xaabb',NOW(),4,n FROM generate_series(0,2) n; ANALYZE packet_observations`); err != nil { t.Fatal(err) } var indexed float32 - if err = conn.QueryRow(ctx, `SELECT reltuples FROM pg_class WHERE oid='idx_observations_route_evidence'::regclass`).Scan(&indexed); err != nil || indexed != 1 { + if err := tx.QueryRow(ctx, `SELECT reltuples FROM pg_class WHERE oid='idx_observations_route_evidence'::regclass`).Scan(&indexed); err != nil || indexed != 1 { t.Fatalf("0/1-hop rows entered the route index: %v %v", indexed, err) } } diff --git a/db/routes_integration_test.go b/db/routes_integration_test.go index 5156dc24..f37d2235 100644 --- a/db/routes_integration_test.go +++ b/db/routes_integration_test.go @@ -20,7 +20,7 @@ import ( //go:embed queries/queries.sql var routeTestQueries string -// All synthetic rows and indexes are temporary and rolled back. The sparse +// All synthetic rows are rolled back. The sparse // region sits behind 100,000 newer routes to expose global timestamp scans. func TestListKnownRoutesPostgres(t *testing.T) { dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") @@ -39,8 +39,9 @@ func TestListKnownRoutesPostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) _, err = tx.Exec(ctx, ` -CREATE TEMP TABLE known_routes (LIKE public.known_routes INCLUDING ALL) ON COMMIT DROP; +INSERT INTO iata_codes (iata) VALUES ('GPT'); INSERT INTO known_routes (id,path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) SELECT i, decode(md5(i::text),'hex'), ARRAY[md5(i::text)::uuid], ARRAY['\x01'::bytea], CASE WHEN i > 100000 THEN 'GPT' ELSE 'YYZ' END, i % 4 + 2, @@ -51,15 +52,6 @@ ANALYZE known_routes;`) if err != nil { t.Fatal(err) } - migration, err := migrationFiles.ReadFile("migrations/027_known_routes_iata_last_seen.sql") - if err != nil { - t.Fatal(err) - } - // A temporary table inside this rollback-only fixture cannot be indexed - // concurrently. Use the migration's exact definition with that option off. - if _, err := tx.Exec(ctx, strings.Replace(string(migration), "CREATE INDEX CONCURRENTLY", "CREATE INDEX", 1)); err != nil { - t.Fatal(err) - } queries := strings.ReplaceAll(routeTestQueries, "\r\n", "\n") querySQL := strings.Split(strings.Split(queries, "-- name: ListKnownRoutes :many\n")[1], "\n-- name:")[0] if _, err := tx.Exec(ctx, "PREPARE route_page AS "+querySQL); err != nil { diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go index 7eef6bbf..b41af9f5 100644 --- a/db/signal_integration_test.go +++ b/db/signal_integration_test.go @@ -37,11 +37,11 @@ func TestSignalPostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) - isolateStatsSchema(t, ctx, tx) + applyBaseline(t, ctx, tx) _, err = tx.Exec(ctx, ` SET LOCAL TIME ZONE 'America/Vancouver'; -CREATE TABLE packet_observations (heard_at timestamptz NOT NULL, iata char(3) NOT NULL, snr real, rssi smallint); -INSERT INTO packet_observations +CREATE TEMP TABLE fixture (heard_at timestamptz NOT NULL, iata char(3) NOT NULL, snr real, rssi smallint); +INSERT INTO fixture SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FROM (VALUES ('00:30:00','YVR','0',-100), ('00:45:00','YVR','0',0), ('00:59:00','YVR',NULL,NULL), ('01:00:00','YVR','-30',-140), ('01:10:00','YVR','-30.25',-141), ('01:20:00','YVR','30',0), @@ -50,7 +50,13 @@ SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FRO ('02:20:00','YYJ','10',-50), ('03:00:00','YYZ','-5',-110), ('04:00:00','YVR','20',-40), ('04:30:00','YVR','20',-40), ('03:10:00','YVR','0',NULL), ('03:20:00','YVR',NULL,0) -) v(at,iata,snr,rssi);`) +) v(at,iata,snr,rssi); +ALTER TABLE fixture ADD COLUMN n serial; +INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT int4send(n),4,0,1,'\x00','\x00',heard_at,heard_at FROM fixture; +INSERT INTO packet_observations (packet_hash,observer_id,heard_at,iata,snr,rssi,path_length_byte,hash_size,hop_count) +SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi,0,1,0 FROM fixture;`) if err != nil { t.Fatal(err) } @@ -58,8 +64,9 @@ SELECT ('2026-01-01 '||at||'+00')::timestamptz,iata,snr::real,rssi::smallint FRO if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { t.Fatal(err) } - applyStatsMigration(t, ctx, tx, "035_mv_signal_stats.sql") - applyStatsMigration(t, ctx, tx, "035_mv_signal_stats.sql") // interrupted journal retry + if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW mv_signal_stats_hourly"); err != nil { + t.Fatal(err) + } store := &Store{q: sqlc.New(tx)} until := since.Add(4 * time.Hour) for _, tc := range []struct { diff --git a/db/stats_migration_integration_test.go b/db/stats_migration_integration_test.go deleted file mode 100644 index cfe3f94f..00000000 --- a/db/stats_migration_integration_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later - -package db - -import ( - "context" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5" -) - -// Views cannot reference temporary tables. Keep fixtures and real migration DDL -// in a unique schema inside the caller's rolled-back transaction instead. -func isolateStatsSchema(t *testing.T, ctx context.Context, tx pgx.Tx) { - t.Helper() - schema := pgx.Identifier{"stats_test_" + uuid.NewString()}.Sanitize() - if _, err := tx.Exec(ctx, "CREATE SCHEMA "+schema+"; SET LOCAL search_path TO "+schema); err != nil { - t.Fatal(err) - } -} - -func applyStatsMigration(t *testing.T, ctx context.Context, tx pgx.Tx, name string) { - t.Helper() - migration, err := migrationFiles.ReadFile("migrations/" + name) - if err != nil { - t.Fatal(err) - } - if _, err := tx.Exec(ctx, string(migration)); err != nil { - t.Fatal(err) - } -} diff --git a/internal/config/border.go b/internal/config/border.go index a297ee9c..285486d5 100644 --- a/internal/config/border.go +++ b/internal/config/border.go @@ -15,7 +15,7 @@ import ( // map: well-formed Feature, geometry Polygon or MultiPolygon, every coordinate within // lon [-180,180] / lat [-90,90], every ring closed (first point == last point, 4+ points -- // see orb.Ring.Closed). On success, returns the Feature re-marshaled to JSON with its bbox -// computed and set, so reads never need to recompute it (db/migrations/011_add_iata_border.sql). +// computed and set, so reads never need to recompute it (iata_codes.border). // // Coordinate order: GeoJSON packs coordinates [lon, lat], not [lat, lon]. A border authored // with the axes swapped is still well-formed JSON and its numbers can still individually fall From 2f49ffb3a32fa2f61b7309a6b96e02089f6920e4 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 22:02:10 -0400 Subject: [PATCH 111/141] feat(api): filter scope lists by region and catalogue membership Region-filtered /scopes and /stats/scopes now list manual scopes configured for the region and imported scopes whose MeshMapper catalogue includes the IATA, instead of inferring membership from observed traffic. Filtered /scopes returns string[] like the unfiltered call. Manual scopes now require a region in config. --- README.md | 6 ++- cmd/beacon/main.go | 2 + config.yaml.example | 6 +++ db/paths_integration_test.go | 2 +- db/queries/queries.sql | 15 ------ db/scopes.go | 18 ------- db/scopes_test.go | 23 -------- db/signal_integration_test.go | 2 +- db/sqlc/mock/querier.go | 15 ------ db/sqlc/querier.go | 1 - db/sqlc/queries.sql.go | 48 ----------------- docs/docs.go | 15 ++++-- docs/swagger.json | 15 ++++-- docs/swagger.yaml | 23 +++++--- internal/api/handlers/limits_test.go | 2 +- .../api/handlers/observer_comparison_test.go | 6 +-- internal/api/handlers/paths_test.go | 6 +-- internal/api/handlers/scope_stats_test.go | 44 ++++++++++++++-- internal/api/handlers/scopes.go | 38 ++++++++------ internal/api/handlers/scopes_test.go | 41 ++++++++++++++- internal/api/handlers/signal_test.go | 6 +-- internal/api/handlers/stats.go | 16 ++++-- internal/api/handlers/stats_test.go | 2 +- internal/api/handlers/stub_reader_test.go | 8 --- internal/api/reader.go | 4 -- internal/api/router/options.go | 2 + internal/api/router/router.go | 4 +- internal/api/scopes.go | 8 --- internal/cache/cache_test.go | 6 +-- internal/cache/reader.go | 13 ----- internal/config/config.go | 6 ++- internal/config/scopes.go | 52 +++++++++++++++++++ internal/config/scopes_test.go | 43 +++++++++++++++ internal/meshmapper/scopes.go | 3 ++ internal/meshmapper/scopes_test.go | 8 ++- internal/scopestore/scopestore.go | 30 +++++++++++ internal/scopestore/scopestore_test.go | 19 +++++++ 37 files changed, 346 insertions(+), 212 deletions(-) create mode 100644 internal/config/scopes.go create mode 100644 internal/config/scopes_test.go diff --git a/README.md b/README.md index dfcec34f..e198987d 100644 --- a/README.md +++ b/README.md @@ -231,9 +231,13 @@ channel_keys: # Regional transport scopes for matching TRANSPORT_FLOOD packets. # Plain names have # prepended automatically (e.g. "bc" → "#bc"). +# region (required) is a configured region slug; region-filtered scope lists +# and scope stats show the scope under that region's IATAs. Matching stays global. scopes: - name: bc + region: western-canada - name: "#west" + region: western-canada # Observer telemetry storage settings. telemetry: @@ -500,7 +504,7 @@ Not yet implemented — see the Authentication section above. | `GET` | `/routes` | List known routes (all hops high confidence) | | `GET` | `/routes/search` | Search routes by source and destination hash | | `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | -| `GET` | `/scopes` | List transport scopes | +| `GET` | `/scopes` | List transport scope names; IATA/region filters use configured regions and MeshMapper catalogues | | `GET` | `/scopes/{name}` | Get scope detail | | `GET` | `/stats/observations` | Hourly observation time series (last 7 days by default) | | `GET` | `/stats/overview` | Network overview stats | diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 5017dffb..6a28fcd4 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -196,6 +196,7 @@ func main() { os.Exit(1) } scopes.Load(scopeEntries) + scopes.SetManualMembers(cfg.ManualScopeMembers()) slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") var scopeImporter *meshmapper.Importer if cfg.MeshMapper.Scopes.Enabled { @@ -371,6 +372,7 @@ func main() { MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, WSAllowedOrigins: cfg.WebSocket.AllowedOrigins, CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, + Scopes: scopes, AdminRoutes: map[string]http.Handler{ "/accounts": handlers.AccountsRouter(store), "/backup": handlers.BackupRouter(backupOpts, ctx), diff --git a/config.yaml.example b/config.yaml.example index 0c6eb6e2..45088531 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -105,9 +105,13 @@ channel_keys: # Regional transport scopes for matching TRANSPORT_FLOOD packets. # Plain names have # prepended automatically (e.g. "bc" → "#bc"). +# region (required) is a configured region slug; region-filtered scope lists +# and scope stats show the scope under that region's IATAs. Matching stays global. scopes: - name: bc + region: western-canada - name: "#west" + region: western-canada telemetry: retention: 672h # 4 weeks @@ -224,6 +228,8 @@ cache: # Refresh success/failure/freshness is logged under component=meshmapper.scopes. # Removing/turning off a source deactivates its imported matching keys on restart; # historical scope identities and recorded evidence are retained. +# Region-filtered scope lists and scope stats show imported names only under the +# IATAs whose catalogue lists them; regions without a source show manual scopes only. #meshmapper: # scopes: # enabled: false diff --git a/db/paths_integration_test.go b/db/paths_integration_test.go index 76a96628..a49f8ccd 100644 --- a/db/paths_integration_test.go +++ b/db/paths_integration_test.go @@ -115,7 +115,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_ } } w := httptest.NewRecorder() - handlers.StatsRouter(store).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) + handlers.StatsRouter(store, nil).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) var response api.PathStats if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 14 || response.Hashed != 5 { t.Fatalf("HTTP %d: %s (%v)", w.Code, w.Body.String(), err) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index ca7584c8..afcd8c5d 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -64,21 +64,6 @@ SELECT id FROM transport_scopes WHERE name = $1; -- name: GetScopeNames :many SELECT name FROM transport_scopes ORDER BY name; --- name: GetScopesByIATAs :many -SELECT - ts.name, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count -FROM transport_scopes ts -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])) -GROUP BY ts.name -ORDER BY ts.name; - -- name: GetScopeByName :one SELECT ts.name, diff --git a/db/scopes.go b/db/scopes.go index af5cf9a2..a9b1cb20 100644 --- a/db/scopes.go +++ b/db/scopes.go @@ -49,24 +49,6 @@ func (s *Store) GetScopeNames(ctx context.Context) ([]string, error) { return s.q.GetScopeNames(ctx) } -// GetScopesByIATAs returns scope summaries filtered by the given IATA codes. -func (s *Store) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) { - rows, err := s.q.GetScopesByIATAs(ctx, iatas) - if err != nil { - return nil, err - } - items := make([]api.ScopeSummary, 0, len(rows)) - for _, r := range rows { - items = append(items, api.ScopeSummary{ - Name: r.Name, - ObserverCount: r.ObserverCount, - NodeCount: r.NodeCount, - IATACount: r.IataCount, - }) - } - return items, nil -} - // GetScopeByName returns full detail for a single scope by its normalized name. func (s *Store) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { row, err := s.q.GetScopeByName(ctx, name) diff --git a/db/scopes_test.go b/db/scopes_test.go index 890e93ad..c581deea 100644 --- a/db/scopes_test.go +++ b/db/scopes_test.go @@ -43,29 +43,6 @@ func TestGetTransportScopes(t *testing.T) { } } -func TestGetScopesByIATAs(t *testing.T) { - ctrl := gomock.NewController(t) - mock := mockdb.NewMockQuerier(ctrl) - - mock.EXPECT(). - GetScopesByIATAs(gomock.Any(), []string{"YVR", "YYJ"}). - Return([]sqlc.GetScopesByIATAsRow{ - {Name: "default", ObserverCount: 3, NodeCount: 10, IataCount: 2}, - }, nil) - - store := &Store{q: mock} - items, err := store.GetScopesByIATAs(context.Background(), []string{"YVR", "YYJ"}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(items) != 1 { - t.Fatalf("expected 1 item, got %d", len(items)) - } - if items[0].IATACount != 2 { - t.Errorf("expected IATACount 2, got %d", items[0].IATACount) - } -} - func TestGetScopeByName(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go index b41af9f5..73bc8252 100644 --- a/db/signal_integration_test.go +++ b/db/signal_integration_test.go @@ -138,7 +138,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi } w := httptest.NewRecorder() request := httptest.NewRequest("GET", fmt.Sprintf("/signal?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx) - handlers.StatsRouter(store).ServeHTTP(w, request) + handlers.StatsRouter(store, nil).ServeHTTP(w, request) var response api.SignalStats if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 13 || response.SNR.Samples != 5 { t.Fatalf("PostgreSQL HTTP response: status=%d body=%s error=%v", w.Code, w.Body.String(), err) diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index f9914cf0..a0aca7b5 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -760,21 +760,6 @@ func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, iatas any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, iatas) } -// GetScopesByIATAs mocks base method. -func (m *MockQuerier) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]db.GetScopesByIATAsRow, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetScopesByIATAs", ctx, dollar_1) - ret0, _ := ret[0].([]db.GetScopesByIATAsRow) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetScopesByIATAs indicates an expected call of GetScopesByIATAs. -func (mr *MockQuerierMockRecorder) GetScopesByIATAs(ctx, dollar_1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopesByIATAs", reflect.TypeOf((*MockQuerier)(nil).GetScopesByIATAs), ctx, dollar_1) -} - // GetSignalStats mocks base method. func (m *MockQuerier) GetSignalStats(ctx context.Context, arg db.GetSignalStatsParams) ([]db.GetSignalStatsRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index ef53cb36..7a64a312 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -97,7 +97,6 @@ type Querier interface { // a cross-join. Empty IATAs keep the original global counts, including associations // whose observations have expired; the filtered aggregates are empty in that case. GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) - GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) // Read compact hourly snapshots, never observations on an HTTP request. // Weight averages by sample counts instead of averaging regional/hourly means. GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 9ea06a5b..e8b52ff7 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1596,54 +1596,6 @@ func (q *Queries) GetScopeStats(ctx context.Context, iatas []string) ([]GetScope return items, nil } -const getScopesByIATAs = `-- name: GetScopesByIATAs :many -SELECT - ts.name, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count -FROM transport_scopes ts -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])) -GROUP BY ts.name -ORDER BY ts.name -` - -type GetScopesByIATAsRow struct { - Name string `json:"name"` - ObserverCount int64 `json:"observer_count"` - NodeCount int64 `json:"node_count"` - IataCount int64 `json:"iata_count"` -} - -func (q *Queries) GetScopesByIATAs(ctx context.Context, dollar_1 []string) ([]GetScopesByIATAsRow, error) { - rows, err := q.db.Query(ctx, getScopesByIATAs, dollar_1) - if err != nil { - return nil, err - } - defer rows.Close() - items := []GetScopesByIATAsRow{} - for rows.Next() { - var i GetScopesByIATAsRow - if err := rows.Scan( - &i.Name, - &i.ObserverCount, - &i.NodeCount, - &i.IataCount, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const getStatsClockDrift = `-- name: GetStatsClockDrift :many WITH page AS ( SELECT diff --git a/docs/docs.go b/docs/docs.go index df1ccc9e..31dab86c 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2219,7 +2219,7 @@ const docTemplate = `{ }, "/scopes": { "get": { - "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", + "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", "produces": [ "application/json" ], @@ -2251,7 +2251,16 @@ const docTemplate = `{ "200": { "description": "OK", "schema": { - "type": "object" + "type": "array", + "items": { + "type": "string" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -2780,7 +2789,7 @@ const docTemplate = `{ }, "/stats/scopes": { "get": { - "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.", + "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], diff --git a/docs/swagger.json b/docs/swagger.json index 183611af..7b2e8f81 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2217,7 +2217,7 @@ }, "/scopes": { "get": { - "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", + "description": "Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes.", "produces": [ "application/json" ], @@ -2249,7 +2249,16 @@ "200": { "description": "OK", "schema": { - "type": "object" + "type": "array", + "items": { + "type": "string" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" } }, "500": { @@ -2778,7 +2787,7 @@ }, "/stats/scopes": { "get": { - "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array.", + "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], diff --git a/docs/swagger.yaml b/docs/swagger.yaml index ca79d350..e45b6762 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -3013,10 +3013,11 @@ paths: - Routes /scopes: get: - description: The unfiltered list includes stored imported names after an importer - is disabled or a source is removed. Those historical identities are retained - indefinitely; listing them does not establish observed traffic or current - source membership. + description: Without filters, lists every stored scope name, including imported + names retained after an importer is disabled or a source is removed. With + IATA or region filters, lists only manual scopes configured for a matching + region and imported scopes whose current MeshMapper catalogue includes a matching + IATA; observed traffic does not add scopes. parameters: - description: Filter by IATA code(s), comma-separated in: query @@ -3036,7 +3037,13 @@ paths: "200": description: OK schema: - type: object + items: + type: string + type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' "500": description: Internal Server Error schema: @@ -3400,8 +3407,10 @@ paths: get: description: Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships - for nodes. Without filters, returns global totals. Scopes with zero matching - counts remain listed; an empty region returns an empty array. + for nodes. Without filters, returns global totals for every stored scope. + With filters, lists only manual scopes configured for a matching region and + imported scopes whose current MeshMapper catalogue includes a matching IATA; + those remain listed with zero counts. An empty region returns an empty array. parameters: - description: Comma-separated IATA codes in: query diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go index 991a2076..75fc8cbe 100644 --- a/internal/api/handlers/limits_test.go +++ b/internal/api/handlers/limits_test.go @@ -113,7 +113,7 @@ func TestListLimits(t *testing.T) { router.Mount("/messages", MessagesRouter(reader)) router.Mount("/routes", RoutesRouter(reader)) router.Mount("/traces", TracesRouter(reader)) - router.Mount("/stats", StatsRouter(reader)) + router.Mount("/stats", StatsRouter(reader, nil)) for _, endpoint := range []struct { path string defaultLimit int32 diff --git a/internal/api/handlers/observer_comparison_test.go b/internal/api/handlers/observer_comparison_test.go index 3a46da7e..0b11e6d1 100644 --- a/internal/api/handlers/observer_comparison_test.go +++ b/internal/api/handlers/observer_comparison_test.go @@ -28,7 +28,7 @@ func TestObserverComparisonRejectsInvalidQuery(t *testing.T) { } { t.Run(query, func(t *testing.T) { w := httptest.NewRecorder() - StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) + StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) if w.Code != 400 { t.Fatalf("status %d; wanted validation error", w.Code) } @@ -42,7 +42,7 @@ func TestObserverComparisonHTTP(t *testing.T) { query := "observerA=" + a.String() + "&observerB=" + b.String() + "&since=0&until=1000" for _, suffix := range []string{"&since=2", "&observerA=" + b.String(), "&until=253402300800000", "&observerB=00000000-0000-0000-0000-000000000000"} { w := httptest.NewRecorder() - StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) + StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) if w.Code != 400 { t.Fatalf("invalid query returned %d", w.Code) } @@ -80,7 +80,7 @@ func TestObserverComparisonHTTP(t *testing.T) { }, } w := httptest.NewRecorder() - StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) + StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) if w.Code != tc.status { t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) } diff --git a/internal/api/handlers/paths_test.go b/internal/api/handlers/paths_test.go index 64473463..190fe69b 100644 --- a/internal/api/handlers/paths_test.go +++ b/internal/api/handlers/paths_test.go @@ -28,7 +28,7 @@ func (s pathReader) GetPathStats(ctx context.Context, since, until time.Time, ia func TestPathsRejectInvalidWindow(t *testing.T) { for _, query := range []string{"", "since=0", "since=x&until=1", "since=-1&until=1", "since=1&until=1", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { w := httptest.NewRecorder() - StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) + StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) if w.Code != 400 { t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) } @@ -66,7 +66,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) { return &api.PathStats{Until: until.UnixMilli(), Receptions: 4, Hashed: 1, Empty: 1, Trace: 1, Unclassified: 1}, tc.err }} w := httptest.NewRecorder() - StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) + StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { t.Fatalf("%d %s", w.Code, w.Body.String()) } @@ -86,7 +86,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) { return nil, ctx.Err() }} w := httptest.NewRecorder() - StatsRouter(r).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) + StatsRouter(r, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) if !called || w.Body.Len() != 0 { t.Fatal("cancellation did not reach reader") } diff --git a/internal/api/handlers/scope_stats_test.go b/internal/api/handlers/scope_stats_test.go index d63ba21b..111681dd 100644 --- a/internal/api/handlers/scope_stats_test.go +++ b/internal/api/handlers/scope_stats_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) func TestGetStatsScopes_RegionErrors(t *testing.T) { @@ -27,7 +28,7 @@ func TestGetStatsScopes_RegionErrors(t *testing.T) { }, } w := httptest.NewRecorder() - getStatsScopes(reader)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+query, nil)) + getStatsScopes(reader, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+query, nil)) if w.Code != http.StatusBadRequest { t.Fatalf("got status %d, want 400", w.Code) } @@ -79,7 +80,7 @@ func TestGetStatsScopes_Filters(t *testing.T) { }, } w := httptest.NewRecorder() - getStatsScopes(reader)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil)) + getStatsScopes(reader, testScopes(map[string][]string{"YVR": {"#test"}, "YYJ": {"#test"}}))(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil)) if w.Code != http.StatusOK { t.Fatalf("status = %d: %s", w.Code, w.Body.String()) } @@ -105,8 +106,45 @@ func TestGetStatsScopes_ReaderError(t *testing.T) { w := httptest.NewRecorder() getStatsScopes(stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) { return nil, errors.New("database unavailable") - }})(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil)) + }}, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil)) if w.Code != http.StatusInternalServerError { t.Fatalf("status = %d, want 500", w.Code) } } + +func testScopes(byIATA map[string][]string) *scopestore.ScopeStore { + s := scopestore.New() + s.SetManualMembers(byIATA) + return s +} + +func TestGetStatsScopes_FiltersToRegionMembers(t *testing.T) { + reader := stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) { + return []api.ScopeStats{{Name: "#elsewhere", PacketCount: 9}, {Name: "#on"}, {Name: "#yow", PacketCount: 4}}, nil + }} + members := testScopes(map[string][]string{"YOW": {"#yow", "#on"}}) + for _, tc := range []struct { + query string + members ScopeMembership + want []string + }{ + {"", members, []string{"#elsewhere", "#on", "#yow"}}, + {"iata=YOW", members, []string{"#on", "#yow"}}, + {"iata=YUL", members, []string{}}, + {"iata=YOW", nil, []string{}}, + } { + w := httptest.NewRecorder() + getStatsScopes(reader, tc.members)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?"+tc.query, nil)) + var rows []api.ScopeStats + if err := json.Unmarshal(w.Body.Bytes(), &rows); err != nil || rows == nil { + t.Fatalf("%q: body %s, err %v", tc.query, w.Body.String(), err) + } + got := []string{} + for _, row := range rows { + got = append(got, row.Name) + } + if !slices.Equal(got, tc.want) { + t.Fatalf("%q: got %v, want %v", tc.query, got, tc.want) + } + } +} diff --git a/internal/api/handlers/scopes.go b/internal/api/handlers/scopes.go index c86de6fa..41cca54d 100644 --- a/internal/api/handlers/scopes.go +++ b/internal/api/handlers/scopes.go @@ -10,13 +10,25 @@ import ( "github.com/go-chi/chi/v5" ) +// ScopeMembership lists scopes by configured region and imported catalogue membership. +type ScopeMembership interface { + NamesForIATAs(iatas []string) []string // sorted +} + +func scopeNamesFor(scopes ScopeMembership, iatas []string) []string { + if scopes == nil { + return []string{} + } + return scopes.NamesForIATAs(iatas) +} + // ScopesRouter mounts all /scopes routes onto a subrouter. // // GET /scopes → listScopes // GET /scopes/{name} → getScope -func ScopesRouter(reader api.Reader) http.Handler { +func ScopesRouter(reader api.Reader, scopes ScopeMembership) http.Handler { r := chi.NewRouter() - r.Get("/", listScopes(reader)) + r.Get("/", listScopes(reader, scopes)) r.Get("/{name}", getScope(reader)) return r } @@ -24,27 +36,28 @@ func ScopesRouter(reader api.Reader) http.Handler { // listScopes godoc // // @Summary List transport scopes -// @Description The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership. +// @Description Without filters, lists every stored scope name, including imported names retained after an importer is disabled or a source is removed. With IATA or region filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; observed traffic does not add scopes. // @Tags Scopes // @Produce json // @Param iatas query string false "Filter by IATA code(s), comma-separated" // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" -// @Success 200 {object} object +// @Success 200 {array} string +// @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /scopes [get] -func listScopes(reader api.Reader) http.HandlerFunc { +func listScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { iatas := parseIATAs(r) - if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { - regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) + regionID, region := r.URL.Query().Get("regionId"), r.URL.Query().Get("region") + if regionID != "" || region != "" { + regionIATAs, err := resolveRegionIATAs(r.Context(), regionID, region, reader) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return } iatas = append(iatas, regionIATAs...) - } - if len(iatas) == 0 { + } else if len(iatas) == 0 { names, err := reader.GetScopeNames(r.Context()) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") @@ -53,12 +66,7 @@ func listScopes(reader api.Reader) http.HandlerFunc { respond(w, http.StatusOK, names) return } - scopes, err := reader.GetScopesByIATAs(r.Context(), iatas) - if err != nil { - respondError(w, http.StatusInternalServerError, "internal server error") - return - } - respond(w, http.StatusOK, scopes) + respond(w, http.StatusOK, scopeNamesFor(scopes, iatas)) } } diff --git a/internal/api/handlers/scopes_test.go b/internal/api/handlers/scopes_test.go index 29abb392..f83262ee 100644 --- a/internal/api/handlers/scopes_test.go +++ b/internal/api/handlers/scopes_test.go @@ -5,8 +5,10 @@ package handlers import ( "context" + "encoding/json" "net/http" "net/http/httptest" + "slices" "testing" "github.com/MeshCore-Beacon/beacon-server/internal/api" @@ -19,7 +21,7 @@ func TestListScopes_NoIATAs_OK(t *testing.T) { getScopeNames: func(_ context.Context) ([]string, error) { return []string{"#bc", "#west"}, nil }, - })) + }, nil)) req := httptest.NewRequest(http.MethodGet, "/scopes", nil) w := httptest.NewRecorder() r.ServeHTTP(w, req) @@ -42,3 +44,40 @@ func TestGetScope_OK(t *testing.T) { t.Errorf("expected 200, got %d", w.Code) } } + +func TestListScopes_FiltersByMembership(t *testing.T) { + reader := stubReader{ + getScopeNames: func(context.Context) ([]string, error) { + t.Fatal("filtered list must not fall back to every stored name") + return nil, nil + }, + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + return &api.Region{IATAs: []string{"YOW", "YGK"}}, nil + }, + } + members := testScopes(map[string][]string{"YOW": {"#on", "#yow"}, "YGK": {"#on"}, "YUL": {"#qc"}}) + for _, tc := range []struct { + query string + members ScopeMembership + want []string + }{ + {"iatas=yow", members, []string{"#on", "#yow"}}, + {"iatas=YUL,YYZ", members, []string{"#qc"}}, + {"region=ottawa", members, []string{"#on", "#yow"}}, + {"region=empty", members, []string{}}, + {"iatas=YOW", nil, []string{}}, + } { + w := httptest.NewRecorder() + listScopes(reader, tc.members)(w, httptest.NewRequest(http.MethodGet, "/scopes?"+tc.query, nil)) + var got []string + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || w.Code != http.StatusOK || got == nil { + t.Fatalf("%q: status %d body %s", tc.query, w.Code, w.Body.String()) + } + if !slices.Equal(got, tc.want) { + t.Fatalf("%q: got %v, want %v", tc.query, got, tc.want) + } + } +} diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go index 405db1ca..f9146fa8 100644 --- a/internal/api/handlers/signal_test.go +++ b/internal/api/handlers/signal_test.go @@ -28,7 +28,7 @@ func (s signalReader) GetSignalStats(ctx context.Context, since, until time.Time func TestSignalRejectsInvalidWindow(t *testing.T) { for _, query := range []string{"", "since=0", "until=1", "since=x&until=1", "since=-1&until=1", "since=0&until=0", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { w := httptest.NewRecorder() - StatsRouter(nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) + StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) if w.Code != 400 { t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) } @@ -70,7 +70,7 @@ func TestSignalFiltersAndErrors(t *testing.T) { return &api.SignalStats{Until: until.UnixMilli(), Receptions: 5, Hourly: []api.SignalHour{}}, tc.err }} w := httptest.NewRecorder() - StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) + StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { t.Fatalf("%d %s", w.Code, w.Body.String()) } @@ -93,7 +93,7 @@ func TestSignalCancellationReachesReader(t *testing.T) { return nil, ctx.Err() }} w := httptest.NewRecorder() - StatsRouter(reader).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) + StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) if !called || w.Body.Len() != 0 { t.Fatalf("cancellation: called=%v body=%s", called, w.Body.String()) } diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index 1309efda..ca064233 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -6,6 +6,7 @@ package handlers import ( "log/slog" "net/http" + "slices" "strconv" "time" @@ -26,7 +27,7 @@ import ( // GET /stats/scopes → GetStatsScopes // // All endpoints accept an optional iata= filter (case-insensitive). -func StatsRouter(reader api.Reader) http.Handler { +func StatsRouter(reader api.Reader, scopes ScopeMembership) http.Handler { r := chi.NewRouter() r.Get("/overview", getStatsOverview(reader)) r.Get("/observations", getStatsObservations(reader)) @@ -40,7 +41,7 @@ func StatsRouter(reader api.Reader) http.Handler { r.Get("/clock-drift", getStatsClockDrift(reader)) r.Get("/top-talkers", getStatsTopTalkers(reader)) r.Get("/radio-presets", getStatsRadioPresets(reader)) - r.Get("/scopes", getStatsScopes(reader)) + r.Get("/scopes", getStatsScopes(reader, scopes)) r.Get("/node-types", getStatsNodeTypes(reader)) return r } @@ -417,7 +418,7 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // getStatsScopes godoc // // @Summary Scope statistics -// @Description Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals. Scopes with zero matching counts remain listed; an empty region returns an empty array. +// @Description Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -428,7 +429,7 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError // @Router /stats/scopes [get] -func getStatsScopes(reader api.Reader) http.HandlerFunc { +func getStatsScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { iatas := parseIATAs(r) if regionID := r.URL.Query().Get("regionId"); regionID != "" || r.URL.Query().Get("region") != "" { @@ -448,6 +449,13 @@ func getStatsScopes(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusInternalServerError, "internal server error") return } + if len(iatas) > 0 { + members := scopeNamesFor(scopes, iatas) + stats = slices.DeleteFunc(slices.Clone(stats), func(s api.ScopeStats) bool { + _, found := slices.BinarySearch(members, s.Name) + return !found + }) + } respond(w, http.StatusOK, stats) } } diff --git a/internal/api/handlers/stats_test.go b/internal/api/handlers/stats_test.go index 8225fc8c..86d9d9b9 100644 --- a/internal/api/handlers/stats_test.go +++ b/internal/api/handlers/stats_test.go @@ -285,7 +285,7 @@ func TestGetStatsScopes_OK(t *testing.T) { getScopeStats: func(_ context.Context, _ []string) ([]api.ScopeStats, error) { return []api.ScopeStats{{Name: "#bc", PacketCount: 100}}, nil }, - })) + }, nil)) req := httptest.NewRequest(http.MethodGet, "/stats/scopes", nil) w := httptest.NewRecorder() r.ServeHTTP(w, req) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index f9f885a6..0c719aca 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -55,7 +55,6 @@ type stubReader struct { getScopeStats func(ctx context.Context, iatas []string) ([]api.ScopeStats, error) getStatsNodeTypes func(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) getScopeNames func(ctx context.Context) ([]string, error) - getScopesByIATAs func(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) getScopeByName func(ctx context.Context, name string) (*api.ScopeDetail, error) listTraceTags func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) getTraceByTag func(ctx context.Context, tag string) (*api.TraceDetail, error) @@ -341,13 +340,6 @@ func (s stubReader) GetScopeNames(ctx context.Context) ([]string, error) { return nil, nil } -func (s stubReader) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) { - if s.getScopesByIATAs != nil { - return s.getScopesByIATAs(ctx, iatas) - } - return nil, nil -} - func (s stubReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { if s.getScopeByName != nil { return s.getScopeByName(ctx, name) diff --git a/internal/api/reader.go b/internal/api/reader.go index d46ec2d9..21d65403 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -210,10 +210,6 @@ type Reader interface { // Use when no geographic filter is applied — returns names only for a lightweight response. GetScopeNames(ctx context.Context) ([]string, error) - // GetScopesByIATAs returns scope summaries filtered by the given IATA codes, - // including observer, node and IATA counts. Expands region/regionId to IATAs automatically. - GetScopesByIATAs(ctx context.Context, iatas []string) ([]ScopeSummary, error) - // GetScopeByName returns full detail for a single scope by its normalized name (e.g. "#bc"), // including packet count, observer count, node count, and the list of IATAs it is active in. // Returns nil if the scope is not found. diff --git a/internal/api/router/options.go b/internal/api/router/options.go index 757a6593..0d9076e1 100644 --- a/internal/api/router/options.go +++ b/internal/api/router/options.go @@ -6,6 +6,7 @@ package router import ( "net/http" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/config" ) @@ -19,6 +20,7 @@ type Options struct { Server config.ServerConfig Auth config.AuthConfig RateLimit config.ResolvedRateLimitConfig + Scopes handlers.ScopeMembership // region-filtered scope lists; nil lists none // AdminRoutes mounts operator-only subrouters at literal paths such as // "/accounts". Each feature owns its dependencies, methods and handlers. // The router applies authentication to every path and method in this group. diff --git a/internal/api/router/router.go b/internal/api/router/router.go index 58bbfac5..b8a5ceab 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -107,8 +107,8 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) r.Mount("/iatas", handlers.IATAsRouter(reader)) r.Mount("/regions", handlers.RegionsRouter(reader)) r.Mount("/routes", handlers.RoutesRouter(reader)) - r.Mount("/scopes", handlers.ScopesRouter(reader)) - r.Mount("/stats", handlers.StatsRouter(reader)) + r.Mount("/scopes", handlers.ScopesRouter(reader, opts.Scopes)) + r.Mount("/stats", handlers.StatsRouter(reader, opts.Scopes)) r.Mount("/traces", handlers.TracesRouter(reader)) }) diff --git a/internal/api/scopes.go b/internal/api/scopes.go index c4b23c1b..58a1fc28 100644 --- a/internal/api/scopes.go +++ b/internal/api/scopes.go @@ -3,14 +3,6 @@ package api -// ScopeSummary is the minimal scope representation used in filtered list responses. -type ScopeSummary struct { - Name string `json:"name"` - ObserverCount int64 `json:"observerCount"` - NodeCount int64 `json:"nodeCount"` - IATACount int64 `json:"iataCount"` -} - // ScopeDetail is the full scope representation including packet count and IATA list. type ScopeDetail struct { Name string `json:"name"` diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 147b9da2..41dde518 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -76,10 +76,6 @@ func (s *stubReader) GetScopeNames(_ context.Context) ([]string, error) { return func (s *stubReader) GetScopeStats(_ context.Context, _ []string) ([]api.ScopeStats, error) { return nil, nil } -func (s *stubReader) GetScopesByIATAs(_ context.Context, _ []string) ([]api.ScopeSummary, error) { - return nil, nil -} - func (s *stubReader) GetScopeByName(_ context.Context, _ string) (*api.ScopeDetail, error) { return nil, nil } @@ -386,7 +382,7 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { func TestInvalidateScopeNamesClearsDerivedKeys(t *testing.T) { c, mr := newTestClient(t) cr := &CachedReader{inner: &stubReader{}, c: c, ttl: CacheTTLs{}} - for _, k := range []string{keyScopeNames, keyScopesByIATAsPrefix + "YOW", keyScopesByIATAsPrefix + "YOW,YVR", keyScopeByNamePrefix + "#yow"} { + for _, k := range []string{keyScopeNames, keyScopeByNamePrefix + "#yow"} { mr.Set(k, "x") } mr.Set("beacon:iatas", "keep") diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 2273447a..cb49338f 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -25,7 +25,6 @@ const ( keyRegionSlugPrefix = "beacon:region:slug:" keyScopeNames = "beacon:scope:names" keyScopeStats = "beacon:scope:stats" - keyScopesByIATAsPrefix = "beacon:scopes:iatas:" keyScopeByNamePrefix = "beacon:scope:name:" keyStatsOverviewPrefix = "beacon:stats:overview:" keyStatsObservationsPrefix = "beacon:stats:observations:" @@ -83,7 +82,6 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { // InvalidateScopeNames makes newly committed catalogue names visible to every scope read. func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { cr.c.del(ctx, keyScopeNames) - cr.c.delPrefix(ctx, keyScopesByIATAsPrefix) cr.c.delPrefix(ctx, keyScopeByNamePrefix) } @@ -174,17 +172,6 @@ func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string) ([]ap }) } -// GetScopesByIATAs implements [api.Reader]. -func (cr *CachedReader) GetScopesByIATAs(ctx context.Context, iatas []string) ([]api.ScopeSummary, error) { - sorted := make([]string, len(iatas)) - copy(sorted, iatas) - sort.Strings(sorted) - key := keyScopesByIATAsPrefix + strings.Join(sorted, ",") - return getOrSet(ctx, cr.c, key, cr.ttl.Reference, func() ([]api.ScopeSummary, error) { - return cr.inner.GetScopesByIATAs(ctx, iatas) - }) -} - // GetScopeByName implements [api.Reader]. func (cr *CachedReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { return getOrSet(ctx, cr.c, keyScopeByNamePrefix+name, cr.ttl.Reference, func() (*api.ScopeDetail, error) { diff --git a/internal/config/config.go b/internal/config/config.go index 85df4692..95ebfcbb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -221,7 +221,8 @@ type CacheTTLsConfig struct { // Name can be provided with or without the # or $ prefix. // Beacon normalizes plain names by prepending #. type ScopeConfig struct { - Name string `yaml:"name"` // e.g. "bc", "#west", "$private" + Name string `yaml:"name"` // e.g. "bc", "#west", "$private" + Region string `yaml:"region"` // configured region slug; lists the scope under that region's IATAs } // TelemetryConfig controls observer telemetry storage behaviour. @@ -413,6 +414,9 @@ func Load(path string) (*Config, error) { if err := cfg.validateMeshMapper(); err != nil { return nil, err } + if err := cfg.validateScopes(); err != nil { + return nil, err + } if cfg.Nodes.MarkForeign && !cfg.MeshMapper.Zones.Enabled && !cfg.hasBorderFile() { return nil, fmt.Errorf("nodes.mark_foreign requires an iatas.*.borderFile or meshmapper.zones") } diff --git a/internal/config/scopes.go b/internal/config/scopes.go new file mode 100644 index 00000000..87c05a56 --- /dev/null +++ b/internal/config/scopes.go @@ -0,0 +1,52 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "fmt" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func (c *Config) validateScopes() error { + for i, s := range c.Scopes { + if strings.TrimSpace(s.Name) == "" { + return fmt.Errorf("scopes[%d].name is required", i) + } + if s.Region == "" { + return fmt.Errorf("scope %q needs a region", s.Name) + } + if c.region(s.Region) == nil { + return fmt.Errorf("scope %q region %q is not a configured region", s.Name, s.Region) + } + } + return nil +} + +func (c *Config) region(slug string) *RegionConfig { + for i := range c.Regions { + if c.Regions[i].Slug == slug { + return &c.Regions[i] + } + } + return nil +} + +// ManualScopeMembers maps each region IATA to the normalized names of its configured scopes. +func (c *Config) ManualScopeMembers() map[string][]string { + members := map[string][]string{} + for _, s := range c.Scopes { + region := c.region(s.Region) + if region == nil { + continue + } + name := scopestore.FromName(s.Name).Name + for _, iata := range region.IATAs { + iata = strings.ToUpper(iata) + members[iata] = append(members[iata], name) + } + } + return members +} diff --git a/internal/config/scopes_test.go b/internal/config/scopes_test.go new file mode 100644 index 00000000..6be61e4f --- /dev/null +++ b/internal/config/scopes_test.go @@ -0,0 +1,43 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestScopesConfig(t *testing.T) { + valid := "regions:\n - slug: ottawa\n iatas: [YOW, YGK]\n - slug: bc\n iatas: [YVR]\nscopes:\n - name: on\n region: ottawa\n - name: \"#west\"\n region: bc\n - name: \"$ottawa\"\n region: ottawa\n" + for _, tc := range []struct { + name, text string + bad bool + }{ + {"valid", valid, false}, + {"missing region", strings.Replace(valid, " region: bc\n", "", 1), true}, + {"unknown region", strings.Replace(valid, "region: bc", "region: prairies", 1), true}, + {"empty name", strings.Replace(valid, "name: on", "name: \"\"", 1), true}, + } { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(p) + if (err != nil) != tc.bad { + t.Fatalf("error=%v", err) + } + if tc.bad { + return + } + want := map[string][]string{"YOW": {"#on", "$ottawa"}, "YGK": {"#on", "$ottawa"}, "YVR": {"#west"}} + if got := cfg.ManualScopeMembers(); !reflect.DeepEqual(got, want) { + t.Fatalf("ManualScopeMembers = %v, want %v", got, want) + } + }) + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index ac389a14..1c267679 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -233,8 +233,10 @@ func (i *Importer) publish() { for _, e := range i.manual { byName[e.Name] = e } + members := make(map[string][]string, len(i.sources)) for _, s := range i.sources { for _, candidate := range s.entries { + members[s.iata] = append(members[s.iata], candidate.Name) entry, exists := byName[candidate.Name] if exists && entry.IATAs == nil { continue @@ -256,6 +258,7 @@ func (i *Importer) publish() { entries = append(entries, byName[name]) } i.scopes.Load(entries) + i.scopes.SetCatalogueMembers(members) } func (i *Importer) log(s source, action string) { diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go index b8e1b24d..95ac0b96 100644 --- a/internal/meshmapper/scopes_test.go +++ b/internal/meshmapper/scopes_test.go @@ -111,6 +111,9 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { if !reflect.DeepEqual(scopes.Entries()[1].IATAs, []string{"YOW"}) { t.Fatal("import not bound to region") } + if got := scopes.NamesForIATAs([]string{"YOW"}); !reflect.DeepEqual(got, []string{"#manual", "#yow"}) { + t.Fatal("catalogue membership must include manually overridden names", got) + } generated := imp.sources[0].generated status = 304 check() @@ -243,8 +246,11 @@ func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { if err != nil || len(scopes.Entries()) != 1 || !reflect.DeepEqual(scopes.Entries()[0].IATAs, []string{"YOW", "YVR"}) { t.Fatal(scopes.Entries(), err) } + if got := scopes.NamesForIATAs([]string{"YVR"}); !reflect.DeepEqual(got, []string{"#can"}) { + t.Fatal(got) + } _, err = New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "changed-source"}}, store, scopes, nil) - if err != nil || len(scopes.Entries()) != 0 { + if err != nil || len(scopes.Entries()) != 0 || len(scopes.NamesForIATAs([]string{"YOW", "YVR"})) != 0 { t.Fatal("removed/changed source remained active", err) } // Disabled mode never calls the store, even if URLs remain configured. diff --git a/internal/scopestore/scopestore.go b/internal/scopestore/scopestore.go index da5c80f4..55d6cd04 100644 --- a/internal/scopestore/scopestore.go +++ b/internal/scopestore/scopestore.go @@ -7,6 +7,7 @@ package scopestore import ( "crypto/sha256" + "slices" "strings" "sync" ) @@ -34,6 +35,8 @@ func FromName(name string) Entry { type ScopeStore struct { mu sync.RWMutex entries []Entry + // IATA -> scope names, for region-filtered listings only; matching uses entries. + manual, catalogue map[string][]string } // New creates an empty ScopeStore. @@ -54,3 +57,30 @@ func (s *ScopeStore) Entries() []Entry { defer s.mu.RUnlock() return s.entries } + +// SetManualMembers publishes configured region membership of manual scopes. +func (s *ScopeStore) SetManualMembers(byIATA map[string][]string) { + s.mu.Lock() + defer s.mu.Unlock() + s.manual = byIATA +} + +// SetCatalogueMembers publishes imported catalogue membership, including names a manual scope overrides. +func (s *ScopeStore) SetCatalogueMembers(byIATA map[string][]string) { + s.mu.Lock() + defer s.mu.Unlock() + s.catalogue = byIATA +} + +// NamesForIATAs returns the sorted names of scopes that belong to any of the given IATAs. +func (s *ScopeStore) NamesForIATAs(iatas []string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + names := []string{} + for _, iata := range iatas { + names = append(names, s.manual[iata]...) + names = append(names, s.catalogue[iata]...) + } + slices.Sort(names) + return slices.Compact(names) +} diff --git a/internal/scopestore/scopestore_test.go b/internal/scopestore/scopestore_test.go index 3e2f3592..08eef4c4 100644 --- a/internal/scopestore/scopestore_test.go +++ b/internal/scopestore/scopestore_test.go @@ -4,6 +4,7 @@ package scopestore import ( + "slices" "testing" ) @@ -54,3 +55,21 @@ func TestEntriesRetainsImmutableSnapshotWithoutAllocation(t *testing.T) { t.Fatal("replacement mutated the previous reader snapshot") } } + +func TestNamesForIATAs(t *testing.T) { + s := New() + if got := s.NamesForIATAs([]string{"YOW"}); got == nil || len(got) != 0 { + t.Fatalf("empty store = %#v, want empty non-nil slice", got) + } + s.SetManualMembers(map[string][]string{"YOW": {"#ottawa", "#on"}, "YVR": {"#bc"}}) + s.SetCatalogueMembers(map[string][]string{"YOW": {"#on", "#gatineau"}, "YUL": {"#qc"}}) + got := s.NamesForIATAs([]string{"YOW", "YUL", "YYZ"}) + want := []string{"#gatineau", "#on", "#ottawa", "#qc"} + if !slices.Equal(got, want) { + t.Fatalf("NamesForIATAs = %v, want %v", got, want) + } + s.SetCatalogueMembers(nil) + if got := s.NamesForIATAs([]string{"YUL"}); len(got) != 0 { + t.Fatalf("cleared catalogue still lists %v", got) + } +} From 11e0ed735313d065fd693df551458dc84e3bdec0 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 22:10:39 -0400 Subject: [PATCH 112/141] docs(config): surface MeshMapper integrations first --- README.md | 15 ++++++++++- config.yaml.example | 61 ++++++++++++++++++++++++--------------------- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index e198987d..535dbf95 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,18 @@ even when the same bearer-authenticated request works with curl. ### Config file (`config.yaml`) ```yaml +# MeshMapper integrations (optional, no API key). Import transport scopes and +# IATA border outlines from MeshMapper instead of maintaining scopes: and +# iatas.*.borderFile by hand. Both need the IATAs to be in a configured region. +# See config.yaml.example for limits and behaviour. +#meshmapper: +# scopes: +# enabled: true +# sources: +# YVR: https://yvr.meshmapper.net/get_scopes.php +# zones: +# enabled: true + # Optional IATA overrides — auto-created on first packet arrival, # only needed if you want to customise display name or coordinates. # borderFile points to a GeoJSON Feature (Polygon or MultiPolygon) for the @@ -202,7 +214,7 @@ iatas: name: Vancouver International lat: 49.1967 lng: -123.1815 - borderFile: borders/yvr.geojson # optional + borderFile: borders/yvr.geojson # optional; or use meshmapper.zones # Super-regions grouping multiple IATAs. regions: @@ -233,6 +245,7 @@ channel_keys: # Plain names have # prepended automatically (e.g. "bc" → "#bc"). # region (required) is a configured region slug; region-filtered scope lists # and scope stats show the scope under that region's IATAs. Matching stays global. +# Optional when meshmapper.scopes covers your regions. scopes: - name: bc region: western-canada diff --git a/config.yaml.example b/config.yaml.example index 45088531..acd61c5f 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -41,13 +41,44 @@ ratelimit: # A 429 returns error.code=rate_limited and Retry-After (1 or 60 seconds). burst: 300 +# MeshMapper integrations (optional, no API key). Import transport scopes and +# IATA border outlines from MeshMapper instead of maintaining scopes: and +# iatas.*.borderFile by hand. Both need the IATAs to be in a configured region below. +# +# Scope-name discovery. Manual scopes remain authoritative. +# Sources must belong to a configured region's IATAs. +# Group endpoints are not supported: their names cannot be attributed to each IATA. +# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping. +# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. +# Refresh success/failure/freshness is logged under component=meshmapper.scopes. +# Removing/turning off a source deactivates its imported matching keys on restart; +# historical scope identities and recorded evidence are retained. +# Region-filtered scope lists and scope stats show imported names only under the +# IATAs whose catalogue lists them; regions without a source show manual scopes only. +#meshmapper: +# scopes: +# enabled: false +# refresh_interval: 1h # 5m-24h; one source checked per 15s tick +# sources: +# YOW: https://yow.meshmapper.net/get_scopes.php +# +# Region boundaries (Zones API). Every IATA in a configured +# region is looked up in its country's zone list, then its get_geojson.php outline +# is imported. An imported boundary overrides that IATA's borderFile on the border +# map and for nodes.mark_foreign. A missing or failed boundary keeps the last good +# import, or the borderFile when there is none. Disabling, or removing an IATA from +# every region, drops its import on restart. Logged under component=meshmapper.zones. +# zones: +# enabled: false +# refresh_interval: 24h # 1h-168h; one request per 15s tick, ETag-conditional + iatas: YVR: name: Vancouver International lat: 49.1967 lng: -123.1815 # borderFile: borders/yvr.geojson # optional GeoJSON Feature (Polygon/MultiPolygon) - # # (meshmapper.zones overrides it when enabled) + # # (or use meshmapper.zones above) # # for the region border map; path is relative to # # this file's directory; validated at boot. YYJ: @@ -107,6 +138,7 @@ channel_keys: # Plain names have # prepended automatically (e.g. "bc" → "#bc"). # region (required) is a configured region slug; region-filtered scope lists # and scope stats show the scope under that region's IATAs. Matching stays global. +# Optional when meshmapper.scopes covers your regions. scopes: - name: bc region: western-canada @@ -220,33 +252,6 @@ cache: # allow_countries: [CA, US] # allow_continents: [NA] -# Optional public MeshMapper scope-name discovery. Manual scopes remain authoritative. -# No API key is needed. Sources must belong to a configured region's IATAs. -# Group endpoints are not supported: their names cannot be attributed to each IATA. -# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping. -# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. -# Refresh success/failure/freshness is logged under component=meshmapper.scopes. -# Removing/turning off a source deactivates its imported matching keys on restart; -# historical scope identities and recorded evidence are retained. -# Region-filtered scope lists and scope stats show imported names only under the -# IATAs whose catalogue lists them; regions without a source show manual scopes only. -#meshmapper: -# scopes: -# enabled: false -# refresh_interval: 1h # 5m-24h; one source checked per 15s tick -# sources: -# YOW: https://yow.meshmapper.net/get_scopes.php -# -# Optional MeshMapper region boundaries (Zones API). Every IATA in a configured -# region is looked up in its country's zone list, then its get_geojson.php outline -# is imported. An imported boundary overrides that IATA's borderFile on the border -# map and for nodes.mark_foreign. A missing or failed boundary keeps the last good -# import, or the borderFile when there is none. Disabling, or removing an IATA from -# every region, drops its import on restart. Logged under component=meshmapper.zones. -# zones: -# enabled: false -# refresh_interval: 24h # 1h-168h; one request per 15s tick, ETag-conditional - # Background task intervals. # Shorter intervals are useful during initial deployment to confirm data is # flowing. Back off to 1h or more once stable. From e2b64f2a3611ca724fb74d427ce416cf3cf0fc72 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 20:02:51 -0400 Subject: [PATCH 113/141] chore: version bump v2.0.0 --- cmd/beacon/main.go | 2 +- docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 6a28fcd4..36d9a58e 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -45,7 +45,7 @@ import ( var version = "dev" // @title MeshCore Beacon API -// @version 1.6.0 +// @version 2.0.0 // @description MeshCore network observation backend. Ingests LoRa packets from MQTT brokers, stores in PostgreSQL, and streams live events via WebSocket. // @description REST requests share a configurable per-client rate limit (default 300/minute and a 300-request one-second burst cap). Exceeded limits return HTTP 429 with error.code=rate_limited and a Retry-After header in seconds. CORS preflights and WebSocket upgrades do not consume this API budget. // @description WebSocket upgrade attempts at /ws have a configurable per-client limit (default 10/minute, including failed handshakes). Rate exhaustion returns HTTP 429 with Retry-After before upgrade. An accepted socket exceeding the concurrent cap closes with code 1013 before hello; established connections remain open. diff --git a/docs/docs.go b/docs/docs.go index 31dab86c..45381385 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -5512,7 +5512,7 @@ const docTemplate = `{ // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ - Version: "1.6.0", + Version: "2.0.0", Host: "localhost:8080", BasePath: "/api/v1", Schemes: []string{"http", "https"}, diff --git a/docs/swagger.json b/docs/swagger.json index 7b2e8f81..4cd76556 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -15,7 +15,7 @@ "license": { "name": "AGPL-3-or-later" }, - "version": "1.6.0" + "version": "2.0.0" }, "host": "localhost:8080", "basePath": "/api/v1", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index e45b6762..4ed5e382 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1521,7 +1521,7 @@ info: name: AGPL-3-or-later termsOfService: https://github.com/MeshCore-Beacon/beacon-server title: MeshCore Beacon API - version: 1.6.0 + version: 2.0.0 paths: /admin/accounts: get: From 9a4e17b270787f231d538d409c8535fc1aeb0ca6 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 22:23:37 -0400 Subject: [PATCH 114/141] feat(meshmapper): import zones for every known IATA --- README.md | 4 +-- cmd/beacon/main.go | 2 +- config.yaml.example | 13 ++++----- db/meshmapper.go | 12 +++++++++ internal/config/meshmapper.go | 16 +----------- internal/config/meshmapper_test.go | 6 +---- internal/meshmapper/zones.go | 42 +++++++++++++++++++++++------- internal/meshmapper/zones_test.go | 35 +++++++++++++++++++++++-- 8 files changed, 89 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 535dbf95..5ab54dba 100644 --- a/README.md +++ b/README.md @@ -194,8 +194,8 @@ even when the same bearer-authenticated request works with curl. ```yaml # MeshMapper integrations (optional, no API key). Import transport scopes and # IATA border outlines from MeshMapper instead of maintaining scopes: and -# iatas.*.borderFile by hand. Both need the IATAs to be in a configured region. -# See config.yaml.example for limits and behaviour. +# iatas.*.borderFile by hand. Scope sources must be in a configured region; +# zones covers every known IATA. See config.yaml.example for limits and behaviour. #meshmapper: # scopes: # enabled: true diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 36d9a58e..7e547514 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -210,7 +210,7 @@ func main() { } // Always restored, so disabling the import prunes it and file borders return. - zones := meshmapper.NewZones(cfg.MeshMapper.Zones, cfg.RegionIATAs(), store) + zones := meshmapper.NewZones(cfg.MeshMapper.Zones, store) if cr, ok := reader.(*cache.CachedReader); ok { zones.OnChange(cr.InvalidateIATABorder) } diff --git a/config.yaml.example b/config.yaml.example index acd61c5f..96967a11 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -62,12 +62,13 @@ ratelimit: # sources: # YOW: https://yow.meshmapper.net/get_scopes.php # -# Region boundaries (Zones API). Every IATA in a configured -# region is looked up in its country's zone list, then its get_geojson.php outline -# is imported. An imported boundary overrides that IATA's borderFile on the border -# map and for nodes.mark_foreign. A missing or failed boundary keeps the last good -# import, or the borderFile when there is none. Disabling, or removing an IATA from -# every region, drops its import on restart. Logged under component=meshmapper.zones. +# Region boundaries (Zones API). Every known IATA, including ones auto-created +# from traffic, is looked up in its country's zone list, then its get_geojson.php +# outline is imported; new IATAs are picked up while running. An imported boundary +# overrides that IATA's borderFile on the border map and for nodes.mark_foreign. +# A missing or failed boundary keeps the last good import, or the borderFile when +# there is none. Disabling drops every import on restart. Logged under +# component=meshmapper.zones. # zones: # enabled: false # refresh_interval: 24h # 1h-168h; one request per 15s tick, ETag-conditional diff --git a/db/meshmapper.go b/db/meshmapper.go index 0e2bc289..884b5506 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -48,6 +48,18 @@ func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache return s.q.SaveScopeCatalogue(ctx, params) } +func (s *Store) ListZoneIATAs(ctx context.Context) ([]string, error) { + rows, err := s.q.ListIATAs(ctx) + if err != nil { + return nil, err + } + iatas := make([]string, 0, len(rows)) + for _, row := range rows { + iatas = append(iatas, row.Iata) + } + return iatas, nil +} + func (s *Store) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { if keep == nil { keep = []string{} diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go index 31bec785..0d80c6a5 100644 --- a/internal/config/meshmapper.go +++ b/internal/config/meshmapper.go @@ -7,7 +7,6 @@ import ( "fmt" "net/url" "regexp" - "slices" "time" ) @@ -19,7 +18,7 @@ type MeshMapperConfig struct { Zones MeshMapperZonesConfig `yaml:"zones"` } -// MeshMapperZonesConfig imports each region IATA's published boundary, overriding borderFile. +// MeshMapperZonesConfig imports each known IATA's published boundary, overriding borderFile. type MeshMapperZonesConfig struct { Enabled bool `yaml:"enabled"` RefreshInterval duration `yaml:"refresh_interval"` @@ -32,16 +31,6 @@ func (c MeshMapperZonesConfig) Interval() time.Duration { return c.RefreshInterval.Duration } -// RegionIATAs returns the sorted, unique IATAs of every configured region. -func (c *Config) RegionIATAs() []string { - var iatas []string - for _, region := range c.Regions { - iatas = append(iatas, region.IATAs...) - } - slices.Sort(iatas) - return slices.Compact(iatas) -} - // MeshMapperScopesConfig augments, but never replaces, the manual scopes list. type MeshMapperScopesConfig struct { Enabled bool `yaml:"enabled"` @@ -64,9 +53,6 @@ func (c *Config) validateMeshMapper() error { if z.Interval() < time.Hour || z.Interval() > 7*24*time.Hour { return fmt.Errorf("meshmapper.zones.refresh_interval must be between 1h and 168h") } - if len(c.RegionIATAs()) == 0 { - return fmt.Errorf("meshmapper.zones needs at least one configured region IATA") - } } s := c.MeshMapper.Scopes if !s.Enabled { diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go index d4e343ba..13d27206 100644 --- a/internal/config/meshmapper_test.go +++ b/internal/config/meshmapper_test.go @@ -59,7 +59,7 @@ func TestMeshMapperZonesConfig(t *testing.T) { {"hourly", valid + " refresh_interval: 1h\n", false, time.Hour}, {"too often", valid + " refresh_interval: 59m\n", true, 0}, {"too rare", valid + " refresh_interval: 169h\n", true, 0}, - {"no regions", "meshmapper:\n zones:\n enabled: true\n", true, 0}, + {"no regions", "meshmapper:\n zones:\n enabled: true\n", false, 24 * time.Hour}, {"disabled", "meshmapper:\n zones:\n enabled: false\n", false, 24 * time.Hour}, } { t.Run(tc.name, func(t *testing.T) { @@ -76,8 +76,4 @@ func TestMeshMapperZonesConfig(t *testing.T) { } }) } - cfg := &Config{Regions: []RegionConfig{{IATAs: []string{"YYZ", "YOW"}}, {IATAs: []string{"YOW"}}}} - if got := strings.Join(cfg.RegionIATAs(), ","); got != "YOW,YYZ" { - t.Fatal(got) - } } diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go index 2d866615..37e6bc0b 100644 --- a/internal/meshmapper/zones.go +++ b/internal/meshmapper/zones.go @@ -42,6 +42,7 @@ type Boundary struct { } type ZoneStore interface { + ListZoneIATAs(ctx context.Context) ([]string, error) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) ListZoneBoundaries(ctx context.Context) ([]Boundary, error) SaveZoneBoundary(ctx context.Context, b Boundary) error @@ -67,7 +68,7 @@ type zoneList struct { type Zones struct { store ZoneStore enabled bool - iatas []string + seen map[string]bool interval time.Duration client *http.Client listURL string @@ -79,8 +80,8 @@ type Zones struct { onUpdate func(imported map[string]json.RawMessage) } -func NewZones(cfg config.MeshMapperZonesConfig, iatas []string, store ZoneStore) *Zones { - return &Zones{store: store, enabled: cfg.Enabled, iatas: iatas, interval: cfg.Interval(), listURL: ZonesURL, +func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore) *Zones { + return &Zones{store: store, enabled: cfg.Enabled, seen: map[string]bool{}, interval: cfg.Interval(), listURL: ZonesURL, boundsURL: boundaryEndpoint, lists: map[string]*zoneList{}, client: &http.Client{ Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, @@ -93,12 +94,16 @@ func (z *Zones) OnChange(fn func(ctx context.Context, iata string)) { z.onChange // OnUpdate receives every imported boundary whenever the set changes. func (z *Zones) OnUpdate(fn func(imported map[string]json.RawMessage)) { z.onUpdate = fn } -// Restore prunes imports for unconfigured IATAs (all of them when disabled) and +// Restore prunes imports for unknown IATAs (all of them when disabled) and // loads saved boundaries, without making HTTP requests. func (z *Zones) Restore(ctx context.Context) error { + iatas, err := z.store.ListZoneIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) + } keep := []string{} if z.enabled { - keep = z.iatas + keep = iatas } pruned, err := z.store.PruneZoneBoundaries(ctx, keep) if err != nil { @@ -119,13 +124,24 @@ func (z *Zones) Restore(ctx context.Context) error { for _, b := range saved { byIATA[b.IATA] = b } - for _, iata := range z.iatas { + z.track(iatas, byIATA) + z.publish() + return nil +} + +// track adds a region for each IATA not seen before. +func (z *Zones) track(iatas []string, saved map[string]Boundary) { + for _, iata := range iatas { + if z.seen[iata] { + continue + } + z.seen[iata] = true country := iatadb.CountryFor(iata) if country == "" { slog.Warn("MeshMapper boundary skipped: IATA has no known country", "component", "meshmapper.zones", "iata", iata) continue } - b, ok := byIATA[iata] + b, ok := saved[iata] if !ok { b = Boundary{IATA: iata} } @@ -133,10 +149,10 @@ func (z *Zones) Restore(ctx context.Context) error { z.retryAfter = b.NextAttempt } z.regions = append(z.regions, &zoneRegion{country: country, b: b}) - z.log(b, "restored") + if ok { + z.log(b, "restored") + } } - z.publish() - return nil } // Refresh makes at most one request. The scheduler serializes calls every 15s. @@ -153,6 +169,12 @@ func (z *Zones) Refresh(ctx context.Context) (err error) { if now.Before(z.retryAfter) { return nil } + // IATAs are created from traffic, so pick up new ones every tick. + iatas, err := z.store.ListZoneIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) + } + z.track(iatas, nil) for _, r := range z.regions { if now.Before(r.b.NextAttempt) { continue diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go index af62cc84..9798e3fe 100644 --- a/internal/meshmapper/zones_test.go +++ b/internal/meshmapper/zones_test.go @@ -25,11 +25,16 @@ func boundaryBody(code, geometry string) string { } type zoneMemoryStore struct { + iatas []string rows map[string]Boundary pruned []string fail bool } +func (s *zoneMemoryStore) ListZoneIATAs(context.Context) ([]string, error) { + return slices.Clone(s.iatas), nil +} + func (s *zoneMemoryStore) PruneZoneBoundaries(_ context.Context, keep []string) ([]string, error) { var removed []string for iata := range s.rows { @@ -113,7 +118,10 @@ type zoneHarness struct { func newZoneHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, enabled bool) *zoneHarness { t.Helper() h := &zoneHarness{store: store} - h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, []string{"YOW"}, store) + if store.iatas == nil { + store.iatas = []string{"YOW"} + } + h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, store) h.z.listURL = f.URL + "/get_zones.php" h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", strings.HasPrefix(site, f.URL) } h.z.OnChange(func(_ context.Context, iata string) { h.changed = append(h.changed, iata) }) @@ -266,7 +274,7 @@ func TestZonesRestoreAndPrune(t *testing.T) { restarted := newZoneHarness(t, f, store, true) if !slices.Equal(store.pruned, []string{"OLD"}) || !slices.Equal(restarted.changed, []string{"OLD"}) { - t.Fatal("unconfigured IATA not pruned", store.pruned, restarted.changed) + t.Fatal("unknown IATA not pruned", store.pruned, restarted.changed) } if restarted.imported["YOW"] == nil || f.calls != 1 { t.Fatal("saved boundary not restored without a request") @@ -281,6 +289,29 @@ func TestZonesRestoreAndPrune(t *testing.T) { } } +func TestZonesPickUpNewIATAs(t *testing.T) { + f := newFakeMeshMapper(t) + store := &zoneMemoryStore{iatas: []string{}, rows: map[string]Boundary{}} + h := newZoneHarness(t, f, store, true) + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if f.listCalls != 0 || f.calls != 0 { + t.Fatal("requested with no IATAs", f.listCalls, f.calls) + } + store.iatas = []string{"YOW", "ZZZ"} + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if len(h.z.regions) != 1 || h.imported["YOW"] == nil || !slices.Equal(h.changed, []string{"YOW"}) { + t.Fatal("new IATA not imported", len(h.z.regions), h.changed) + } +} + func TestZonesPersistFailureWaitsForNextAttempt(t *testing.T) { f := newFakeMeshMapper(t) h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}, fail: true}, true) From 7267b8bca361ccd1e2628b3c6bd93af8bf06dd85 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 22:27:36 -0400 Subject: [PATCH 115/141] feat(meshmapper): accept scope sources outside configured regions --- README.md | 4 ++-- config.yaml.example | 4 ++-- internal/config/meshmapper.go | 16 +++++++--------- internal/config/meshmapper_test.go | 5 ++++- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5ab54dba..5b07c462 100644 --- a/README.md +++ b/README.md @@ -194,8 +194,8 @@ even when the same bearer-authenticated request works with curl. ```yaml # MeshMapper integrations (optional, no API key). Import transport scopes and # IATA border outlines from MeshMapper instead of maintaining scopes: and -# iatas.*.borderFile by hand. Scope sources must be in a configured region; -# zones covers every known IATA. See config.yaml.example for limits and behaviour. +# iatas.*.borderFile by hand. Neither needs the IATAs to be in a configured +# region. See config.yaml.example for limits and behaviour. #meshmapper: # scopes: # enabled: true diff --git a/config.yaml.example b/config.yaml.example index 96967a11..39160c2a 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -43,10 +43,10 @@ ratelimit: # MeshMapper integrations (optional, no API key). Import transport scopes and # IATA border outlines from MeshMapper instead of maintaining scopes: and -# iatas.*.borderFile by hand. Both need the IATAs to be in a configured region below. +# iatas.*.borderFile by hand. # # Scope-name discovery. Manual scopes remain authoritative. -# Sources must belong to a configured region's IATAs. +# Sources are keyed by IATA code; the IATA need not be in a configured region. # Group endpoints are not supported: their names cannot be attributed to each IATA. # At most 16 sources, 64 names per source, 64 KiB per response; no page scraping. # Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go index 0d80c6a5..e0bc1077 100644 --- a/internal/config/meshmapper.go +++ b/internal/config/meshmapper.go @@ -45,7 +45,10 @@ func (c MeshMapperScopesConfig) Interval() time.Duration { return c.RefreshInterval.Duration } -var meshMapperHost = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) +var ( + meshMapperHost = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) + iataCode = regexp.MustCompile(`^[A-Z]{3}$`) +) func (c *Config) validateMeshMapper() error { if z := c.MeshMapper.Zones; z.Enabled { @@ -64,15 +67,10 @@ func (c *Config) validateMeshMapper() error { if s.Interval() < 5*time.Minute || s.Interval() > 24*time.Hour { return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 5m and 24h") } - configured := map[string]bool{} - for _, region := range c.Regions { - for _, iata := range region.IATAs { - configured[iata] = true - } - } for iata, endpoint := range s.Sources { - if !configured[iata] { - return fmt.Errorf("meshmapper.scopes source %q must belong to a configured region", iata) + // Catalogues must name the source IATA exactly, so reject keys that could never match. + if !iataCode.MatchString(iata) { + return fmt.Errorf("meshmapper.scopes source %q must be a three-letter uppercase IATA code", iata) } u, err := url.Parse(endpoint) if err != nil || u.Scheme != "https" || u.User != nil || !meshMapperHost.MatchString(u.Host) || diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go index 13d27206..cbaf0b3e 100644 --- a/internal/config/meshmapper_test.go +++ b/internal/config/meshmapper_test.go @@ -18,7 +18,10 @@ func TestMeshMapperConfig(t *testing.T) { bad bool }{ {"valid", valid, false}, - {"unknown region", strings.Replace(valid, "iatas: [YOW]", "iatas: [YVR]", 1), true}, + {"outside any region", strings.Replace(valid, "iatas: [YOW]", "iatas: [YVR]", 1), false}, + {"no regions", valid[strings.Index(valid, "meshmapper:"):], false}, + {"lowercase IATA", strings.Replace(valid, " YOW:", " yow:", 1), true}, + {"long IATA", strings.Replace(valid, " YOW:", " YOWX:", 1), true}, {"http", strings.Replace(valid, "https:", "http:", 1), true}, {"private host", strings.Replace(valid, "yow.meshmapper.net", "127.0.0.1", 1), true}, {"credentials", strings.Replace(valid, "https://", "https://secret@", 1), true}, From bfda0d5281e3ab865a916932631c09e6655bcd16 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 22:46:00 -0400 Subject: [PATCH 116/141] feat(meshmapper): discover scope sources from the zone list --- README.md | 6 +- cmd/beacon/main.go | 5 +- config.yaml.example | 15 +-- db/meshmapper.go | 19 ++- db/meshmapper_integration_test.go | 9 +- db/queries/queries.sql | 3 + db/sqlc/mock/querier.go | 15 +++ db/sqlc/querier.go | 1 + db/sqlc/queries.sql.go | 33 ++++++ internal/config/meshmapper.go | 29 +---- internal/config/meshmapper_test.go | 24 ++-- internal/meshmapper/directory.go | 139 ++++++++++++++++++++++ internal/meshmapper/scopes.go | 125 +++++++++++++++----- internal/meshmapper/scopes_test.go | 184 +++++++++++++++++++++-------- internal/meshmapper/zones.go | 154 +++++++----------------- internal/meshmapper/zones_test.go | 11 +- 16 files changed, 516 insertions(+), 256 deletions(-) create mode 100644 internal/meshmapper/directory.go diff --git a/README.md b/README.md index 5b07c462..119c3291 100644 --- a/README.md +++ b/README.md @@ -194,13 +194,11 @@ even when the same bearer-authenticated request works with curl. ```yaml # MeshMapper integrations (optional, no API key). Import transport scopes and # IATA border outlines from MeshMapper instead of maintaining scopes: and -# iatas.*.borderFile by hand. Neither needs the IATAs to be in a configured -# region. See config.yaml.example for limits and behaviour. +# iatas.*.borderFile by hand. Both cover every known IATA that MeshMapper lists; +# no sources to configure. See config.yaml.example for limits and behaviour. #meshmapper: # scopes: # enabled: true -# sources: -# YVR: https://yvr.meshmapper.net/get_scopes.php # zones: # enabled: true diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 7e547514..c6f93a1b 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -198,10 +198,11 @@ func main() { scopes.Load(scopeEntries) scopes.SetManualMembers(cfg.ManualScopeMembers()) slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") + directory := meshmapper.NewDirectory() var scopeImporter *meshmapper.Importer if cfg.MeshMapper.Scopes.Enabled { restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) - scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, scopes, scopeEntries) + scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, directory, scopes, scopeEntries) cancelRestore() if err != nil { slog.Error("failed to restore MeshMapper scope catalogues", "component", "startup", "error", err) @@ -210,7 +211,7 @@ func main() { } // Always restored, so disabling the import prunes it and file borders return. - zones := meshmapper.NewZones(cfg.MeshMapper.Zones, store) + zones := meshmapper.NewZones(cfg.MeshMapper.Zones, store, directory) if cr, ok := reader.(*cache.CachedReader); ok { zones.OnChange(cr.InvalidateIATABorder) } diff --git a/config.yaml.example b/config.yaml.example index 39160c2a..cc863fc7 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -46,21 +46,22 @@ ratelimit: # iatas.*.borderFile by hand. # # Scope-name discovery. Manual scopes remain authoritative. -# Sources are keyed by IATA code; the IATA need not be in a configured region. -# Group endpoints are not supported: their names cannot be attributed to each IATA. -# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping. +# Every known IATA, including ones auto-created from traffic, is looked up in its +# country's zone list (shared with zones below) and its site's get_scopes.php +# catalogue is imported; IATAs MeshMapper doesn't list are skipped. No sources +# to configure; a legacy sources: list is ignored with a warning. +# Catalogues covering more than one IATA are rejected. +# 64 names per catalogue, 64 KiB per response; no page scraping. # Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. # Refresh success/failure/freshness is logged under component=meshmapper.scopes. -# Removing/turning off a source deactivates its imported matching keys on restart; +# Turning scopes off deactivates imported matching keys on restart; # historical scope identities and recorded evidence are retained. # Region-filtered scope lists and scope stats show imported names only under the -# IATAs whose catalogue lists them; regions without a source show manual scopes only. +# IATAs whose catalogue lists them; IATAs without a catalogue show manual scopes only. #meshmapper: # scopes: # enabled: false # refresh_interval: 1h # 5m-24h; one source checked per 15s tick -# sources: -# YOW: https://yow.meshmapper.net/get_scopes.php # # Region boundaries (Zones API). Every known IATA, including ones auto-created # from traffic, is looked up in its country's zone list, then its get_geojson.php diff --git a/db/meshmapper.go b/db/meshmapper.go index 884b5506..02e0de20 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -29,6 +29,23 @@ func (s *Store) GetScopeCatalogue(ctx context.Context, iata, url string) (*meshm return cache, nil } +func (s *Store) ListScopeCatalogues(ctx context.Context) ([]meshmapper.Catalogue, error) { + rows, err := s.q.ListScopeCatalogues(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.Catalogue, 0, len(rows)) + for _, row := range rows { + c := meshmapper.Catalogue{IATA: row.Iata, URL: row.Url, Cache: meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}} + if row.Etag != nil { + c.ETag = *row.Etag + } + out = append(out, c) + } + return out, nil +} + func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, entries []scopestore.Entry) error { params := sqlc.SaveScopeCatalogueParams{ Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError, @@ -48,7 +65,7 @@ func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache return s.q.SaveScopeCatalogue(ctx, params) } -func (s *Store) ListZoneIATAs(ctx context.Context) ([]string, error) { +func (s *Store) ListKnownIATAs(ctx context.Context) ([]string, error) { rows, err := s.q.ListIATAs(ctx) if err != nil { return nil, err diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go index 7d8c2a11..e1589a6d 100644 --- a/db/meshmapper_integration_test.go +++ b/db/meshmapper_integration_test.go @@ -44,7 +44,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) - for _, table := range []string{"transport_scopes", "meshmapper_scope_catalogues"} { + for _, table := range []string{"iata_codes", "transport_scopes", "meshmapper_scope_catalogues"} { if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { t.Fatal(err) } @@ -55,6 +55,9 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { if err := store.UpsertTransportScope(ctx, manual.Name, "Operator label", manual.TransportKey, manual.KeyFingerprint); err != nil { t.Fatal(err) } + if err := store.UpsertIATA(ctx, "YOW"); err != nil { + t.Fatal(err) + } url := "https://yow.meshmapper.net/get_scopes.php" if row, err := store.GetScopeCatalogue(ctx, "YOW", url); err != nil || row != nil { t.Fatal(row, err) @@ -79,7 +82,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { t.Fatal(row, err) } scopes := scopestore.New() - _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(), scopes, manualRows) if err != nil || len(scopes.Entries()) != 2 { t.Fatal("restart did not restore imported membership", err) } @@ -102,7 +105,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, nil); err != nil { t.Fatal(err) } - _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(), scopes, manualRows) if err != nil || len(scopes.Entries()) != 1 { t.Fatal("removed imported membership remained active", err) } diff --git a/db/queries/queries.sql b/db/queries/queries.sql index afcd8c5d..f215ce06 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1576,6 +1576,9 @@ SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, -- name: GetScopeCatalogue :one SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; +-- name: ListScopeCatalogues :many +SELECT * FROM meshmapper_scope_catalogues ORDER BY iata, attempted_at; + -- name: SaveScopeCatalogue :exec -- One statement commits the validated snapshot and its lookup identities together. -- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index a0aca7b5..5ab8a156 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -1254,6 +1254,21 @@ func (mr *MockQuerierMockRecorder) ListRouteEvidence(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRouteEvidence", reflect.TypeOf((*MockQuerier)(nil).ListRouteEvidence), ctx, arg) } +// ListScopeCatalogues mocks base method. +func (m *MockQuerier) ListScopeCatalogues(ctx context.Context) ([]db.MeshmapperScopeCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListScopeCatalogues", ctx) + ret0, _ := ret[0].([]db.MeshmapperScopeCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListScopeCatalogues indicates an expected call of ListScopeCatalogues. +func (mr *MockQuerierMockRecorder) ListScopeCatalogues(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListScopeCatalogues", reflect.TypeOf((*MockQuerier)(nil).ListScopeCatalogues), ctx) +} + // ListTraceTags mocks base method. func (m *MockQuerier) ListTraceTags(ctx context.Context, arg db.ListTraceTagsParams) ([]db.ListTraceTagsRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 7a64a312..1b3bbc99 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -207,6 +207,7 @@ type Querier interface { // Full-byte equality is required even when the compact digest matches. TRACE path bytes // carry readings; unclassified legacy observations cannot be safely called ordinary paths. ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) + ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCatalogue, error) // ============================================================ // TRACES // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index e8b52ff7..bc833771 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -3723,6 +3723,39 @@ func (q *Queries) ListRegions(ctx context.Context) ([]ListRegionsRow, error) { return items, nil } +const listScopeCatalogues = `-- name: ListScopeCatalogues :many +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues ORDER BY iata, attempted_at +` + +func (q *Queries) ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCatalogue, error) { + rows, err := q.db.Query(ctx, listScopeCatalogues) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperScopeCatalogue{} + for rows.Next() { + var i MeshmapperScopeCatalogue + if err := rows.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listTraceTags = `-- name: ListTraceTags :many WITH tags AS ( diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go index e0bc1077..d4fa6428 100644 --- a/internal/config/meshmapper.go +++ b/internal/config/meshmapper.go @@ -5,14 +5,10 @@ package config import ( "fmt" - "net/url" - "regexp" + "log/slog" "time" ) -// MaxScopeSources bounds both network refresh work and imported matching keys. -const MaxScopeSources = 16 - type MeshMapperConfig struct { Scopes MeshMapperScopesConfig `yaml:"scopes"` Zones MeshMapperZonesConfig `yaml:"zones"` @@ -32,9 +28,10 @@ func (c MeshMapperZonesConfig) Interval() time.Duration { } // MeshMapperScopesConfig augments, but never replaces, the manual scopes list. +// Sources are discovered per known IATA from the zone list. type MeshMapperScopesConfig struct { Enabled bool `yaml:"enabled"` - Sources map[string]string `yaml:"sources"` // configured IATA -> published regional endpoint + Sources map[string]string `yaml:"sources"` // deprecated: ignored, kept to warn RefreshInterval duration `yaml:"refresh_interval"` } @@ -45,11 +42,6 @@ func (c MeshMapperScopesConfig) Interval() time.Duration { return c.RefreshInterval.Duration } -var ( - meshMapperHost = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) - iataCode = regexp.MustCompile(`^[A-Z]{3}$`) -) - func (c *Config) validateMeshMapper() error { if z := c.MeshMapper.Zones; z.Enabled { // The Zones API asks clients not to poll more than once an hour. @@ -61,22 +53,11 @@ func (c *Config) validateMeshMapper() error { if !s.Enabled { return nil } - if len(s.Sources) == 0 || len(s.Sources) > MaxScopeSources { - return fmt.Errorf("meshmapper.scopes.sources must contain 1-%d regional endpoints", MaxScopeSources) - } if s.Interval() < 5*time.Minute || s.Interval() > 24*time.Hour { return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 5m and 24h") } - for iata, endpoint := range s.Sources { - // Catalogues must name the source IATA exactly, so reject keys that could never match. - if !iataCode.MatchString(iata) { - return fmt.Errorf("meshmapper.scopes source %q must be a three-letter uppercase IATA code", iata) - } - u, err := url.Parse(endpoint) - if err != nil || u.Scheme != "https" || u.User != nil || !meshMapperHost.MatchString(u.Host) || - u.Path != "/get_scopes.php" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { - return fmt.Errorf("meshmapper.scopes source %q must be a published https://.meshmapper.net/get_scopes.php endpoint without credentials or parameters", iata) - } + if len(s.Sources) > 0 { + slog.Warn("meshmapper.scopes.sources is ignored; sources are discovered from the MeshMapper zone list", "component", "config") } return nil } diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go index cbaf0b3e..8cd3264d 100644 --- a/internal/config/meshmapper_test.go +++ b/internal/config/meshmapper_test.go @@ -12,22 +12,16 @@ import ( ) func TestMeshMapperConfig(t *testing.T) { - valid := "regions:\n - slug: ottawa\n iatas: [YOW]\nmeshmapper:\n scopes:\n enabled: true\n sources:\n YOW: https://yow.meshmapper.net/get_scopes.php\n" + valid := "meshmapper:\n scopes:\n enabled: true\n" for _, tc := range []struct { name, text string bad bool }{ {"valid", valid, false}, - {"outside any region", strings.Replace(valid, "iatas: [YOW]", "iatas: [YVR]", 1), false}, - {"no regions", valid[strings.Index(valid, "meshmapper:"):], false}, - {"lowercase IATA", strings.Replace(valid, " YOW:", " yow:", 1), true}, - {"long IATA", strings.Replace(valid, " YOW:", " YOWX:", 1), true}, - {"http", strings.Replace(valid, "https:", "http:", 1), true}, - {"private host", strings.Replace(valid, "yow.meshmapper.net", "127.0.0.1", 1), true}, - {"credentials", strings.Replace(valid, "https://", "https://secret@", 1), true}, - {"query", strings.Replace(valid, "get_scopes.php", "get_scopes.php?key=secret", 1), true}, - {"non API", strings.Replace(valid, "get_scopes.php", "index.php", 1), true}, + {"daily", valid + " refresh_interval: 24h\n", false}, + {"legacy sources ignored", valid + " sources:\n yow: http://127.0.0.1/\n", false}, {"fast", valid + " refresh_interval: 1s\n", true}, + {"slow", valid + " refresh_interval: 25h\n", true}, {"negative", valid + " refresh_interval: -1h\n", true}, {"disabled", strings.Replace(valid, "enabled: true", "enabled: false", 1), false}, } { @@ -36,18 +30,14 @@ func TestMeshMapperConfig(t *testing.T) { if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { t.Fatal(err) } - cfg, err := Load(p) + _, err := Load(p) if (err != nil) != tc.bad { t.Fatalf("error=%v", err) } - if err == nil && cfg.MeshMapper.Scopes.Interval() != time.Hour { - t.Fatal("wrong default") - } }) } - cfg := Config{MeshMapper: MeshMapperConfig{Scopes: MeshMapperScopesConfig{Enabled: true}}} - if err := cfg.validateMeshMapper(); err == nil { - t.Fatal("empty sources accepted") + if (MeshMapperScopesConfig{}).Interval() != time.Hour { + t.Fatal("wrong default") } } diff --git a/internal/meshmapper/directory.go b/internal/meshmapper/directory.go new file mode 100644 index 00000000..0ead3474 --- /dev/null +++ b/internal/meshmapper/directory.go @@ -0,0 +1,139 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +type zoneEntry struct { + url string + hasBoundary bool +} + +type zoneList struct { + zones map[string]zoneEntry + etag string + fetchedAt, nextAttempt time.Time +} + +// Directory caches MeshMapper's per-country zone lists, shared by the zones and scopes imports. +type Directory struct { + mu sync.Mutex + client *http.Client + listURL string + lists map[string]*zoneList + retryAfter time.Time +} + +func NewDirectory() *Directory { + return &Directory{listURL: ZonesURL, lists: map[string]*zoneList{}, client: &http.Client{ + Timeout: 30 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} +} + +// pausedUntil is when a list 429 stops pausing every MeshMapper request. +func (d *Directory) pausedUntil() time.Time { + d.mu.Lock() + defer d.mu.Unlock() + return d.retryAfter +} + +// List returns the country's fresh zone list. fetched reports that this call +// spent the caller's one request; nil zones without fetched means back off. +func (d *Directory) List(ctx context.Context, country string, now time.Time) (zones map[string]zoneEntry, fetched bool, err error) { + d.mu.Lock() + defer d.mu.Unlock() + list := d.lists[country] + if list != nil && list.zones != nil && now.Sub(list.fetchedAt) < zoneListFresh { + return list.zones, false, nil + } + if now.Before(d.retryAfter) || (list != nil && now.Before(list.nextAttempt)) { + return nil, false, nil + } + return nil, true, d.fetch(ctx, country, now) +} + +func (d *Directory) fetch(ctx context.Context, country string, now time.Time) error { + list := d.lists[country] + if list == nil { + list = &zoneList{} + d.lists[country] = list + } + endpoint := d.listURL + "?country=" + url.QueryEscape(country) + etag := "" + if list.zones != nil { + etag = list.etag + } + status, body, header, err := get(ctx, d.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxZoneList) + if err != nil { + return err + } + problem := "" + switch status { + case http.StatusOK: + zones, decodeErr := decodeZones(body, country) + if decodeErr != nil { + problem = "invalid response" + } else { + list.zones, list.etag = zones, header.Get("ETag") + } + case http.StatusNotModified: + if list.zones == nil { + problem = "304 without cached list" + } + default: + var retryAt time.Time + problem = statusProblem(status, header, now, &retryAt) + if retryAt.After(list.nextAttempt) { + list.nextAttempt = retryAt + } + if until, ok := rateLimited(status, retryAt, now); ok { + d.retryAfter = until + } + } + if problem == "" { + list.fetchedAt = now + } else if list.nextAttempt.Before(now.Add(zoneFailureRetry)) { + list.nextAttempt = now.Add(zoneFailureRetry) + } + level := slog.LevelInfo + if problem != "" { + level = slog.LevelWarn + } + slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country, + "zones", len(list.zones), "last_error", problem) + return nil +} + +func decodeZones(body []byte, country string) (map[string]zoneEntry, error) { + var document struct { + Country string `json:"country"` + Zones *[]struct { + Code string `json:"code"` + URL string `json:"url"` + HasBoundary bool `json:"has_boundary"` + } `json:"zones"` + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, err + } + if !strings.EqualFold(document.Country, country) || document.Zones == nil { + return nil, fmt.Errorf("invalid zone list") + } + zones := make(map[string]zoneEntry, len(*document.Zones)) + for _, zone := range *document.Zones { + zones[strings.ToUpper(zone.Code)] = zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary} + } + return zones, nil +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index 1c267679..907f4272 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -19,11 +19,12 @@ import ( "unicode/utf8" "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) const ( - MaxScopes = 64 // per source; at most 16 configured sources (1024 imported identities) + MaxScopes = 64 // per source MaxBody = 64 << 10 PollInterval = 15 * time.Second // one source per tick, at most four requests/minute @@ -38,16 +39,23 @@ type Cache struct { LastError string } +// Catalogue is one saved source snapshot. +type Catalogue struct { + IATA, URL string + Cache +} + type Store interface { - GetScopeCatalogue(context.Context, string, string) (*Cache, error) + ListKnownIATAs(context.Context) ([]string, error) + ListScopeCatalogues(context.Context) ([]Catalogue, error) SaveScopeCatalogue(context.Context, string, string, Cache, []scopestore.Entry) error } type source struct { - iata, url string - cache Cache - entries []scopestore.Entry - generated time.Time + iata, country, url string + cache Cache + entries []scopestore.Entry + generated time.Time } // Importer is owned by one background task; ScopeStore synchronizes its consumers. @@ -55,6 +63,9 @@ type Importer struct { store Store scopes *scopestore.ScopeStore manual []scopestore.Entry + dir *Directory + scopesURL func(site string) (string, bool) + seen map[string]bool sources []source interval time.Duration client *http.Client @@ -66,44 +77,65 @@ type Importer struct { func (i *Importer) SetCacheInvalidator(fn func(context.Context)) { i.onChange = fn } // New restores validated snapshots before ingestion, without making HTTP requests. -func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) { - i := &Importer{store: store, scopes: scopes, manual: manual, interval: cfg.Interval(), client: &http.Client{ - Timeout: 10 * time.Second, - CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, - }} +func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, dir *Directory, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) { + i := &Importer{store: store, scopes: scopes, manual: manual, dir: dir, scopesURL: scopesEndpoint, seen: map[string]bool{}, + interval: cfg.Interval(), client: &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} if !cfg.Enabled { return i, nil } - keys := make([]string, 0, len(cfg.Sources)) - for key := range cfg.Sources { - keys = append(keys, key) + saved, err := store.ListScopeCatalogues(ctx) + if err != nil { + return nil, fmt.Errorf("restore scope catalogues: %w", err) + } + // A site that moved leaves its old row behind; the latest attempt wins. + latest := map[string]Catalogue{} + for _, c := range saved { + if old, ok := latest[c.IATA]; !ok || c.AttemptedAt.After(old.AttemptedAt) { + latest[c.IATA] = c + } + } + iatas, err := store.ListKnownIATAs(ctx) + if err != nil { + return nil, fmt.Errorf("list IATAs for MeshMapper scopes: %w", err) } - slices.Sort(keys) - for _, key := range keys { - s := source{iata: key, url: cfg.Sources[key]} - cached, err := store.GetScopeCatalogue(ctx, s.iata, s.url) - if err != nil { - return nil, fmt.Errorf("restore scope catalogue %s: %w", key, err) + i.track(iatas, latest) + i.publish() + return i, nil +} + +// track adds a source for each IATA not seen before. +func (i *Importer) track(iatas []string, saved map[string]Catalogue) { + for _, iata := range iatas { + if i.seen[iata] { + continue + } + i.seen[iata] = true + country := iatadb.CountryFor(iata) + if country == "" { + continue } - if cached != nil { - s.cache = *cached - if len(cached.Payload) > 0 { - s.entries, s.generated, err = decode(cached.Payload, key) + s := source{iata: iata, country: country} + if c, ok := saved[iata]; ok { + s.url, s.cache = c.URL, c.Cache + if len(c.Payload) > 0 { + var err error + s.entries, s.generated, err = decode(c.Payload, iata) if err != nil { s.entries = nil s.cache.Payload, s.cache.ETag = nil, "" s.cache.LastError = "invalid saved catalogue; awaiting refresh" } } - if cached.LastError == "HTTP 429" && cached.NextAttempt.After(i.retryAfter) { - i.retryAfter = cached.NextAttempt + if c.LastError == "HTTP 429" && c.NextAttempt.After(i.retryAfter) { + i.retryAfter = c.NextAttempt } + i.log(s, "restored") } i.sources = append(i.sources, s) - i.log(s, "restored") } - i.publish() - return i, nil } // Refresh checks only one due source. The scheduler serializes calls every 15s. @@ -117,14 +149,43 @@ func (i *Importer) Refresh(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() now := time.Now().UTC() - if now.Before(i.retryAfter) { + if now.Before(i.retryAfter) || now.Before(i.dir.pausedUntil()) { return nil } + iatas, err := i.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper scopes: %w", err) + } + i.track(iatas, nil) for n := range i.sources { s := &i.sources[n] if now.Before(s.cache.NextAttempt) { continue } + zones, fetched, err := i.dir.List(ctx, s.country, now) + if err != nil || fetched { + return err + } + if zones == nil { + continue + } + entry, listed := zones[s.iata] + endpoint, valid := "", false + if listed { + endpoint, valid = i.scopesURL(entry.url) + } + if !valid { + // No request to make; names already imported stay until a catalogue replaces them. + s.cache.NextAttempt, s.cache.LastError = now.Add(i.interval), "not listed" + if listed { + s.cache.LastError = "invalid site URL" + } + i.log(*s, "skipped") + continue + } + if endpoint != s.url { + s.url, s.cache.ETag = endpoint, "" + } return i.refresh(ctx, s, now) } return nil @@ -263,13 +324,15 @@ func (i *Importer) publish() { func (i *Importer) log(s source, action string) { level := slog.LevelInfo - if s.cache.LastError != "" { + if s.cache.LastError != "" && s.cache.LastError != "not listed" { level = slog.LevelWarn } slog.Log(context.Background(), level, "MeshMapper scopes "+action, "component", "meshmapper.scopes", "iata", s.iata, "source", s.url, "names", len(s.entries), "generated_at", s.generated, "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError) } +func scopesEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_scopes.php") } + func decode(body []byte, iata string) ([]scopestore.Entry, time.Time, error) { var document struct { GeneratedAt time.Time `json:"generated_at"` diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go index 95ac0b96..d12dfc0c 100644 --- a/internal/meshmapper/scopes_test.go +++ b/internal/meshmapper/scopes_test.go @@ -11,6 +11,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "slices" "strings" "sync/atomic" "testing" @@ -29,24 +30,36 @@ func catalogue(names ...string) []byte { return b } +type sourceKey struct{ iata, url string } + type memoryStore struct { - rows map[string]Cache + iatas []string + rows map[sourceKey]Cache fail bool saved []scopestore.Entry } -func (s *memoryStore) GetScopeCatalogue(_ context.Context, iata, url string) (*Cache, error) { - row, ok := s.rows[iata+url] - if !ok { - return nil, nil +func newMemoryStore(iatas ...string) *memoryStore { + return &memoryStore{iatas: iatas, rows: map[sourceKey]Cache{}} +} + +func (s *memoryStore) ListKnownIATAs(context.Context) ([]string, error) { + return slices.Clone(s.iatas), nil +} + +func (s *memoryStore) ListScopeCatalogues(context.Context) ([]Catalogue, error) { + var out []Catalogue + for key, c := range s.rows { + out = append(out, Catalogue{IATA: key.iata, URL: key.url, Cache: c}) } - return &row, nil + return out, nil } + func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, next Cache, entries []scopestore.Entry) error { if s.fail { return errors.New("offline") } - old := s.rows[iata+url] + old := s.rows[sourceKey{iata, url}] if next.Payload == nil { next.Payload = old.Payload } @@ -54,11 +67,25 @@ func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, ne next.CheckedAt = old.CheckedAt next.ETag = old.ETag } - s.rows[iata+url] = next + s.rows[sourceKey{iata, url}] = next s.saved = append(s.saved, entries...) return nil } +// newImporter restores an importer for YOW whose source points at url. +func newImporter(t *testing.T, cfg config.MeshMapperScopesConfig, store *memoryStore, url string, scopes *scopestore.ScopeStore, manual ...scopestore.Entry) *Importer { + t.Helper() + imp, err := New(context.Background(), cfg, store, NewDirectory(), scopes, manual) + if err != nil { + t.Fatal(err) + } + if len(imp.sources) != 1 { + t.Fatal("expected one source", imp.sources) + } + imp.sources[0].url = url + return imp +} + func names(store *scopestore.ScopeStore) []string { out := []string{} for _, e := range store.Entries() { @@ -84,15 +111,12 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { } })) defer server.Close() - cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} - store := &memoryStore{rows: map[string]Cache{}} + cfg := config.MeshMapperScopesConfig{Enabled: true} + store := newMemoryStore("YOW") scopes := scopestore.New() manual := scopestore.FromName("manual") manual.TransportKey = []byte("manual wins") - imp, err := New(ctx, cfg, store, scopes, []scopestore.Entry{manual}) - if err != nil { - t.Fatal(err) - } + imp := newImporter(t, cfg, store, server.URL, scopes, manual) now := time.Now().UTC() check := func() { t.Helper() @@ -134,10 +158,7 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { t.Fatal("Retry-After ignored") } restored := scopestore.New() - restart, err := New(ctx, cfg, store, restored, []scopestore.Entry{manual}) - if err != nil { - t.Fatal(err) - } + restart := newImporter(t, cfg, store, server.URL, restored, manual) if !reflect.DeepEqual(names(restored), names(scopes)) { t.Fatal("outage restart lost cached names") } @@ -215,11 +236,7 @@ func TestNoRedirectAndResponseBounds(t *testing.T) { } })) defer server.Close() - store := &memoryStore{rows: map[string]Cache{}} - imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopestore.New(), nil) - if err != nil { - t.Fatal(err) - } + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) imp.client.Timeout = 20 * time.Millisecond for _, value := range []string{"redirect", "oversize", "timeout", "304"} { mode.Store(value) @@ -237,25 +254,22 @@ func TestNoRedirectAndResponseBounds(t *testing.T) { func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { ctx := context.Background() - store := &memoryStore{rows: map[string]Cache{}} + store := newMemoryStore("YOW", "YVR") for _, iata := range []string{"YOW", "YVR"} { - store.rows[iata+iata] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata))} + store.rows[sourceKey{iata, "old-" + iata}] = Cache{Payload: []byte(`{}`)} + store.rows[sourceKey{iata, iata}] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata)), AttemptedAt: time.Now()} } scopes := scopestore.New() - _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "YOW", "YVR": "YVR"}}, store, scopes, nil) + _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, NewDirectory(), scopes, nil) if err != nil || len(scopes.Entries()) != 1 || !reflect.DeepEqual(scopes.Entries()[0].IATAs, []string{"YOW", "YVR"}) { - t.Fatal(scopes.Entries(), err) + t.Fatal("latest saved catalogue per IATA not restored", scopes.Entries(), err) } if got := scopes.NamesForIATAs([]string{"YVR"}); !reflect.DeepEqual(got, []string{"#can"}) { t.Fatal(got) } - _, err = New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "changed-source"}}, store, scopes, nil) - if err != nil || len(scopes.Entries()) != 0 || len(scopes.NamesForIATAs([]string{"YOW", "YVR"})) != 0 { - t.Fatal("removed/changed source remained active", err) - } - // Disabled mode never calls the store, even if URLs remain configured. - _, err = New(ctx, config.MeshMapperScopesConfig{Sources: map[string]string{"YOW": "YOW"}}, nil, scopes, nil) - if err != nil { + // Disabled mode never calls the store. + disabled := scopestore.New() + if _, err = New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(), disabled, nil); err != nil || len(disabled.Entries()) != 0 { t.Fatal(err) } } @@ -268,19 +282,20 @@ func TestInvalidSavedCatalogueDoesNotBlockStartupOrRefresh(t *testing.T) { _, _ = w.Write(catalogue("yow")) })) defer server.Close() - store := &memoryStore{rows: map[string]Cache{"YOW" + server.URL: {Payload: []byte(`{"invalid":true}`), ETag: `"old"`}}} + store := newMemoryStore("YOW") + store.rows[sourceKey{"YOW", server.URL}] = Cache{Payload: []byte(`{"invalid":true}`), ETag: `"old"`} scopes := scopestore.New() - imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopes, []scopestore.Entry{scopestore.FromName("manual")}) - if err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual"}) { - t.Fatal("invalid optional source blocked manual startup", err) + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, store, server.URL, scopes, scopestore.FromName("manual")) + if !reflect.DeepEqual(names(scopes), []string{"#manual"}) { + t.Fatal("invalid optional source blocked manual startup") } - if err = imp.Refresh(context.Background()); err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual", "#yow"}) { + if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual", "#yow"}) { t.Fatal("source did not recover on refresh", err, names(scopes)) } ctx, cancel := context.WithCancel(context.Background()) cancel() imp.sources[0].cache.NextAttempt = time.Time{} - if err = imp.Refresh(ctx); err != nil { + if err := imp.Refresh(ctx); err != nil { t.Fatalf("normal shutdown is a task failure: %v", err) } } @@ -292,11 +307,7 @@ func TestRetryAfterShorterThanIntervalIsHonoured(t *testing.T) { w.WriteHeader(http.StatusTooManyRequests) })) defer server.Close() - cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} - imp, err := New(ctx, cfg, &memoryStore{rows: map[string]Cache{}}, scopestore.New(), nil) - if err != nil { - t.Fatal(err) - } + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) now := time.Now().UTC() if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { t.Fatal(err) @@ -310,11 +321,7 @@ func TestFailedRefreshRetriesSoon(t *testing.T) { ctx := context.Background() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) })) defer server.Close() - cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} - imp, err := New(ctx, cfg, &memoryStore{rows: map[string]Cache{}}, scopestore.New(), nil) - if err != nil { - t.Fatal(err) - } + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) now := time.Now().UTC() if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { t.Fatal(err) @@ -323,3 +330,80 @@ func TestFailedRefreshRetriesSoon(t *testing.T) { t.Fatalf("next attempt after failure in %v, want %v", got, failureRetry) } } + +// fakeSites serves a CA zone list and YOW's scope catalogue. +type fakeSites struct { + *httptest.Server + list string + listCalls, scopeHits int +} + +func newFakeSites(t *testing.T) *fakeSites { + f := &fakeSites{} + f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/get_zones.php": + f.listCalls++ + _, _ = w.Write([]byte(f.list)) + case "/get_scopes.php": + f.scopeHits++ + _, _ = w.Write(catalogue("yow")) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + f.list = `{"country":"CA","zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}]}` + t.Cleanup(f.Close) + return f +} + +func newDiscoveringImporter(t *testing.T, f *fakeSites, store *memoryStore, scopes *scopestore.ScopeStore) *Importer { + t.Helper() + dir := NewDirectory() + dir.listURL = f.URL + "/get_zones.php" + imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopes, nil) + if err != nil { + t.Fatal(err) + } + imp.scopesURL = func(site string) (string, bool) { return site + "get_scopes.php", strings.HasPrefix(site, f.URL) } + return imp +} + +func refreshN(t *testing.T, imp *Importer, n int) { + t.Helper() + for range n { + if err := imp.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestScopesDiscoverSourcesFromZoneList(t *testing.T) { + f := newFakeSites(t) + store := newMemoryStore() + scopes := scopestore.New() + imp := newDiscoveringImporter(t, f, store, scopes) + refreshN(t, imp, 2) + if f.listCalls != 0 || f.scopeHits != 0 { + t.Fatal("requested with no IATAs", f.listCalls, f.scopeHits) + } + store.iatas = []string{"YOW", "ZZZ"} + refreshN(t, imp, 3) + if f.listCalls != 1 || f.scopeHits != 1 || !reflect.DeepEqual(names(scopes), []string{"#yow"}) { + t.Fatal("source not discovered", f.listCalls, f.scopeHits, names(scopes)) + } + if _, ok := store.rows[sourceKey{"YOW", f.URL + "/get_scopes.php"}]; !ok { + t.Fatal("catalogue not saved under the discovered URL", store.rows) + } +} + +func TestScopesSkipUnlistedIATAs(t *testing.T) { + f := newFakeSites(t) + f.list = `{"country":"CA","zones":[]}` + imp := newDiscoveringImporter(t, f, newMemoryStore("YOW"), scopestore.New()) + refreshN(t, imp, 3) + s := imp.sources[0] + if f.scopeHits != 0 || s.cache.LastError != "not listed" || s.cache.NextAttempt.Sub(time.Now()) < 59*time.Minute { + t.Fatalf("unlisted IATA requested or retried early: hits=%d %+v", f.scopeHits, s.cache) + } +} diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go index 37e6bc0b..55d23985 100644 --- a/internal/meshmapper/zones.go +++ b/internal/meshmapper/zones.go @@ -42,7 +42,7 @@ type Boundary struct { } type ZoneStore interface { - ListZoneIATAs(ctx context.Context) ([]string, error) + ListKnownIATAs(ctx context.Context) ([]string, error) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) ListZoneBoundaries(ctx context.Context) ([]Boundary, error) SaveZoneBoundary(ctx context.Context, b Boundary) error @@ -53,17 +53,6 @@ type zoneRegion struct { b Boundary } -type zoneEntry struct { - url string - hasBoundary bool -} - -type zoneList struct { - zones map[string]zoneEntry - etag string - fetchedAt, nextAttempt time.Time -} - // Zones is owned by one background task; listeners are wired before Restore. type Zones struct { store ZoneStore @@ -71,18 +60,17 @@ type Zones struct { seen map[string]bool interval time.Duration client *http.Client - listURL string + dir *Directory boundsURL func(site string) (string, bool) regions []*zoneRegion - lists map[string]*zoneList retryAfter time.Time onChange func(ctx context.Context, iata string) onUpdate func(imported map[string]json.RawMessage) } -func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore) *Zones { - return &Zones{store: store, enabled: cfg.Enabled, seen: map[string]bool{}, interval: cfg.Interval(), listURL: ZonesURL, - boundsURL: boundaryEndpoint, lists: map[string]*zoneList{}, client: &http.Client{ +func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore, dir *Directory) *Zones { + return &Zones{store: store, enabled: cfg.Enabled, seen: map[string]bool{}, interval: cfg.Interval(), dir: dir, + boundsURL: boundaryEndpoint, client: &http.Client{ Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }} @@ -97,7 +85,7 @@ func (z *Zones) OnUpdate(fn func(imported map[string]json.RawMessage)) { z.onUpd // Restore prunes imports for unknown IATAs (all of them when disabled) and // loads saved boundaries, without making HTTP requests. func (z *Zones) Restore(ctx context.Context) error { - iatas, err := z.store.ListZoneIATAs(ctx) + iatas, err := z.store.ListKnownIATAs(ctx) if err != nil { return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) } @@ -166,11 +154,11 @@ func (z *Zones) Refresh(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() now := time.Now().UTC() - if now.Before(z.retryAfter) { + if now.Before(z.retryAfter) || now.Before(z.dir.pausedUntil()) { return nil } // IATAs are created from traffic, so pick up new ones every tick. - iatas, err := z.store.ListZoneIATAs(ctx) + iatas, err := z.store.ListKnownIATAs(ctx) if err != nil { return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) } @@ -179,67 +167,22 @@ func (z *Zones) Refresh(ctx context.Context) (err error) { if now.Before(r.b.NextAttempt) { continue } - list := z.lists[r.country] - if list == nil || now.Sub(list.fetchedAt) >= zoneListFresh { - if list != nil && now.Before(list.nextAttempt) { - continue - } - return z.fetchList(ctx, r.country, now) - } - return z.refresh(ctx, r, list, now) - } - return nil -} - -func (z *Zones) fetchList(ctx context.Context, country string, now time.Time) error { - list := z.lists[country] - if list == nil { - list = &zoneList{} - z.lists[country] = list - } - endpoint := z.listURL + "?country=" + url.QueryEscape(country) - etag := "" - if list.zones != nil { - etag = list.etag - } - status, body, header, err := z.get(ctx, endpoint, etag, MaxZoneList) - if err != nil { - return err - } - problem := "" - switch status { - case http.StatusOK: - zones, decodeErr := decodeZones(body, country) - if decodeErr != nil { - problem = "invalid response" - } else { - list.zones, list.etag = zones, header.Get("ETag") + zones, fetched, err := z.dir.List(ctx, r.country, now) + if err != nil || fetched { + return err } - case http.StatusNotModified: - if list.zones == nil { - problem = "304 without cached list" + if zones == nil { + continue } - default: - problem = z.statusProblem(status, header, now, &list.nextAttempt) - } - if problem == "" { - list.fetchedAt = now - } else if list.nextAttempt.Before(now.Add(zoneFailureRetry)) { - list.nextAttempt = now.Add(zoneFailureRetry) - } - level := slog.LevelInfo - if problem != "" { - level = slog.LevelWarn + return z.refresh(ctx, r, zones, now) } - slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country, - "zones", len(list.zones), "last_error", problem) return nil } -func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now time.Time) error { +func (z *Zones) refresh(ctx context.Context, r *zoneRegion, zones map[string]zoneEntry, now time.Time) error { update := Boundary{IATA: r.b.IATA, URL: r.b.URL, AttemptedAt: now, NextAttempt: now.Add(z.interval)} var retryAt time.Time - entry, listed := list.zones[r.b.IATA] + entry, listed := zones[r.b.IATA] endpoint, valid := "", false if listed { update.URL = entry.url @@ -257,7 +200,7 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now if r.b.Feature != nil { etag = r.b.ETag } - status, body, header, err := z.get(ctx, endpoint, etag, MaxBoundaryBody) + status, body, header, err := get(ctx, z.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxBoundaryBody) if err != nil { return err } @@ -277,7 +220,10 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now update.LastError = "304 without cached boundary" } default: - update.LastError = z.statusProblem(status, header, now, &retryAt) + update.LastError = statusProblem(status, header, now, &retryAt) + if until, ok := rateLimited(status, retryAt, now); ok { + z.retryAfter = until // the rate limit is shared by every endpoint + } } if update.LastError == "" { etag := header.Get("ETag") @@ -318,17 +264,17 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, list *zoneList, now } // get returns transport errors only when ctx ended; other failures become status 0. -func (z *Zones) get(ctx context.Context, endpoint, etag string, limit int64) (int, []byte, http.Header, error) { +func get(ctx context.Context, client *http.Client, agent, endpoint, etag string, limit int64) (int, []byte, http.Header, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return 0, nil, nil, err } request.Header.Set("Accept", "application/json") - request.Header.Set("User-Agent", "Beacon-MeshMapper-Zones/1") + request.Header.Set("User-Agent", agent) if etag != "" { request.Header.Set("If-None-Match", etag) } - response, err := z.client.Do(request) + response, err := client.Do(request) if err != nil { if ctx.Err() != nil { return 0, nil, nil, ctx.Err() @@ -346,8 +292,8 @@ func (z *Zones) get(ctx context.Context, endpoint, etag string, limit int64) (in return response.StatusCode, body, response.Header, nil } -// statusProblem describes a failed response and honours Retry-After on 429/503. -func (z *Zones) statusProblem(status int, header http.Header, now time.Time, retryAt *time.Time) string { +// statusProblem describes a failed response and reads Retry-After on 429/503. +func statusProblem(status int, header http.Header, now time.Time, retryAt *time.Time) string { if status == 0 { return "request failed" // don't persist untrusted error text } @@ -355,17 +301,21 @@ func (z *Zones) statusProblem(status int, header http.Header, now time.Time, ret if seconds, err := strconv.ParseInt(header.Get("Retry-After"), 10, 64); err == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) { *retryAt = now.Add(time.Duration(seconds) * time.Second) } - // The rate limit is shared by both endpoints, so pause every request. - if status == http.StatusTooManyRequests { - z.retryAfter = *retryAt - if retryAt.IsZero() { - z.retryAfter = now.Add(time.Minute) - } - } } return fmt.Sprintf("HTTP %d", status) } +// rateLimited returns how long a 429 pauses every request. +func rateLimited(status int, retryAt, now time.Time) (time.Time, bool) { + if status != http.StatusTooManyRequests { + return time.Time{}, false + } + if retryAt.IsZero() { + return now.Add(time.Minute), true + } + return retryAt, true +} + func (z *Zones) changed(ctx context.Context, iata string) { if z.onChange != nil { z.onChange(ctx, iata) @@ -394,36 +344,16 @@ func (z *Zones) log(b Boundary, action string) { "source", b.URL, "imported", b.Feature != nil, "checked_at", b.CheckedAt, "next_attempt", b.NextAttempt, "last_error", b.LastError) } -// boundaryEndpoint accepts only a published region site root. -func boundaryEndpoint(site string) (string, bool) { +func boundaryEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_geojson.php") } + +// siteEndpoint accepts only a published region site root. +func siteEndpoint(site, file string) (string, bool) { u, err := url.Parse(site) if err != nil || u.Scheme != "https" || u.User != nil || !zoneSite.MatchString(u.Host) || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { return "", false } - return "https://" + u.Host + "/get_geojson.php", true -} - -func decodeZones(body []byte, country string) (map[string]zoneEntry, error) { - var document struct { - Country string `json:"country"` - Zones *[]struct { - Code string `json:"code"` - URL string `json:"url"` - HasBoundary bool `json:"has_boundary"` - } `json:"zones"` - } - if err := json.Unmarshal(body, &document); err != nil { - return nil, err - } - if !strings.EqualFold(document.Country, country) || document.Zones == nil { - return nil, fmt.Errorf("invalid zone list") - } - zones := make(map[string]zoneEntry, len(*document.Zones)) - for _, zone := range *document.Zones { - zones[strings.ToUpper(zone.Code)] = zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary} - } - return zones, nil + return "https://" + u.Host + "/" + file, true } // decodeBoundary returns nil when MeshMapper has no usable outline for the region. diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go index 9798e3fe..f93152ee 100644 --- a/internal/meshmapper/zones_test.go +++ b/internal/meshmapper/zones_test.go @@ -31,7 +31,7 @@ type zoneMemoryStore struct { fail bool } -func (s *zoneMemoryStore) ListZoneIATAs(context.Context) ([]string, error) { +func (s *zoneMemoryStore) ListKnownIATAs(context.Context) ([]string, error) { return slices.Clone(s.iatas), nil } @@ -121,8 +121,9 @@ func newZoneHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, ena if store.iatas == nil { store.iatas = []string{"YOW"} } - h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, store) - h.z.listURL = f.URL + "/get_zones.php" + dir := NewDirectory() + dir.listURL = f.URL + "/get_zones.php" + h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, store, dir) h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", strings.HasPrefix(site, f.URL) } h.z.OnChange(func(_ context.Context, iata string) { h.changed = append(h.changed, iata) }) h.z.OnUpdate(func(imported map[string]json.RawMessage) { h.imported = imported }) @@ -239,9 +240,9 @@ func TestZonesListFailureBacksOff(t *testing.T) { t.Fatal("failed list retried early", f.listCalls, f.calls) } f.listStatus, f.list = 200, `{"country":"US","zones":[]}` - h.z.lists["CA"].nextAttempt, h.z.lists["CA"].fetchedAt = time.Time{}, time.Time{} + h.z.dir.lists["CA"].nextAttempt, h.z.dir.lists["CA"].fetchedAt = time.Time{}, time.Time{} _ = h.z.Refresh(context.Background()) - if h.z.lists["CA"].zones != nil { + if h.z.dir.lists["CA"].zones != nil { t.Fatal("list for another country accepted") } } From a970484b00ac195c4634e68dc4bd3d28ec9c13f9 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 22:51:08 -0400 Subject: [PATCH 117/141] feat(meshmapper): import zone groups as regions --- README.md | 1 + cmd/beacon/main.go | 1 + config.yaml.example | 9 ++ db/config.go | 14 ++- db/meshmapper.go | 33 +++++++ db/meshmapper_integration_test.go | 88 +++++++++++++++++ db/migrations/001_baseline.sql | 3 +- db/queries/queries.sql | 31 +++++- db/sqlc/mock/querier.go | 87 ++++++++++++++--- db/sqlc/models.go | 1 + db/sqlc/querier.go | 8 +- db/sqlc/queries.sql.go | 140 +++++++++++++++++++++++---- internal/cache/reader.go | 6 ++ internal/config/meshmapper.go | 4 + internal/config/meshmapper_test.go | 2 + internal/config/seed.go | 9 +- internal/config/seed_test.go | 4 +- internal/meshmapper/directory.go | 74 +++++++++++++-- internal/meshmapper/groups.go | 119 +++++++++++++++++++++++ internal/meshmapper/groups_test.go | 146 +++++++++++++++++++++++++++++ internal/meshmapper/zones.go | 23 ++++- internal/meshmapper/zones_test.go | 40 +++++++- 22 files changed, 785 insertions(+), 58 deletions(-) create mode 100644 internal/meshmapper/groups.go create mode 100644 internal/meshmapper/groups_test.go diff --git a/README.md b/README.md index 119c3291..8052707e 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,7 @@ even when the same bearer-authenticated request works with curl. # enabled: true # zones: # enabled: true +# import_groups: true # MeshMapper zone groups become regions # Optional IATA overrides — auto-created on first packet arrival, # only needed if you want to customise display name or coordinates. diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index c6f93a1b..fa4eab14 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -214,6 +214,7 @@ func main() { zones := meshmapper.NewZones(cfg.MeshMapper.Zones, store, directory) if cr, ok := reader.(*cache.CachedReader); ok { zones.OnChange(cr.InvalidateIATABorder) + zones.OnRegionsChange(cr.InvalidateRegions) } if localBorders != nil { zones.OnUpdate(func(imported map[string]json.RawMessage) { diff --git a/config.yaml.example b/config.yaml.example index cc863fc7..014c9d27 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -70,8 +70,15 @@ ratelimit: # A missing or failed boundary keeps the last good import, or the borderFile when # there is none. Disabling drops every import on restart. Logged under # component=meshmapper.zones. +# import_groups turns MeshMapper zone groups (e.g. ONQC: YOW, YQB, YUL) into +# regions, from the zone lists already fetched. A group is imported when at least +# one member is a known IATA, with its full member list; slug = lowercased group +# code, listed after the regions below. A region below with the same slug wins. +# Imported regions are removed when the group disappears or stops qualifying, +# and all of them when import_groups is turned off. Requires zones.enabled. # zones: # enabled: false +# import_groups: false # refresh_interval: 24h # 1h-168h; one request per 15s tick, ETag-conditional iatas: @@ -104,6 +111,8 @@ iatas: lat: 43.6777 lng: -79.6248 +# Hand-written regions; optional with meshmapper.zones.import_groups. Seeding +# sets each region's members to exactly this list. regions: - slug: western-canada name: Western Canada diff --git a/db/config.go b/db/config.go index ac15b6f8..3836a331 100644 --- a/db/config.go +++ b/db/config.go @@ -166,9 +166,13 @@ func (s *Store) GetRegionBySlug(ctx context.Context, slug string) (*api.Region, return &result, nil } -func (s *Store) UpsertRegionIATA(ctx context.Context, regionID int32, iata string) error { - return s.q.UpsertRegionIATA(ctx, sqlc.UpsertRegionIATAParams{ - RegionID: regionID, - Iata: iata, - }) +// SetRegionIATAs makes iatas the region's exact member list. +func (s *Store) SetRegionIATAs(ctx context.Context, regionID int32, iatas []string) error { + if iatas == nil { + iatas = []string{} + } + if err := s.q.DeleteRegionIATAsNotIn(ctx, sqlc.DeleteRegionIATAsNotInParams{RegionID: regionID, Keep: iatas}); err != nil { + return err + } + return s.q.AddRegionIATAs(ctx, sqlc.AddRegionIATAsParams{RegionID: regionID, Iatas: iatas}) } diff --git a/db/meshmapper.go b/db/meshmapper.go index 02e0de20..7b4f3cb3 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -113,3 +113,36 @@ func (s *Store) SaveZoneBoundary(ctx context.Context, b meshmapper.Boundary) err } return s.q.SaveZoneBoundary(ctx, params) } + +func (s *Store) ListRegionState(ctx context.Context) ([]meshmapper.RegionState, error) { + rows, err := s.q.ListRegionState(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.RegionState, 0, len(rows)) + for _, row := range rows { + out = append(out, meshmapper.RegionState{Slug: row.Slug, Name: row.Name, DisplayOrder: int(row.DisplayOrder), + Imported: row.Imported, IATAs: row.Iatas}) + } + return out, nil +} + +// SaveImportedRegion reports false when a hand-written region owns the slug. +func (s *Store) SaveImportedRegion(ctx context.Context, r meshmapper.RegionState) (bool, error) { + order := int32(r.DisplayOrder) + id, err := s.q.UpsertImportedRegion(ctx, sqlc.UpsertImportedRegionParams{Slug: r.Slug, Name: r.Name, DisplayOrder: &order}) + if errors.Is(err, pgx.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + return true, s.SetRegionIATAs(ctx, id, r.IATAs) +} + +func (s *Store) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { + if keep == nil { + keep = []string{} + } + return s.q.PruneImportedRegions(ctx, keep) +} diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go index e1589a6d..5db0161b 100644 --- a/db/meshmapper_integration_test.go +++ b/db/meshmapper_integration_test.go @@ -226,3 +226,91 @@ func TestMeshMapperZoneBoundariesPostgres(t *testing.T) { t.Fatal("manual border did not return after pruning") } } + +func TestMeshMapperRegionsPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"regions", "region_iatas"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + state := func() map[string]meshmapper.RegionState { + t.Helper() + rows, err := store.ListRegionState(ctx) + if err != nil { + t.Fatal(err) + } + out := map[string]meshmapper.RegionState{} + for _, r := range rows { + out[r.Slug] = r + } + return out + } + hand, err := store.UpsertRegion(ctx, "east", "East", "", 3, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := store.SetRegionIATAs(ctx, hand, []string{"YOW", "YUL"}); err != nil { + t.Fatal(err) + } + if err := store.SetRegionIATAs(ctx, hand, []string{"YOW"}); err != nil { + t.Fatal(err) + } + if got := state()["east"]; got.Imported || got.DisplayOrder != 3 || strings.Join(got.IATAs, ",") != "YOW" { + t.Fatal("configured members not replaced", got) + } + group := meshmapper.RegionState{Slug: "onqc", Name: "Corridor", DisplayOrder: 4, IATAs: []string{"YOW", "YQB"}} + if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved { + t.Fatal(saved, err) + } + group.IATAs = []string{"YQB", "YUL"} + if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved { + t.Fatal(saved, err) + } + if got := state()["onqc"]; !got.Imported || strings.Join(got.IATAs, ",") != "YQB,YUL" { + t.Fatal("imported region not updated", got) + } + if saved, err := store.SaveImportedRegion(ctx, meshmapper.RegionState{Slug: "east", Name: "Group", IATAs: []string{"YYZ"}}); err != nil || saved { + t.Fatal("imported group replaced a configured region", saved, err) + } + if got := state()["east"]; got.Name != "East" || strings.Join(got.IATAs, ",") != "YOW" { + t.Fatal("configured region changed by a clash", got) + } + if _, err := store.UpsertRegion(ctx, "onqc", "Mine", "", 1, nil, nil, nil); err != nil { + t.Fatal(err) + } + if state()["onqc"].Imported { + t.Fatal("config did not take over an imported slug") + } + if _, err := store.SaveImportedRegion(ctx, meshmapper.RegionState{Slug: "golm", Name: "Lakes", IATAs: []string{"YYZ"}}); err != nil { + t.Fatal(err) + } + if pruned, err := store.PruneImportedRegions(ctx, nil); err != nil || strings.Join(pruned, ",") != "golm" { + t.Fatal("prune touched configured regions", pruned, err) + } +} diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql index b858627c..ca09fe37 100644 --- a/db/migrations/001_baseline.sql +++ b/db/migrations/001_baseline.sql @@ -911,7 +911,8 @@ CREATE TABLE regions ( center_lng double precision, zoom_level integer DEFAULT 8, created_at timestamp with time zone DEFAULT now() NOT NULL, - updated_at timestamp with time zone DEFAULT now() NOT NULL + updated_at timestamp with time zone DEFAULT now() NOT NULL, + imported boolean DEFAULT false NOT NULL ); CREATE SEQUENCE regions_id_seq diff --git a/db/queries/queries.sql b/db/queries/queries.sql index f215ce06..902baca4 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1200,14 +1200,41 @@ ON CONFLICT (slug) DO UPDATE SET center_lat = EXCLUDED.center_lat, center_lng = EXCLUDED.center_lng, zoom_level = EXCLUDED.zoom_level, + imported = FALSE, -- config owns the slug from now on updated_at = NOW() RETURNING id; --- name: UpsertRegionIATA :exec +-- name: DeleteRegionIATAsNotIn :exec +DELETE FROM region_iatas WHERE region_id = @region_id AND NOT (iata = ANY(@keep::bpchar[])); + +-- name: AddRegionIATAs :exec INSERT INTO region_iatas (region_id, iata) -VALUES ($1, $2) +SELECT @region_id, unnest(@iatas::bpchar[]) ON CONFLICT (region_id, iata) DO NOTHING; +-- name: ListRegionState :many +-- Every region with its members, for reconciling imported MeshMapper groups. +SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, + COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas +FROM regions r +LEFT JOIN region_iatas ri ON ri.region_id = r.id +GROUP BY r.id +ORDER BY r.slug; + +-- name: UpsertImportedRegion :one +-- A hand-written region owns its slug: the WHERE turns a clash into no row. +INSERT INTO regions (slug, name, display_order, zoom_level, imported, updated_at) +VALUES (@slug, @name, @display_order, NULL, TRUE, NOW()) +ON CONFLICT (slug) DO UPDATE SET + name = EXCLUDED.name, + display_order = EXCLUDED.display_order, + updated_at = NOW() +WHERE regions.imported +RETURNING id; + +-- name: PruneImportedRegions :many +DELETE FROM regions WHERE imported AND NOT (slug = ANY(@keep::text[])) RETURNING slug; + -- ============================================================ -- TRACES -- ============================================================ diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 5ab8a156..12e4e2b7 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -44,6 +44,20 @@ func (m *MockQuerier) EXPECT() *MockQuerierMockRecorder { return m.recorder } +// AddRegionIATAs mocks base method. +func (m *MockQuerier) AddRegionIATAs(ctx context.Context, arg db.AddRegionIATAsParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddRegionIATAs", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddRegionIATAs indicates an expected call of AddRegionIATAs. +func (mr *MockQuerierMockRecorder) AddRegionIATAs(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddRegionIATAs", reflect.TypeOf((*MockQuerier)(nil).AddRegionIATAs), ctx, arg) +} + // AmbiguousPrefixes mocks base method. func (m *MockQuerier) AmbiguousPrefixes(ctx context.Context) ([]db.AmbiguousPrefixesRow, error) { m.ctrl.T.Helper() @@ -190,6 +204,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldTraceIATAs(ctx, lastHeard any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceIATAs", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceIATAs), ctx, lastHeard) } +// DeleteRegionIATAsNotIn mocks base method. +func (m *MockQuerier) DeleteRegionIATAsNotIn(ctx context.Context, arg db.DeleteRegionIATAsNotInParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRegionIATAsNotIn", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteRegionIATAsNotIn indicates an expected call of DeleteRegionIATAsNotIn. +func (mr *MockQuerierMockRecorder) DeleteRegionIATAsNotIn(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRegionIATAsNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteRegionIATAsNotIn), ctx, arg) +} + // GetAccount mocks base method. func (m *MockQuerier) GetAccount(ctx context.Context, id uuid.UUID) (db.Account, error) { m.ctrl.T.Helper() @@ -1224,6 +1252,21 @@ func (mr *MockQuerierMockRecorder) ListPacketsByIATAs(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPacketsByIATAs", reflect.TypeOf((*MockQuerier)(nil).ListPacketsByIATAs), ctx, arg) } +// ListRegionState mocks base method. +func (m *MockQuerier) ListRegionState(ctx context.Context) ([]db.ListRegionStateRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRegionState", ctx) + ret0, _ := ret[0].([]db.ListRegionStateRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRegionState indicates an expected call of ListRegionState. +func (mr *MockQuerierMockRecorder) ListRegionState(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegionState", reflect.TypeOf((*MockQuerier)(nil).ListRegionState), ctx) +} + // ListRegions mocks base method. func (m *MockQuerier) ListRegions(ctx context.Context) ([]db.ListRegionsRow, error) { m.ctrl.T.Helper() @@ -1314,6 +1357,21 @@ func (mr *MockQuerierMockRecorder) ListZoneBoundaries(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneBoundaries", reflect.TypeOf((*MockQuerier)(nil).ListZoneBoundaries), ctx) } +// PruneImportedRegions mocks base method. +func (m *MockQuerier) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PruneImportedRegions", ctx, keep) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PruneImportedRegions indicates an expected call of PruneImportedRegions. +func (mr *MockQuerierMockRecorder) PruneImportedRegions(ctx, keep any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PruneImportedRegions", reflect.TypeOf((*MockQuerier)(nil).PruneImportedRegions), ctx, keep) +} + // PruneZoneBoundaries mocks base method. func (m *MockQuerier) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { m.ctrl.T.Helper() @@ -1844,6 +1902,21 @@ func (mr *MockQuerierMockRecorder) UpsertIATADetails(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertIATADetails", reflect.TypeOf((*MockQuerier)(nil).UpsertIATADetails), ctx, arg) } +// UpsertImportedRegion mocks base method. +func (m *MockQuerier) UpsertImportedRegion(ctx context.Context, arg db.UpsertImportedRegionParams) (int32, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertImportedRegion", ctx, arg) + ret0, _ := ret[0].(int32) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpsertImportedRegion indicates an expected call of UpsertImportedRegion. +func (mr *MockQuerierMockRecorder) UpsertImportedRegion(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertImportedRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertImportedRegion), ctx, arg) +} + // UpsertKnownRoute mocks base method. func (m *MockQuerier) UpsertKnownRoute(ctx context.Context, arg db.UpsertKnownRouteParams) error { m.ctrl.T.Helper() @@ -1988,20 +2061,6 @@ func (mr *MockQuerierMockRecorder) UpsertRegion(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertRegion), ctx, arg) } -// UpsertRegionIATA mocks base method. -func (m *MockQuerier) UpsertRegionIATA(ctx context.Context, arg db.UpsertRegionIATAParams) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpsertRegionIATA", ctx, arg) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpsertRegionIATA indicates an expected call of UpsertRegionIATA. -func (mr *MockQuerierMockRecorder) UpsertRegionIATA(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegionIATA", reflect.TypeOf((*MockQuerier)(nil).UpsertRegionIATA), ctx, arg) -} - // UpsertTraceIATA mocks base method. func (m *MockQuerier) UpsertTraceIATA(ctx context.Context, arg db.UpsertTraceIATAParams) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index bf81a1d3..eba458ab 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -529,6 +529,7 @@ type Region struct { ZoomLevel *int32 `json:"zoom_level"` CreatedAt pgtype.Timestamptz `json:"created_at"` UpdatedAt pgtype.Timestamptz `json:"updated_at"` + Imported bool `json:"imported"` } type RegionIata struct { diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 1b3bbc99..f319a90b 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -13,6 +13,7 @@ import ( ) type Querier interface { + AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error // Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. AmbiguousPrefixes(ctx context.Context) ([]AmbiguousPrefixesRow, error) CreateAccount(ctx context.Context, name string) (Account, error) @@ -41,6 +42,7 @@ type Querier interface { DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error // Keeps the trace IATA filter in step with packet retention. DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error + DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error GetAccount(ctx context.Context, id uuid.UUID) (Account, error) GetChannelByID(ctx context.Context, id int32) (Channel, error) // Returns neighbors of a node that are in a different IATA. @@ -199,6 +201,8 @@ type Querier interface { // A site that filled scan_depth still has unread history below its floor. // The newest such floor is the point above which every site is covered. ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAsParams) ([]ListPacketsByIATAsRow, error) + // Every region with its members, for reconciling imported MeshMapper groups. + ListRegionState(ctx context.Context) ([]ListRegionStateRow, error) // ============================================================ // REGIONS // ============================================================ @@ -221,6 +225,7 @@ type Querier interface { // internal/ingest.BackfillChannelMessages. ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) + PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) // Drops imports for IATAs no longer configured, so their manual border returns. PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) // Delete node_neighbors where the neighbor has departed from node_short_ids @@ -302,6 +307,8 @@ type Querier interface { // bbox already computed -- see internal/config/border.go. UpsertIATABorder(ctx context.Context, arg UpsertIATABorderParams) error UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsParams) error + // A hand-written region owns its slug: the WHERE turns a clash into no row. + UpsertImportedRegion(ctx context.Context, arg UpsertImportedRegionParams) (int32, error) // ============================================================ // ROUTES // ============================================================ @@ -343,7 +350,6 @@ type Querier interface { // ============================================================ UpsertPacket(ctx context.Context, arg UpsertPacketParams) (UpsertPacketRow, error) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int32, error) - UpsertRegionIATA(ctx context.Context, arg UpsertRegionIATAParams) error // Refreshes at most hourly so repeat hears don't churn the row. UpsertTraceIATA(ctx context.Context, arg UpsertTraceIATAParams) error // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index bc833771..e16f105d 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -13,6 +13,22 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const addRegionIATAs = `-- name: AddRegionIATAs :exec +INSERT INTO region_iatas (region_id, iata) +SELECT $1, unnest($2::bpchar[]) +ON CONFLICT (region_id, iata) DO NOTHING +` + +type AddRegionIATAsParams struct { + RegionID int32 `json:"region_id"` + Iatas []string `json:"iatas"` +} + +func (q *Queries) AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error { + _, err := q.db.Exec(ctx, addRegionIATAs, arg.RegionID, arg.Iatas) + return err +} + const ambiguousPrefixes = `-- name: AmbiguousPrefixes :many SELECT iata::text AS iata, 1::int AS len, prefix_1 AS prefix FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 UNION ALL @@ -235,6 +251,20 @@ func (q *Queries) DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Time return err } +const deleteRegionIATAsNotIn = `-- name: DeleteRegionIATAsNotIn :exec +DELETE FROM region_iatas WHERE region_id = $1 AND NOT (iata = ANY($2::bpchar[])) +` + +type DeleteRegionIATAsNotInParams struct { + RegionID int32 `json:"region_id"` + Keep []string `json:"keep"` +} + +func (q *Queries) DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error { + _, err := q.db.Exec(ctx, deleteRegionIATAsNotIn, arg.RegionID, arg.Keep) + return err +} + const getAccount = `-- name: GetAccount :one SELECT id, name, created_at, deactivated_at FROM accounts WHERE id = $1 ` @@ -3687,6 +3717,50 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs return items, nil } +const listRegionState = `-- name: ListRegionState :many +SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, + COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas +FROM regions r +LEFT JOIN region_iatas ri ON ri.region_id = r.id +GROUP BY r.id +ORDER BY r.slug +` + +type ListRegionStateRow struct { + Slug string `json:"slug"` + Name string `json:"name"` + DisplayOrder int32 `json:"display_order"` + Imported bool `json:"imported"` + Iatas []string `json:"iatas"` +} + +// Every region with its members, for reconciling imported MeshMapper groups. +func (q *Queries) ListRegionState(ctx context.Context) ([]ListRegionStateRow, error) { + rows, err := q.db.Query(ctx, listRegionState) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListRegionStateRow{} + for rows.Next() { + var i ListRegionStateRow + if err := rows.Scan( + &i.Slug, + &i.Name, + &i.DisplayOrder, + &i.Imported, + &i.Iatas, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listRegions = `-- name: ListRegions :many SELECT id, slug, name @@ -3926,6 +4000,30 @@ func (q *Queries) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBound return items, nil } +const pruneImportedRegions = `-- name: PruneImportedRegions :many +DELETE FROM regions WHERE imported AND NOT (slug = ANY($1::text[])) RETURNING slug +` + +func (q *Queries) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { + rows, err := q.db.Query(ctx, pruneImportedRegions, keep) + if err != nil { + return nil, err + } + defer rows.Close() + items := []string{} + for rows.Next() { + var slug string + if err := rows.Scan(&slug); err != nil { + return nil, err + } + items = append(items, slug) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const pruneZoneBoundaries = `-- name: PruneZoneBoundaries :many DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY($1::text[])) RETURNING iata ` @@ -4941,6 +5039,31 @@ func (q *Queries) UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsPa return err } +const upsertImportedRegion = `-- name: UpsertImportedRegion :one +INSERT INTO regions (slug, name, display_order, zoom_level, imported, updated_at) +VALUES ($1, $2, $3, NULL, TRUE, NOW()) +ON CONFLICT (slug) DO UPDATE SET + name = EXCLUDED.name, + display_order = EXCLUDED.display_order, + updated_at = NOW() +WHERE regions.imported +RETURNING id +` + +type UpsertImportedRegionParams struct { + Slug string `json:"slug"` + Name string `json:"name"` + DisplayOrder *int32 `json:"display_order"` +} + +// A hand-written region owns its slug: the WHERE turns a clash into no row. +func (q *Queries) UpsertImportedRegion(ctx context.Context, arg UpsertImportedRegionParams) (int32, error) { + row := q.db.QueryRow(ctx, upsertImportedRegion, arg.Slug, arg.Name, arg.DisplayOrder) + var id int32 + err := row.Scan(&id) + return id, err +} + const upsertKnownRoute = `-- name: UpsertKnownRoute :exec INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) @@ -5325,6 +5448,7 @@ ON CONFLICT (slug) DO UPDATE SET center_lat = EXCLUDED.center_lat, center_lng = EXCLUDED.center_lng, zoom_level = EXCLUDED.zoom_level, + imported = FALSE, -- config owns the slug from now on updated_at = NOW() RETURNING id ` @@ -5354,22 +5478,6 @@ func (q *Queries) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int return id, err } -const upsertRegionIATA = `-- name: UpsertRegionIATA :exec -INSERT INTO region_iatas (region_id, iata) -VALUES ($1, $2) -ON CONFLICT (region_id, iata) DO NOTHING -` - -type UpsertRegionIATAParams struct { - RegionID int32 `json:"region_id"` - Iata string `json:"iata"` -} - -func (q *Queries) UpsertRegionIATA(ctx context.Context, arg UpsertRegionIATAParams) error { - _, err := q.db.Exec(ctx, upsertRegionIATA, arg.RegionID, arg.Iata) - return err -} - const upsertTraceIATA = `-- name: UpsertTraceIATA :exec INSERT INTO trace_iatas (trace_tag, iata, last_heard) VALUES ($1, $2, $3) diff --git a/internal/cache/reader.go b/internal/cache/reader.go index cb49338f..737cd377 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -90,6 +90,12 @@ func (cr *CachedReader) InvalidateIATABorder(ctx context.Context, iata string) { cr.c.del(ctx, keyIATABorderPrefix+iata) } +// InvalidateRegions makes imported MeshMapper regions visible to every region read. +func (cr *CachedReader) InvalidateRegions(ctx context.Context) { + cr.c.del(ctx, keyRegions) + cr.c.delPrefix(ctx, keyRegionPrefix) +} + // InvalidateNode removes the cached entries for a node by UUID. // Should be called from the ingest path after a node upsert. func (cr *CachedReader) InvalidateNode(ctx context.Context, nodeID uuid.UUID) { diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go index d4fa6428..b1c70944 100644 --- a/internal/config/meshmapper.go +++ b/internal/config/meshmapper.go @@ -17,6 +17,7 @@ type MeshMapperConfig struct { // MeshMapperZonesConfig imports each known IATA's published boundary, overriding borderFile. type MeshMapperZonesConfig struct { Enabled bool `yaml:"enabled"` + ImportGroups bool `yaml:"import_groups"` // zone groups become regions RefreshInterval duration `yaml:"refresh_interval"` } @@ -43,6 +44,9 @@ func (c MeshMapperScopesConfig) Interval() time.Duration { } func (c *Config) validateMeshMapper() error { + if z := c.MeshMapper.Zones; z.ImportGroups && !z.Enabled { + return fmt.Errorf("meshmapper.zones.import_groups requires meshmapper.zones.enabled") + } if z := c.MeshMapper.Zones; z.Enabled { // The Zones API asks clients not to poll more than once an hour. if z.Interval() < time.Hour || z.Interval() > 7*24*time.Hour { diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go index 8cd3264d..43c8c22d 100644 --- a/internal/config/meshmapper_test.go +++ b/internal/config/meshmapper_test.go @@ -53,6 +53,8 @@ func TestMeshMapperZonesConfig(t *testing.T) { {"too often", valid + " refresh_interval: 59m\n", true, 0}, {"too rare", valid + " refresh_interval: 169h\n", true, 0}, {"no regions", "meshmapper:\n zones:\n enabled: true\n", false, 24 * time.Hour}, + {"groups", valid + " import_groups: true\n", false, 24 * time.Hour}, + {"groups without zones", "meshmapper:\n zones:\n import_groups: true\n", true, 0}, {"disabled", "meshmapper:\n zones:\n enabled: false\n", false, 24 * time.Hour}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/config/seed.go b/internal/config/seed.go index a4195730..86d32492 100644 --- a/internal/config/seed.go +++ b/internal/config/seed.go @@ -19,7 +19,7 @@ type Seeder interface { UpsertIATADetails(ctx context.Context, iata string, name string, lat, lng *float64) error UpsertIATABorder(ctx context.Context, iata string, border json.RawMessage) error UpsertRegion(ctx context.Context, slug, name, description string, displayOrder int, centerLat, centerLng *float64, zoomLevel *int) (int32, error) - UpsertRegionIATA(ctx context.Context, regionID int32, iata string) error + SetRegionIATAs(ctx context.Context, regionID int32, iatas []string) error UpsertTransportScope(ctx context.Context, name, displayName string, transportKey, keyFingerprint []byte) error } @@ -56,9 +56,10 @@ func Seed(ctx context.Context, cfg *Config, db Seeder) error { if err := db.UpsertIATA(ctx, iata); err != nil { return err } - if err := db.UpsertRegionIATA(ctx, id, iata); err != nil { - return err - } + } + // Config owns the member list, including when it takes over an imported slug. + if err := db.SetRegionIATAs(ctx, id, r.IATAs); err != nil { + return err } } // Transport Codes diff --git a/internal/config/seed_test.go b/internal/config/seed_test.go index 04c8e63a..6241390c 100644 --- a/internal/config/seed_test.go +++ b/internal/config/seed_test.go @@ -52,8 +52,8 @@ func (s *stubSeeder) UpsertRegion(_ context.Context, slug, _, _ string, _ int, _ return id, s.upsertErr } -func (s *stubSeeder) UpsertRegionIATA(_ context.Context, regionID int32, iata string) error { - s.regionIATAs[regionID] = append(s.regionIATAs[regionID], iata) +func (s *stubSeeder) SetRegionIATAs(_ context.Context, regionID int32, iatas []string) error { + s.regionIATAs[regionID] = iatas return s.upsertErr } diff --git a/internal/meshmapper/directory.go b/internal/meshmapper/directory.go index 0ead3474..c4a9d858 100644 --- a/internal/meshmapper/directory.go +++ b/internal/meshmapper/directory.go @@ -10,9 +10,12 @@ import ( "log/slog" "net/http" "net/url" + "regexp" + "slices" "strings" "sync" "time" + "unicode" ) type zoneEntry struct { @@ -20,8 +23,14 @@ type zoneEntry struct { hasBoundary bool } +type zoneGroup struct { + name string + members []string +} + type zoneList struct { zones map[string]zoneEntry + groups map[string]zoneGroup etag string fetchedAt, nextAttempt time.Time } @@ -33,6 +42,7 @@ type Directory struct { listURL string lists map[string]*zoneList retryAfter time.Time + version int // bumped whenever a list is replaced } func NewDirectory() *Directory { @@ -82,11 +92,12 @@ func (d *Directory) fetch(ctx context.Context, country string, now time.Time) er problem := "" switch status { case http.StatusOK: - zones, decodeErr := decodeZones(body, country) + zones, groups, decodeErr := decodeZones(body, country) if decodeErr != nil { problem = "invalid response" } else { - list.zones, list.etag = zones, header.Get("ETag") + list.zones, list.groups, list.etag = zones, groups, header.Get("ETag") + d.version++ } case http.StatusNotModified: if list.zones == nil { @@ -116,7 +127,36 @@ func (d *Directory) fetch(ctx context.Context, country string, now time.Time) er return nil } -func decodeZones(body []byte, country string) (map[string]zoneEntry, error) { +// groups merges every loaded list's groups by code. complete is false until each +// country has a list, so a cold start can't look like every group disappearing. +func (d *Directory) groups(countries []string) (merged map[string]zoneGroup, version int, complete bool) { + d.mu.Lock() + defer d.mu.Unlock() + merged = map[string]zoneGroup{} + for _, country := range countries { + list := d.lists[country] + if list == nil || list.zones == nil { + return nil, d.version, false + } + for code, g := range list.groups { + if have, ok := merged[code]; ok { + g.members = append(slices.Clone(have.members), g.members...) + slices.Sort(g.members) + g.members = slices.Compact(g.members) + g.name = have.name + } + merged[code] = g + } + } + return merged, d.version, true +} + +var ( + groupCode = regexp.MustCompile(`^[A-Z0-9]{2,16}$`) + memberIATA = regexp.MustCompile(`^[A-Z]{3}$`) +) + +func decodeZones(body []byte, country string) (map[string]zoneEntry, map[string]zoneGroup, error) { var document struct { Country string `json:"country"` Zones *[]struct { @@ -124,16 +164,38 @@ func decodeZones(body []byte, country string) (map[string]zoneEntry, error) { URL string `json:"url"` HasBoundary bool `json:"has_boundary"` } `json:"zones"` + Groups []struct { + Code string `json:"code"` + Name string `json:"name"` + Members []string `json:"members"` + } `json:"groups"` } if err := json.Unmarshal(body, &document); err != nil { - return nil, err + return nil, nil, err } if !strings.EqualFold(document.Country, country) || document.Zones == nil { - return nil, fmt.Errorf("invalid zone list") + return nil, nil, fmt.Errorf("invalid zone list") } zones := make(map[string]zoneEntry, len(*document.Zones)) for _, zone := range *document.Zones { zones[strings.ToUpper(zone.Code)] = zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary} } - return zones, nil + groups := map[string]zoneGroup{} + for _, g := range document.Groups { + code, name := strings.ToUpper(g.Code), strings.TrimSpace(g.Name) + if !groupCode.MatchString(code) || name == "" || len(name) > 128 || strings.ContainsFunc(name, unicode.IsControl) { + continue // a bad group never blocks the zone list + } + var members []string + for _, m := range g.Members { + if m = strings.ToUpper(m); memberIATA.MatchString(m) { + members = append(members, m) + } + } + slices.Sort(members) + if members = slices.Compact(members); len(members) > 0 { + groups[code] = zoneGroup{name: name, members: members} + } + } + return zones, groups, nil } diff --git a/internal/meshmapper/groups.go b/internal/meshmapper/groups.go new file mode 100644 index 00000000..464c06f4 --- /dev/null +++ b/internal/meshmapper/groups.go @@ -0,0 +1,119 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "cmp" + "context" + "fmt" + "log/slog" + "slices" + "strings" +) + +// RegionState is a stored region; imported ones are also what SaveImportedRegion writes. +type RegionState struct { + Slug, Name string + DisplayOrder int + Imported bool + IATAs []string +} + +// OnRegionsChange is told whenever the imported regions change, for cache invalidation. +func (z *Zones) OnRegionsChange(fn func(ctx context.Context)) { z.onRegions = fn } + +// pruneGroups drops every imported region when group import is off. +func (z *Zones) pruneGroups(ctx context.Context) error { + pruned, err := z.store.PruneImportedRegions(ctx, nil) + if err != nil { + return fmt.Errorf("prune MeshMapper regions: %w", err) + } + z.regionsRemoved(pruned) + if len(pruned) > 0 { + z.regionsChanged(ctx) + } + return nil +} + +// syncGroups reconciles imported regions with the loaded zone lists. It only runs +// once every tracked country has a list, and again when a list or the IATA set changes. +func (z *Zones) syncGroups(ctx context.Context, iatas []string) error { + if !z.importGroups { + return nil + } + var countries []string + for _, r := range z.regions { + countries = append(countries, r.country) + } + slices.Sort(countries) + groups, version, complete := z.dir.groups(slices.Compact(countries)) + if !complete || (z.groupsSynced && version == z.groupsVersion && len(iatas) == z.groupsKnown) { + return nil + } + known := make(map[string]bool, len(iatas)) + for _, iata := range iatas { + known[iata] = true + } + state, err := z.store.ListRegionState(ctx) + if err != nil { + return fmt.Errorf("list regions for MeshMapper groups: %w", err) + } + stored, handOrder := map[string]RegionState{}, 0 + for _, r := range state { + stored[r.Slug] = r + if !r.Imported { + handOrder = max(handOrder, r.DisplayOrder) + } + } + var want []RegionState + for code, g := range groups { + if slices.ContainsFunc(g.members, func(m string) bool { return known[m] }) { + want = append(want, RegionState{Slug: strings.ToLower(code), Name: g.name, Imported: true, IATAs: g.members}) + } + } + slices.SortFunc(want, func(a, b RegionState) int { return cmp.Or(cmp.Compare(a.Name, b.Name), cmp.Compare(a.Slug, b.Slug)) }) + keep, changed := []string{}, false + for n, r := range want { + r.DisplayOrder = handOrder + 1 + n + if cur, ok := stored[r.Slug]; ok && !cur.Imported { + slog.Info("MeshMapper group skipped: a configured region uses its slug", "component", "meshmapper.zones", "slug", r.Slug) + continue + } else if ok && cur.Name == r.Name && cur.DisplayOrder == r.DisplayOrder && slices.Equal(cur.IATAs, r.IATAs) { + keep = append(keep, r.Slug) + continue + } + saved, err := z.store.SaveImportedRegion(ctx, r) + if err != nil { + return fmt.Errorf("save MeshMapper region %s: %w", r.Slug, err) + } + if !saved { + slog.Info("MeshMapper group skipped: a configured region uses its slug", "component", "meshmapper.zones", "slug", r.Slug) + continue + } + keep, changed = append(keep, r.Slug), true + slog.Info("MeshMapper region imported", "component", "meshmapper.zones", "slug", r.Slug, "name", r.Name, "iatas", r.IATAs) + } + pruned, err := z.store.PruneImportedRegions(ctx, keep) + if err != nil { + return fmt.Errorf("prune MeshMapper regions: %w", err) + } + z.regionsRemoved(pruned) + if changed || len(pruned) > 0 { + z.regionsChanged(ctx) + } + z.groupsSynced, z.groupsVersion, z.groupsKnown = true, version, len(iatas) + return nil +} + +func (z *Zones) regionsRemoved(slugs []string) { + for _, slug := range slugs { + slog.Info("MeshMapper region removed", "component", "meshmapper.zones", "slug", slug) + } +} + +func (z *Zones) regionsChanged(ctx context.Context) { + if z.onRegions != nil { + z.onRegions(ctx) + } +} diff --git a/internal/meshmapper/groups_test.go b/internal/meshmapper/groups_test.go new file mode 100644 index 00000000..81bd67a0 --- /dev/null +++ b/internal/meshmapper/groups_test.go @@ -0,0 +1,146 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" +) + +const groupedList = `{"country":"CA","zones":[{"code":"YOW","url":"URL/","has_boundary":true,"group":"ONQC"}],"groups":[` + + `{"code":"ONQC","name":"Ottawa-Quebec Corridor","members":["YUL","YOW","YQB"],"url":"https://onqc.meshmapper.net/"},` + + `{"code":"GOLM","name":"Great Ontario Lake Mesh","members":["YYZ","YKF"],"url":"https://golm.meshmapper.net/"}]}` + +type groupHarness struct { + z *Zones + f *fakeMeshMapper + store *zoneMemoryStore + invalidated int +} + +func newGroupHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, importGroups bool) *groupHarness { + t.Helper() + if store.iatas == nil { + store.iatas = []string{"YOW"} + } + if store.rows == nil { + store.rows = map[string]Boundary{} + } + h := &groupHarness{f: f, store: store} + dir := NewDirectory() + dir.listURL = f.URL + "/get_zones.php" + h.z = NewZones(config.MeshMapperZonesConfig{Enabled: true, ImportGroups: importGroups}, store, dir) + h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", true } + h.z.OnRegionsChange(func(context.Context) { h.invalidated++ }) + if err := h.z.Restore(context.Background()); err != nil { + t.Fatal(err) + } + return h +} + +func groupedFake(t *testing.T) *fakeMeshMapper { + f := newFakeMeshMapper(t) + f.list = strings.ReplaceAll(groupedList, "URL", f.URL) + return f +} + +// refetch makes the country list and YOW due, then runs the fetch and the sync tick. +func (h *groupHarness) refetch(t *testing.T) { + t.Helper() + if list := h.z.dir.lists["CA"]; list != nil { + list.fetchedAt, list.nextAttempt = time.Time{}, time.Time{} + } + for _, r := range h.z.regions { + r.b.NextAttempt = time.Time{} + } + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestGroupImportedWhenAMemberIsKnown(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + got, ok := h.store.regions["onqc"] + if !ok || got.Name != "Ottawa-Quebec Corridor" || !slices.Equal(got.IATAs, []string{"YOW", "YQB", "YUL"}) || got.DisplayOrder != 1 { + t.Fatalf("group not imported with full members: %+v", h.store.regions) + } + if _, ok := h.store.regions["golm"]; ok { + t.Fatal("group with no known member imported") + } + if h.invalidated != 1 { + t.Fatal("region caches not invalidated", h.invalidated) + } + h.refetch(t) + if h.invalidated != 1 { + t.Fatal("unchanged groups invalidated again", h.invalidated) + } +} + +func TestGroupSlugClashKeepsConfiguredRegion(t *testing.T) { + hand := RegionState{Slug: "onqc", Name: "Mine", DisplayOrder: 4, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{regions: map[string]RegionState{"onqc": hand}} + h := newGroupHarness(t, groupedFake(t), store, true) + h.refetch(t) + if got := store.regions["onqc"]; got.Imported || got.Name != "Mine" || !slices.Equal(got.IATAs, []string{"YOW"}) || h.invalidated != 0 { + t.Fatalf("configured region replaced: %+v", got) + } +} + +func TestGroupDroppedRemovesRegion(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + h.f.list = strings.ReplaceAll(`{"country":"CA","zones":[{"code":"YOW","url":"URL/","has_boundary":true,"group":null}],"groups":[]}`, "URL", h.f.URL) + h.refetch(t) + if len(h.store.regions) != 0 || h.invalidated != 2 { + t.Fatalf("dropped group kept: %+v invalidated=%d", h.store.regions, h.invalidated) + } +} + +func TestGroupImportOffRemovesImportedRegions(t *testing.T) { + hand := RegionState{Slug: "east", Name: "East", IATAs: []string{"YOW"}} + imported := RegionState{Slug: "onqc", Name: "Ottawa-Quebec Corridor", Imported: true, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{regions: map[string]RegionState{"east": hand, "onqc": imported}} + h := newGroupHarness(t, groupedFake(t), store, false) + if _, ok := store.regions["onqc"]; ok || h.invalidated != 1 { + t.Fatal("imported region kept with import_groups off") + } + if _, ok := store.regions["east"]; !ok { + t.Fatal("configured region removed") + } + h.refetch(t) + if len(store.regions) != 1 { + t.Fatal("groups imported with import_groups off", store.regions) + } +} + +func TestGroupFailedFetchKeepsGroups(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + h.f.listStatus = 500 + h.z.groupsSynced = false // force a reconcile against the retained list + h.refetch(t) + if _, ok := h.store.regions["onqc"]; !ok { + t.Fatal("failed fetch dropped imported region") + } +} + +func TestGroupColdStartWaitsForEveryList(t *testing.T) { + old := RegionState{Slug: "old", Name: "Old", Imported: true, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{regions: map[string]RegionState{"old": old}} + f := groupedFake(t) + f.listStatus = 500 + h := newGroupHarness(t, f, store, true) + h.refetch(t) + if _, ok := store.regions["old"]; !ok { + t.Fatal("imported regions pruned before any list loaded") + } +} diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go index 55d23985..b37b49b9 100644 --- a/internal/meshmapper/zones.go +++ b/internal/meshmapper/zones.go @@ -46,6 +46,9 @@ type ZoneStore interface { PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) ListZoneBoundaries(ctx context.Context) ([]Boundary, error) SaveZoneBoundary(ctx context.Context, b Boundary) error + ListRegionState(ctx context.Context) ([]RegionState, error) + SaveImportedRegion(ctx context.Context, r RegionState) (bool, error) + PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) } type zoneRegion struct { @@ -66,10 +69,15 @@ type Zones struct { retryAfter time.Time onChange func(ctx context.Context, iata string) onUpdate func(imported map[string]json.RawMessage) + + importGroups bool + groupsSynced bool + groupsVersion, groupsKnown int + onRegions func(ctx context.Context) } func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore, dir *Directory) *Zones { - return &Zones{store: store, enabled: cfg.Enabled, seen: map[string]bool{}, interval: cfg.Interval(), dir: dir, + return &Zones{store: store, enabled: cfg.Enabled, importGroups: cfg.Enabled && cfg.ImportGroups, seen: map[string]bool{}, interval: cfg.Interval(), dir: dir, boundsURL: boundaryEndpoint, client: &http.Client{ Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, @@ -82,9 +90,15 @@ func (z *Zones) OnChange(fn func(ctx context.Context, iata string)) { z.onChange // OnUpdate receives every imported boundary whenever the set changes. func (z *Zones) OnUpdate(fn func(imported map[string]json.RawMessage)) { z.onUpdate = fn } -// Restore prunes imports for unknown IATAs (all of them when disabled) and -// loads saved boundaries, without making HTTP requests. +// Restore prunes imports for unknown IATAs (all of them when disabled), and +// imported regions when group import is off, then loads saved boundaries, +// without making HTTP requests. func (z *Zones) Restore(ctx context.Context) error { + if !z.importGroups { + if err := z.pruneGroups(ctx); err != nil { + return err + } + } iatas, err := z.store.ListKnownIATAs(ctx) if err != nil { return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) @@ -163,6 +177,9 @@ func (z *Zones) Refresh(ctx context.Context) (err error) { return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) } z.track(iatas, nil) + if err := z.syncGroups(ctx, iatas); err != nil { + return err + } for _, r := range z.regions { if now.Before(r.b.NextAttempt) { continue diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go index f93152ee..6dc4cf6d 100644 --- a/internal/meshmapper/zones_test.go +++ b/internal/meshmapper/zones_test.go @@ -25,10 +25,42 @@ func boundaryBody(code, geometry string) string { } type zoneMemoryStore struct { - iatas []string - rows map[string]Boundary - pruned []string - fail bool + iatas []string + rows map[string]Boundary + pruned []string + fail bool + regions map[string]RegionState +} + +func (s *zoneMemoryStore) ListRegionState(context.Context) ([]RegionState, error) { + var out []RegionState + for _, r := range s.regions { + out = append(out, r) + } + return out, nil +} + +func (s *zoneMemoryStore) SaveImportedRegion(_ context.Context, r RegionState) (bool, error) { + if s.regions == nil { + s.regions = map[string]RegionState{} + } + if cur, ok := s.regions[r.Slug]; ok && !cur.Imported { + return false, nil + } + r.Imported = true + s.regions[r.Slug] = r + return true, nil +} + +func (s *zoneMemoryStore) PruneImportedRegions(_ context.Context, keep []string) ([]string, error) { + var removed []string + for slug, r := range s.regions { + if r.Imported && !slices.Contains(keep, slug) { + delete(s.regions, slug) + removed = append(removed, slug) + } + } + return removed, nil } func (s *zoneMemoryStore) ListKnownIATAs(context.Context) ([]string, error) { From 38b1a206b1f487f5b714ff371b6062fe09bd8e7e Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 22:56:04 -0400 Subject: [PATCH 118/141] fix(meshmapper): respect MeshMapper's per-endpoint rate limits --- cmd/beacon/main.go | 11 +++- config.yaml.example | 7 ++- db/meshmapper.go | 30 +++++++++++ db/meshmapper_integration_test.go | 20 +++++-- db/migrations/001_baseline.sql | 13 +++++ db/queries/queries.sql | 16 ++++++ db/sqlc/mock/querier.go | 29 +++++++++++ db/sqlc/models.go | 10 ++++ db/sqlc/querier.go | 3 ++ db/sqlc/queries.sql.go | 69 ++++++++++++++++++++++++ internal/config/meshmapper.go | 16 ++++-- internal/config/meshmapper_test.go | 8 ++- internal/meshmapper/directory.go | 74 ++++++++++++++++++-------- internal/meshmapper/groups_test.go | 2 +- internal/meshmapper/scopes.go | 34 +++++------- internal/meshmapper/scopes_test.go | 45 ++++++++-------- internal/meshmapper/zones.go | 45 +++++----------- internal/meshmapper/zones_test.go | 84 +++++++++++++++++++++++------- 18 files changed, 384 insertions(+), 132 deletions(-) diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index fa4eab14..2194636f 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -198,7 +198,16 @@ func main() { scopes.Load(scopeEntries) scopes.SetManualMembers(cfg.ManualScopeMembers()) slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") - directory := meshmapper.NewDirectory() + directory := meshmapper.NewDirectory(store) + if cfg.MeshMapper.Scopes.Enabled || cfg.MeshMapper.Zones.Enabled { + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + err = directory.Restore(restoreCtx) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper zone lists", "component", "startup", "error", err) + os.Exit(1) + } + } var scopeImporter *meshmapper.Importer if cfg.MeshMapper.Scopes.Enabled { restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) diff --git a/config.yaml.example b/config.yaml.example index 014c9d27..e315c9e8 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -61,7 +61,7 @@ ratelimit: #meshmapper: # scopes: # enabled: false -# refresh_interval: 1h # 5m-24h; one source checked per 15s tick +# refresh_interval: 1h # 1h-24h (MeshMapper allows one get_scopes.php per region per 55m); one source per 15s tick # # Region boundaries (Zones API). Every known IATA, including ones auto-created # from traffic, is looked up in its country's zone list, then its get_geojson.php @@ -70,6 +70,9 @@ ratelimit: # A missing or failed boundary keeps the last good import, or the borderFile when # there is none. Disabling drops every import on restart. Logged under # component=meshmapper.zones. +# Each country's zone list is fetched at most once a day (MeshMapper allows one +# get_zones.php per country per 23.5h) and saved, so restarts don't refetch it. +# Failed or rate-limited requests also wait out the limit, or Retry-After if longer. # import_groups turns MeshMapper zone groups (e.g. ONQC: YOW, YQB, YUL) into # regions, from the zone lists already fetched. A group is imported when at least # one member is a known IATA, with its full member list; slug = lowercased group @@ -79,7 +82,7 @@ ratelimit: # zones: # enabled: false # import_groups: false -# refresh_interval: 24h # 1h-168h; one request per 15s tick, ETag-conditional +# refresh_interval: 24h # 24h-168h (one get_geojson.php per region per 23.5h); one request per 15s tick, ETag-conditional iatas: YVR: diff --git a/db/meshmapper.go b/db/meshmapper.go index 7b4f3cb3..6ff3fa05 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -114,6 +114,36 @@ func (s *Store) SaveZoneBoundary(ctx context.Context, b meshmapper.Boundary) err return s.q.SaveZoneBoundary(ctx, params) } +func (s *Store) ListZoneLists(ctx context.Context) ([]meshmapper.ZoneList, error) { + rows, err := s.q.ListZoneLists(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.ZoneList, 0, len(rows)) + for _, row := range rows { + l := meshmapper.ZoneList{Country: row.Country, Payload: row.Payload, FetchedAt: row.FetchedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError} + if row.Etag != nil { + l.ETag = *row.Etag + } + out = append(out, l) + } + return out, nil +} + +func (s *Store) SaveZoneList(ctx context.Context, l meshmapper.ZoneList) error { + params := sqlc.SaveZoneListParams{ + Country: l.Country, Payload: l.Payload, LastError: l.LastError, + FetchedAt: pgtype.Timestamptz{Time: l.FetchedAt, Valid: !l.FetchedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: l.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: l.NextAttempt, Valid: true}, + } + if !l.FetchedAt.IsZero() { + params.Etag = &l.ETag + } + return s.q.SaveZoneList(ctx, params) +} + func (s *Store) ListRegionState(ctx context.Context) ([]meshmapper.RegionState, error) { rows, err := s.q.ListRegionState(ctx) if err != nil { diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go index 5db0161b..7dd8c504 100644 --- a/db/meshmapper_integration_test.go +++ b/db/meshmapper_integration_test.go @@ -82,7 +82,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { t.Fatal(row, err) } scopes := scopestore.New() - _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(), scopes, manualRows) + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(store), scopes, manualRows) if err != nil || len(scopes.Entries()) != 2 { t.Fatal("restart did not restore imported membership", err) } @@ -105,7 +105,7 @@ func TestMeshMapperCataloguePostgres(t *testing.T) { if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, nil); err != nil { t.Fatal(err) } - _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(), scopes, manualRows) + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, meshmapper.NewDirectory(store), scopes, manualRows) if err != nil || len(scopes.Entries()) != 1 { t.Fatal("removed imported membership remained active", err) } @@ -168,7 +168,7 @@ func TestMeshMapperZoneBoundariesPostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) - for _, table := range []string{"iata_codes", "meshmapper_zone_boundaries"} { + for _, table := range []string{"iata_codes", "meshmapper_zone_boundaries", "meshmapper_zone_lists"} { if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { t.Fatal(err) } @@ -225,6 +225,20 @@ func TestMeshMapperZoneBoundariesPostgres(t *testing.T) { if !strings.Contains(border("YOW"), `"file"`) { t.Fatal("manual border did not return after pruning") } + list := meshmapper.ZoneList{Country: "CA", Payload: []byte(`{"country":"CA","zones":[]}`), ETag: `"z1"`, + FetchedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)} + if err := store.SaveZoneList(ctx, list); err != nil { + t.Fatal(err) + } + later := now.Add(25 * time.Hour) + if err := store.SaveZoneList(ctx, meshmapper.ZoneList{Country: "CA", AttemptedAt: later, NextAttempt: later.Add(48 * time.Hour), LastError: "HTTP 429"}); err != nil { + t.Fatal(err) + } + lists, err := store.ListZoneLists(ctx) + if err != nil || len(lists) != 1 || len(lists[0].Payload) == 0 || lists[0].ETag != `"z1"` || !lists[0].FetchedAt.Equal(now) || + lists[0].LastError != "HTTP 429" || !lists[0].NextAttempt.Equal(later.Add(48*time.Hour)) { + t.Fatal("failed fetch lost the last good zone list", lists, err) + } } func TestMeshMapperRegionsPostgres(t *testing.T) { diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql index ca09fe37..515df569 100644 --- a/db/migrations/001_baseline.sql +++ b/db/migrations/001_baseline.sql @@ -665,6 +665,16 @@ CREATE TABLE meshmapper_zone_boundaries ( last_error text DEFAULT ''::text NOT NULL ); +CREATE TABLE meshmapper_zone_lists ( + country text NOT NULL, + payload jsonb, + etag text, + fetched_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS WITH combined AS ( SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats @@ -1023,6 +1033,9 @@ ALTER TABLE ONLY meshmapper_scope_catalogues ALTER TABLE ONLY meshmapper_zone_boundaries ADD CONSTRAINT meshmapper_zone_boundaries_pkey PRIMARY KEY (iata); +ALTER TABLE ONLY meshmapper_zone_lists + ADD CONSTRAINT meshmapper_zone_lists_pkey PRIMARY KEY (country); + ALTER TABLE ONLY node_iatas ADD CONSTRAINT node_iatas_pkey PRIMARY KEY (node_id, iata); diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 902baca4..2e999336 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1645,6 +1645,22 @@ ON CONFLICT (iata) DO UPDATE SET next_attempt = EXCLUDED.next_attempt, last_error = EXCLUDED.last_error; +-- name: ListZoneLists :many +SELECT * FROM meshmapper_zone_lists ORDER BY country; + +-- name: SaveZoneList :exec +-- NULL payload/etag/fetched_at retain the last good list after an error or 304. +INSERT INTO meshmapper_zone_lists (country, payload, etag, fetched_at, attempted_at, next_attempt, last_error) +VALUES (@country, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(fetched_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (country) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_zone_lists.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_lists.etag), + fetched_at = COALESCE(EXCLUDED.fetched_at, meshmapper_zone_lists.fetched_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; + -- name: PruneZoneBoundaries :many -- Drops imports for IATAs no longer configured, so their manual border returns. DELETE FROM meshmapper_zone_boundaries WHERE NOT (iata = ANY(@keep::text[])) RETURNING iata; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 12e4e2b7..81e4ff43 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -1357,6 +1357,21 @@ func (mr *MockQuerierMockRecorder) ListZoneBoundaries(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneBoundaries", reflect.TypeOf((*MockQuerier)(nil).ListZoneBoundaries), ctx) } +// ListZoneLists mocks base method. +func (m *MockQuerier) ListZoneLists(ctx context.Context) ([]db.MeshmapperZoneList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListZoneLists", ctx) + ret0, _ := ret[0].([]db.MeshmapperZoneList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListZoneLists indicates an expected call of ListZoneLists. +func (mr *MockQuerierMockRecorder) ListZoneLists(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneLists", reflect.TypeOf((*MockQuerier)(nil).ListZoneLists), ctx) +} + // PruneImportedRegions mocks base method. func (m *MockQuerier) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { m.ctrl.T.Helper() @@ -1674,6 +1689,20 @@ func (mr *MockQuerierMockRecorder) SaveZoneBoundary(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneBoundary", reflect.TypeOf((*MockQuerier)(nil).SaveZoneBoundary), ctx, arg) } +// SaveZoneList mocks base method. +func (m *MockQuerier) SaveZoneList(ctx context.Context, arg db.SaveZoneListParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveZoneList", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveZoneList indicates an expected call of SaveZoneList. +func (mr *MockQuerierMockRecorder) SaveZoneList(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveZoneList", reflect.TypeOf((*MockQuerier)(nil).SaveZoneList), ctx, arg) +} + // SearchKnownRoutes mocks base method. func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index eba458ab..5ddfc8bf 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -256,6 +256,16 @@ type MeshmapperZoneBoundary struct { LastError string `json:"last_error"` } +type MeshmapperZoneList struct { + Country string `json:"country"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + FetchedAt pgtype.Timestamptz `json:"fetched_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + type MvHourlyIataStat struct { Iata string `json:"iata"` Hour pgtype.Timestamptz `json:"hour"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index f319a90b..00d07da2 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -225,6 +225,7 @@ type Querier interface { // internal/ingest.BackfillChannelMessages. ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) + ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) // Drops imports for IATAs no longer configured, so their manual border returns. PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) @@ -270,6 +271,8 @@ type Querier interface { SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error // NULL feature/etag/checked_at retain the last good boundary after an error or 304. SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryParams) error + // NULL payload/etag/fetched_at retain the last good list after an error or 304. + SaveZoneList(ctx context.Context, arg SaveZoneListParams) error // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index e16f105d..d7ee3318 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -4000,6 +4000,38 @@ func (q *Queries) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBound return items, nil } +const listZoneLists = `-- name: ListZoneLists :many +SELECT country, payload, etag, fetched_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_lists ORDER BY country +` + +func (q *Queries) ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) { + rows, err := q.db.Query(ctx, listZoneLists) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperZoneList{} + for rows.Next() { + var i MeshmapperZoneList + if err := rows.Scan( + &i.Country, + &i.Payload, + &i.Etag, + &i.FetchedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const pruneImportedRegions = `-- name: PruneImportedRegions :many DELETE FROM regions WHERE imported AND NOT (slug = ANY($1::text[])) RETURNING slug ` @@ -4636,6 +4668,43 @@ func (q *Queries) SaveZoneBoundary(ctx context.Context, arg SaveZoneBoundaryPara return err } +const saveZoneList = `-- name: SaveZoneList :exec +INSERT INTO meshmapper_zone_lists (country, payload, etag, fetched_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2::jsonb, $3::text, + $4::timestamptz, $5, $6, $7) +ON CONFLICT (country) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_zone_lists.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_zone_lists.etag), + fetched_at = COALESCE(EXCLUDED.fetched_at, meshmapper_zone_lists.fetched_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveZoneListParams struct { + Country string `json:"country"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + FetchedAt pgtype.Timestamptz `json:"fetched_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +// NULL payload/etag/fetched_at retain the last good list after an error or 304. +func (q *Queries) SaveZoneList(ctx context.Context, arg SaveZoneListParams) error { + _, err := q.db.Exec(ctx, saveZoneList, + arg.Country, + arg.Payload, + arg.Etag, + arg.FetchedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + ) + return err +} + const searchKnownRoutes = `-- name: SearchKnownRoutes :many SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go index b1c70944..f76d3bde 100644 --- a/internal/config/meshmapper.go +++ b/internal/config/meshmapper.go @@ -9,6 +9,13 @@ import ( "time" ) +// MeshMapper rate-limits get_zones/get_geojson to one request per target per +// 23.5h and get_scopes to one per region per 55m; minimums round up. +const ( + MinZonesRefresh = 24 * time.Hour + MinScopesRefresh = time.Hour +) + type MeshMapperConfig struct { Scopes MeshMapperScopesConfig `yaml:"scopes"` Zones MeshMapperZonesConfig `yaml:"zones"` @@ -48,17 +55,16 @@ func (c *Config) validateMeshMapper() error { return fmt.Errorf("meshmapper.zones.import_groups requires meshmapper.zones.enabled") } if z := c.MeshMapper.Zones; z.Enabled { - // The Zones API asks clients not to poll more than once an hour. - if z.Interval() < time.Hour || z.Interval() > 7*24*time.Hour { - return fmt.Errorf("meshmapper.zones.refresh_interval must be between 1h and 168h") + if z.Interval() < MinZonesRefresh || z.Interval() > 7*24*time.Hour { + return fmt.Errorf("meshmapper.zones.refresh_interval must be between 24h and 168h") } } s := c.MeshMapper.Scopes if !s.Enabled { return nil } - if s.Interval() < 5*time.Minute || s.Interval() > 24*time.Hour { - return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 5m and 24h") + if s.Interval() < MinScopesRefresh || s.Interval() > 24*time.Hour { + return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 1h and 24h") } if len(s.Sources) > 0 { slog.Warn("meshmapper.scopes.sources is ignored; sources are discovered from the MeshMapper zone list", "component", "config") diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go index 43c8c22d..51279593 100644 --- a/internal/config/meshmapper_test.go +++ b/internal/config/meshmapper_test.go @@ -20,6 +20,8 @@ func TestMeshMapperConfig(t *testing.T) { {"valid", valid, false}, {"daily", valid + " refresh_interval: 24h\n", false}, {"legacy sources ignored", valid + " sources:\n yow: http://127.0.0.1/\n", false}, + {"hourly", valid + " refresh_interval: 1h\n", false}, + {"under the rate limit", valid + " refresh_interval: 55m\n", true}, {"fast", valid + " refresh_interval: 1s\n", true}, {"slow", valid + " refresh_interval: 25h\n", true}, {"negative", valid + " refresh_interval: -1h\n", true}, @@ -49,8 +51,10 @@ func TestMeshMapperZonesConfig(t *testing.T) { interval time.Duration }{ {"default", valid, false, 24 * time.Hour}, - {"hourly", valid + " refresh_interval: 1h\n", false, time.Hour}, - {"too often", valid + " refresh_interval: 59m\n", true, 0}, + {"daily", valid + " refresh_interval: 24h\n", false, 24 * time.Hour}, + {"weekly", valid + " refresh_interval: 168h\n", false, 168 * time.Hour}, + {"under the rate limit", valid + " refresh_interval: 23h30m\n", true, 0}, + {"hourly", valid + " refresh_interval: 1h\n", true, 0}, {"too rare", valid + " refresh_interval: 169h\n", true, 0}, {"no regions", "meshmapper:\n zones:\n enabled: true\n", false, 24 * time.Hour}, {"groups", valid + " import_groups: true\n", false, 24 * time.Hour}, diff --git a/internal/meshmapper/directory.go b/internal/meshmapper/directory.go index c4a9d858..e429b920 100644 --- a/internal/meshmapper/directory.go +++ b/internal/meshmapper/directory.go @@ -35,28 +35,54 @@ type zoneList struct { fetchedAt, nextAttempt time.Time } +// ZoneList is one country's saved get_zones.php response and fetch state. +type ZoneList struct { + Country string + Payload json.RawMessage + ETag string + FetchedAt, AttemptedAt, NextAttempt time.Time + LastError string +} + +type DirectoryStore interface { + ListZoneLists(ctx context.Context) ([]ZoneList, error) + SaveZoneList(ctx context.Context, l ZoneList) error +} + // Directory caches MeshMapper's per-country zone lists, shared by the zones and scopes imports. type Directory struct { - mu sync.Mutex - client *http.Client - listURL string - lists map[string]*zoneList - retryAfter time.Time - version int // bumped whenever a list is replaced + mu sync.Mutex + store DirectoryStore + client *http.Client + listURL string + lists map[string]*zoneList + version int // bumped whenever a list is replaced } -func NewDirectory() *Directory { - return &Directory{listURL: ZonesURL, lists: map[string]*zoneList{}, client: &http.Client{ +func NewDirectory(store DirectoryStore) *Directory { + return &Directory{store: store, listURL: ZonesURL, lists: map[string]*zoneList{}, client: &http.Client{ Timeout: 30 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }} } -// pausedUntil is when a list 429 stops pausing every MeshMapper request. -func (d *Directory) pausedUntil() time.Time { +// Restore loads saved lists so a restart doesn't spend each country's daily request. +func (d *Directory) Restore(ctx context.Context) error { + saved, err := d.store.ListZoneLists(ctx) + if err != nil { + return fmt.Errorf("restore MeshMapper zone lists: %w", err) + } d.mu.Lock() defer d.mu.Unlock() - return d.retryAfter + for _, l := range saved { + list := &zoneList{nextAttempt: l.NextAttempt} + if zones, groups, err := decodeZones(l.Payload, l.Country); err == nil { + list.zones, list.groups, list.etag, list.fetchedAt = zones, groups, l.ETag, l.FetchedAt + d.version++ + } + d.lists[l.Country] = list + } + return nil } // List returns the country's fresh zone list. fetched reports that this call @@ -68,7 +94,7 @@ func (d *Directory) List(ctx context.Context, country string, now time.Time) (zo if list != nil && list.zones != nil && now.Sub(list.fetchedAt) < zoneListFresh { return list.zones, false, nil } - if now.Before(d.retryAfter) || (list != nil && now.Before(list.nextAttempt)) { + if list != nil && now.Before(list.nextAttempt) { return nil, false, nil } return nil, true, d.fetch(ctx, country, now) @@ -90,6 +116,8 @@ func (d *Directory) fetch(ctx context.Context, country string, now time.Time) er return err } problem := "" + var payload json.RawMessage + var retryAt time.Time switch status { case http.StatusOK: zones, groups, decodeErr := decodeZones(body, country) @@ -97,6 +125,7 @@ func (d *Directory) fetch(ctx context.Context, country string, now time.Time) er problem = "invalid response" } else { list.zones, list.groups, list.etag = zones, groups, header.Get("ETag") + payload = body d.version++ } case http.StatusNotModified: @@ -104,26 +133,27 @@ func (d *Directory) fetch(ctx context.Context, country string, now time.Time) er problem = "304 without cached list" } default: - var retryAt time.Time problem = statusProblem(status, header, now, &retryAt) - if retryAt.After(list.nextAttempt) { - list.nextAttempt = retryAt - } - if until, ok := rateLimited(status, retryAt, now); ok { - d.retryAfter = until - } } + // Every request counts against the country's 23.5h allowance, failed or not. + list.nextAttempt = now.Add(zoneListFresh) + if retryAt.After(list.nextAttempt) { + list.nextAttempt = retryAt + } + saved := ZoneList{Country: country, Payload: payload, AttemptedAt: now, NextAttempt: list.nextAttempt, LastError: problem} if problem == "" { list.fetchedAt = now - } else if list.nextAttempt.Before(now.Add(zoneFailureRetry)) { - list.nextAttempt = now.Add(zoneFailureRetry) + saved.FetchedAt, saved.ETag = now, list.etag } level := slog.LevelInfo if problem != "" { level = slog.LevelWarn } slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country, - "zones", len(list.zones), "last_error", problem) + "zones", len(list.zones), "groups", len(list.groups), "next_attempt", list.nextAttempt, "last_error", problem) + if err := d.store.SaveZoneList(ctx, saved); err != nil { + return fmt.Errorf("persist MeshMapper zone list %s: %w", country, err) + } return nil } diff --git a/internal/meshmapper/groups_test.go b/internal/meshmapper/groups_test.go index 81bd67a0..e6328888 100644 --- a/internal/meshmapper/groups_test.go +++ b/internal/meshmapper/groups_test.go @@ -33,7 +33,7 @@ func newGroupHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, im store.rows = map[string]Boundary{} } h := &groupHarness{f: f, store: store} - dir := NewDirectory() + dir := NewDirectory(newZoneListMemory()) dir.listURL = f.URL + "/get_zones.php" h.z = NewZones(config.MeshMapperZonesConfig{Enabled: true, ImportGroups: importGroups}, store, dir) h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", true } diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index 907f4272..a4171be9 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -28,7 +28,7 @@ const ( MaxBody = 64 << 10 PollInterval = 15 * time.Second // one source per tick, at most four requests/minute - failureRetry = 5 * time.Minute // transient failures shouldn't cost a whole refresh interval + failureRetry = config.MinScopesRefresh // get_scopes.php allows one request per region per 55m ) // Cache stores source provenance and freshness separately from packet evidence. @@ -60,17 +60,16 @@ type source struct { // Importer is owned by one background task; ScopeStore synchronizes its consumers. type Importer struct { - store Store - scopes *scopestore.ScopeStore - manual []scopestore.Entry - dir *Directory - scopesURL func(site string) (string, bool) - seen map[string]bool - sources []source - interval time.Duration - client *http.Client - retryAfter time.Time - onChange func(context.Context) + store Store + scopes *scopestore.ScopeStore + manual []scopestore.Entry + dir *Directory + scopesURL func(site string) (string, bool) + seen map[string]bool + sources []source + interval time.Duration + client *http.Client + onChange func(context.Context) } // SetCacheInvalidator is wired once at startup, before the background task starts. @@ -129,9 +128,6 @@ func (i *Importer) track(iatas []string, saved map[string]Catalogue) { s.cache.LastError = "invalid saved catalogue; awaiting refresh" } } - if c.LastError == "HTTP 429" && c.NextAttempt.After(i.retryAfter) { - i.retryAfter = c.NextAttempt - } i.log(s, "restored") } i.sources = append(i.sources, s) @@ -149,9 +145,6 @@ func (i *Importer) Refresh(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, 15*time.Second) defer cancel() now := time.Now().UTC() - if now.Before(i.retryAfter) || now.Before(i.dir.pausedUntil()) { - return nil - } iatas, err := i.store.ListKnownIATAs(ctx) if err != nil { return fmt.Errorf("list IATAs for MeshMapper scopes: %w", err) @@ -253,15 +246,12 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error } } if update.LastError != "" { - // Transient failures shouldn't cost a whole refresh interval, but never beat a Retry-After. + // Failures retry before the full interval, but never sooner than the rate limit or a Retry-After. update.NextAttempt = now.Add(min(i.interval, failureRetry)) if retryAfter.After(update.NextAttempt) { update.NextAttempt = retryAfter } } - if update.LastError == "HTTP 429" { - i.retryAfter = update.NextAttempt - } if err := i.store.SaveScopeCatalogue(ctx, s.iata, s.url, update, entries); err != nil { // Avoid a retry every scheduler tick during a DB outage; never publish uncommitted keys. s.cache.NextAttempt = update.NextAttempt diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go index d12dfc0c..48104f5a 100644 --- a/internal/meshmapper/scopes_test.go +++ b/internal/meshmapper/scopes_test.go @@ -75,7 +75,7 @@ func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, ne // newImporter restores an importer for YOW whose source points at url. func newImporter(t *testing.T, cfg config.MeshMapperScopesConfig, store *memoryStore, url string, scopes *scopestore.ScopeStore, manual ...scopestore.Entry) *Importer { t.Helper() - imp, err := New(context.Background(), cfg, store, NewDirectory(), scopes, manual) + imp, err := New(context.Background(), cfg, store, NewDirectory(newZoneListMemory()), scopes, manual) if err != nil { t.Fatal(err) } @@ -154,7 +154,7 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { } status, after = 429, "7200" check() - if !imp.retryAfter.Equal(now.Add(2 * time.Hour)) { + if !imp.sources[0].cache.NextAttempt.Equal(now.Add(2 * time.Hour)) { t.Fatal("Retry-After ignored") } restored := scopestore.New() @@ -167,7 +167,7 @@ func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { t.Fatal(err) } if calls != before { - t.Fatal("restart ignored shared rate-limit cooldown") + t.Fatal("restart retried before Retry-After") } status, after, body = 200, "", catalogue() check() @@ -260,7 +260,7 @@ func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { store.rows[sourceKey{iata, iata}] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata)), AttemptedAt: time.Now()} } scopes := scopestore.New() - _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, NewDirectory(), scopes, nil) + _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true}, store, NewDirectory(newZoneListMemory()), scopes, nil) if err != nil || len(scopes.Entries()) != 1 || !reflect.DeepEqual(scopes.Entries()[0].IATAs, []string{"YOW", "YVR"}) { t.Fatal("latest saved catalogue per IATA not restored", scopes.Entries(), err) } @@ -269,7 +269,7 @@ func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { } // Disabled mode never calls the store. disabled := scopestore.New() - if _, err = New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(), disabled, nil); err != nil || len(disabled.Entries()) != 0 { + if _, err = New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(newZoneListMemory()), disabled, nil); err != nil || len(disabled.Entries()) != 0 { t.Fatal(err) } } @@ -300,20 +300,21 @@ func TestInvalidSavedCatalogueDoesNotBlockStartupOrRefresh(t *testing.T) { } } -func TestRetryAfterShorterThanIntervalIsHonoured(t *testing.T) { - ctx := context.Background() - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Retry-After", "1800") - w.WriteHeader(http.StatusTooManyRequests) - })) - defer server.Close() - imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) - now := time.Now().UTC() - if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { - t.Fatal(err) - } - if got := imp.sources[0].cache.NextAttempt.Sub(now); got != 30*time.Minute { - t.Fatalf("next attempt after 429 in %v, want %v (server Retry-After must not be overridden by the 1h interval)", got, 30*time.Minute) +func TestRetryAfterNeverBeatsTheRateLimit(t *testing.T) { + for retryAfter, want := range map[string]time.Duration{"1800": time.Hour, "7200": 2 * time.Hour} { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", retryAfter) + w.WriteHeader(http.StatusTooManyRequests) + })) + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, newMemoryStore("YOW"), server.URL, scopestore.New()) + now := time.Now().UTC() + if err := imp.refresh(context.Background(), &imp.sources[0], now); err != nil { + t.Fatal(err) + } + server.Close() + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != want { + t.Fatalf("Retry-After %ss: next attempt in %v, want %v", retryAfter, got, want) + } } } @@ -326,8 +327,8 @@ func TestFailedRefreshRetriesSoon(t *testing.T) { if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { t.Fatal(err) } - if got := imp.sources[0].cache.NextAttempt.Sub(now); got != failureRetry { - t.Fatalf("next attempt after failure in %v, want %v", got, failureRetry) + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != time.Hour { + t.Fatalf("next attempt after failure in %v, want 1h", got) } } @@ -359,7 +360,7 @@ func newFakeSites(t *testing.T) *fakeSites { func newDiscoveringImporter(t *testing.T, f *fakeSites, store *memoryStore, scopes *scopestore.ScopeStore) *Importer { t.Helper() - dir := NewDirectory() + dir := NewDirectory(newZoneListMemory()) dir.listURL = f.URL + "/get_zones.php" imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true}, store, dir, scopes, nil) if err != nil { diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go index b37b49b9..d14ab4cd 100644 --- a/internal/meshmapper/zones.go +++ b/internal/meshmapper/zones.go @@ -26,8 +26,8 @@ const ( MaxZoneList = 1 << 20 MaxBoundaryBody = 8 << 20 // outlines are never simplified - zoneListFresh = time.Hour // the API's own cache lifetime - zoneFailureRetry = time.Hour // fair use: no more than one poll an hour + zoneListFresh = config.MinZonesRefresh // get_zones.php allows one request per country per 23.5h + zoneFailureRetry = config.MinZonesRefresh // failed requests count too ) var zoneSite = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) @@ -58,17 +58,16 @@ type zoneRegion struct { // Zones is owned by one background task; listeners are wired before Restore. type Zones struct { - store ZoneStore - enabled bool - seen map[string]bool - interval time.Duration - client *http.Client - dir *Directory - boundsURL func(site string) (string, bool) - regions []*zoneRegion - retryAfter time.Time - onChange func(ctx context.Context, iata string) - onUpdate func(imported map[string]json.RawMessage) + store ZoneStore + enabled bool + seen map[string]bool + interval time.Duration + client *http.Client + dir *Directory + boundsURL func(site string) (string, bool) + regions []*zoneRegion + onChange func(ctx context.Context, iata string) + onUpdate func(imported map[string]json.RawMessage) importGroups bool groupsSynced bool @@ -147,9 +146,6 @@ func (z *Zones) track(iatas []string, saved map[string]Boundary) { if !ok { b = Boundary{IATA: iata} } - if b.LastError == "HTTP 429" && b.NextAttempt.After(z.retryAfter) { - z.retryAfter = b.NextAttempt - } z.regions = append(z.regions, &zoneRegion{country: country, b: b}) if ok { z.log(b, "restored") @@ -168,9 +164,6 @@ func (z *Zones) Refresh(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() now := time.Now().UTC() - if now.Before(z.retryAfter) || now.Before(z.dir.pausedUntil()) { - return nil - } // IATAs are created from traffic, so pick up new ones every tick. iatas, err := z.store.ListKnownIATAs(ctx) if err != nil { @@ -238,9 +231,6 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, zones map[string]zon } default: update.LastError = statusProblem(status, header, now, &retryAt) - if until, ok := rateLimited(status, retryAt, now); ok { - z.retryAfter = until // the rate limit is shared by every endpoint - } } if update.LastError == "" { etag := header.Get("ETag") @@ -322,17 +312,6 @@ func statusProblem(status int, header http.Header, now time.Time, retryAt *time. return fmt.Sprintf("HTTP %d", status) } -// rateLimited returns how long a 429 pauses every request. -func rateLimited(status int, retryAt, now time.Time) (time.Time, bool) { - if status != http.StatusTooManyRequests { - return time.Time{}, false - } - if retryAt.IsZero() { - return now.Add(time.Minute), true - } - return retryAt, true -} - func (z *Zones) changed(ctx context.Context, iata string) { if z.onChange != nil { z.onChange(ctx, iata) diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go index 6dc4cf6d..bf141208 100644 --- a/internal/meshmapper/zones_test.go +++ b/internal/meshmapper/zones_test.go @@ -30,6 +30,7 @@ type zoneMemoryStore struct { pruned []string fail bool regions map[string]RegionState + lists *zoneListMemory } func (s *zoneMemoryStore) ListRegionState(context.Context) ([]RegionState, error) { @@ -102,6 +103,30 @@ func (s *zoneMemoryStore) SaveZoneBoundary(_ context.Context, b Boundary) error return nil } +type zoneListMemory struct{ rows map[string]ZoneList } + +func newZoneListMemory() *zoneListMemory { return &zoneListMemory{rows: map[string]ZoneList{}} } + +func (s *zoneListMemory) ListZoneLists(context.Context) ([]ZoneList, error) { + var out []ZoneList + for _, l := range s.rows { + out = append(out, l) + } + return out, nil +} + +func (s *zoneListMemory) SaveZoneList(_ context.Context, l ZoneList) error { + old := s.rows[l.Country] + if l.Payload == nil { + l.Payload = old.Payload + } + if l.FetchedAt.IsZero() { + l.FetchedAt, l.ETag = old.FetchedAt, old.ETag + } + s.rows[l.Country] = l + return nil +} + type fakeMeshMapper struct { *httptest.Server list, boundary string @@ -153,8 +178,14 @@ func newZoneHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, ena if store.iatas == nil { store.iatas = []string{"YOW"} } - dir := NewDirectory() + if store.lists == nil { + store.lists = newZoneListMemory() + } + dir := NewDirectory(store.lists) dir.listURL = f.URL + "/get_zones.php" + if err := dir.Restore(context.Background()); err != nil { + t.Fatal(err) + } h.z = NewZones(config.MeshMapperZonesConfig{Enabled: enabled}, store, dir) h.z.boundsURL = func(site string) (string, bool) { return site + "get_geojson.php", strings.HasPrefix(site, f.URL) } h.z.OnChange(func(_ context.Context, iata string) { h.changed = append(h.changed, iata) }) @@ -216,11 +247,11 @@ func TestZonesKeepLastGoodBoundary(t *testing.T) { wantRetry time.Duration }{ {"null geometry", boundaryBody("YOW", "null"), 200, "no boundary", 24 * time.Hour}, - {"not found", "", 404, "HTTP 404", time.Hour}, - {"unavailable", "", 503, "HTTP 503", time.Hour}, - {"truncated", boundaryBody("YOW", square)[:80], 200, "invalid response", time.Hour}, - {"other region", boundaryBody("YVR", square), 200, "invalid response", time.Hour}, - {"oversized", `{"type":"FeatureCollection","features":[],"pad":"` + strings.Repeat("x", MaxBoundaryBody) + `"}`, 200, "invalid response", time.Hour}, + {"not found", "", 404, "HTTP 404", 24 * time.Hour}, + {"unavailable", "", 503, "HTTP 503", 24 * time.Hour}, + {"truncated", boundaryBody("YOW", square)[:80], 200, "invalid response", 24 * time.Hour}, + {"other region", boundaryBody("YVR", square), 200, "invalid response", 24 * time.Hour}, + {"oversized", `{"type":"FeatureCollection","features":[],"pad":"` + strings.Repeat("x", MaxBoundaryBody) + `"}`, 200, "invalid response", 24 * time.Hour}, } { t.Run(tc.name, func(t *testing.T) { f := newFakeMeshMapper(t) @@ -279,22 +310,37 @@ func TestZonesListFailureBacksOff(t *testing.T) { } } -func TestZonesRateLimitPausesEveryRequest(t *testing.T) { +func TestZonesRateLimitWaitsOnlyForThatRegion(t *testing.T) { + for retryAfter, want := range map[string]time.Duration{"120": 24 * time.Hour, "172800": 48 * time.Hour} { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + f.status, f.retryAfter = 429, retryAfter + b := h.tick(t) + if b.LastError != "HTTP 429" || b.NextAttempt.Sub(b.AttemptedAt) != want { + t.Fatalf("Retry-After %ss: next attempt in %v, want %v", retryAfter, b.NextAttempt.Sub(b.AttemptedAt), want) + } + restarted := newZoneHarness(t, f, h.store, true) + if err := restarted.z.Refresh(context.Background()); err != nil || f.calls != 1 { + t.Fatal("requested again before the rate limit allows", err, f.calls) + } + } +} + +func TestZoneListSurvivesRestart(t *testing.T) { f := newFakeMeshMapper(t) - h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) - f.status, f.retryAfter = 429, "120" - b := h.tick(t) - if b.LastError != "HTTP 429" || b.NextAttempt.Sub(b.AttemptedAt) != time.Hour || h.z.retryAfter.Sub(b.AttemptedAt) != 2*time.Minute { - t.Fatalf("429 not honoured: %+v retryAfter=%v", b, h.z.retryAfter) + store := &zoneMemoryStore{rows: map[string]Boundary{}} + h := newZoneHarness(t, f, store, true) + h.tick(t) + if f.listCalls != 1 { + t.Fatal("list not fetched", f.listCalls) } - h.z.regions[0].b.NextAttempt = time.Time{} - _ = h.z.Refresh(context.Background()) - if f.calls != 1 { - t.Fatal("requested during Retry-After") + if got := h.z.dir.lists["CA"].nextAttempt.Sub(h.z.dir.lists["CA"].fetchedAt); got != 24*time.Hour { + t.Fatal("zone list refetched sooner than the rate limit allows", got) } - restarted := newZoneHarness(t, f, h.store, true) - if !restarted.z.retryAfter.Equal(b.NextAttempt) { - t.Fatal("persisted 429 not restored") + restarted := newZoneHarness(t, f, store, true) + restarted.tick(t) + if f.listCalls != 1 || f.calls != 2 { + t.Fatal("restart refetched the zone list", f.listCalls, f.calls) } } From f8ee1ac8846187e990b80480516ccd4e1c82d838 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 23:09:40 -0400 Subject: [PATCH 119/141] feat(meshmapper): import regional channels and scope channel lists by region --- README.md | 7 +- cmd/beacon/main.go | 29 ++- config.yaml.example | 17 ++ db/channels.go | 25 +++ db/meshmapper.go | 52 +++++ db/meshmapper_integration_test.go | 106 +++++++++ db/migrations/001_baseline.sql | 31 +++ db/queries/queries.sql | 66 +++++- db/sqlc/mock/querier.go | 114 ++++++++++ db/sqlc/models.go | 21 ++ db/sqlc/querier.go | 16 +- db/sqlc/queries.sql.go | 193 ++++++++++++++++- docs/docs.go | 4 +- docs/swagger.json | 4 +- docs/swagger.yaml | 6 +- internal/api/handlers/channels.go | 4 +- internal/config/channels.go | 66 ++++++ internal/config/channels_test.go | 88 ++++++++ internal/config/config.go | 12 +- internal/config/meshmapper.go | 26 ++- internal/config/seed.go | 10 + internal/config/seed_test.go | 32 ++- internal/ingest/backfill.go | 15 +- internal/ingest/backfill_test.go | 17 ++ internal/ingest/ingest.go | 3 + internal/ingest/ingest_test.go | 7 + internal/keystore/keystore.go | 45 +++- internal/keystore/keystore_test.go | 30 +++ internal/meshmapper/channels.go | 307 +++++++++++++++++++++++++++ internal/meshmapper/channels_test.go | 234 ++++++++++++++++++++ 30 files changed, 1533 insertions(+), 54 deletions(-) create mode 100644 internal/config/channels.go create mode 100644 internal/config/channels_test.go create mode 100644 internal/meshmapper/channels.go create mode 100644 internal/meshmapper/channels_test.go diff --git a/README.md b/README.md index 8052707e..855b9b8a 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,8 @@ even when the same bearer-authenticated request works with curl. # zones: # enabled: true # import_groups: true # MeshMapper zone groups become regions +# channels: +# enabled: true # import each region's public hashtag channels # Optional IATA overrides — auto-created on first packet arrival, # only needed if you want to customise display name or coordinates. @@ -229,9 +231,12 @@ regions: channel_keys: # Hashtag channels: Beacon derives the PSK from the tag name automatically. # secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0] - # Tag names should be provided without the # prefix. + # Tag names should be provided without the # prefix. Plain names show under + # every region; {name, region: } limits a channel to one region. hashtags: - meshcore + - name: vancouver-mesh + region: western-canada # Explicit keys: channel hash (hex) and key (hex), with optional display name. # The public MeshCore channel key is included in config.yaml.example. diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 2194636f..4e4cf9a5 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -199,7 +199,7 @@ func main() { scopes.SetManualMembers(cfg.ManualScopeMembers()) slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") directory := meshmapper.NewDirectory(store) - if cfg.MeshMapper.Scopes.Enabled || cfg.MeshMapper.Zones.Enabled { + if cfg.MeshMapper.Scopes.Enabled || cfg.MeshMapper.Zones.Enabled || cfg.MeshMapper.Channels.Enabled { restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) err = directory.Restore(restoreCtx) cancelRestore() @@ -250,7 +250,8 @@ func main() { entries := make(map[string][]keystore.Entry) // Hashtag-derived channels: secret = SHA256("#tag")[:16], hash = SHA256(secret)[0] - for _, tag := range cfg.ChannelKeys.Hashtags { + for _, hashtag := range cfg.ChannelKeys.Hashtags { + tag := hashtag.Name secret, channelHash, fingerprint := keystore.DeriveHashtagKey(tag) hashHex := fmt.Sprintf("%02x", channelHash) entry := keystore.Entry{ @@ -285,6 +286,27 @@ func main() { keys := keystore.NewMapKeyStore(entries) + // Restored before the boot backfill so saved MeshMapper keys decrypt history too. + restoreCtx, cancelRestore = context.WithTimeout(ctx, 10*time.Second) + channelImporter, err := meshmapper.NewChannels(restoreCtx, cfg.MeshMapper.Channels, store, directory, keys) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper channels", "component", "startup", "error", err) + os.Exit(1) + } + channelImporter.OnNewKeys(func(hashes [][]byte) { + // Off the refresh tick: a backfill can outlast it, and history isn't urgent. + go func() { + backfillCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + if n, err := ingest.BackfillChannelHashes(backfillCtx, store, keys, hashes); err != nil { + slog.Error("MeshMapper channel backfill failed", "component", "meshmapper.channels", "error", err) + } else if n > 0 { + slog.Info(fmt.Sprintf("backfilled %d channel message(s) for imported keys", n), "component", "meshmapper.channels") + } + }() + }) + // ── Backfill channel messages ──────────────────────────────────────────── // Packets whose channel key wasn't yet configured at ingest time were stored as // hash-only channels and never decrypted. Retry them now against the keystore we just @@ -366,6 +388,9 @@ func main() { if cfg.MeshMapper.Zones.Enabled { tasks = append(tasks, background.Task{Name: "meshmapper.zones", Interval: meshmapper.PollInterval, Run: zones.Refresh}) } + if cfg.MeshMapper.Channels.Enabled { + tasks = append(tasks, background.Task{Name: "meshmapper.channels", Interval: meshmapper.PollInterval, Run: channelImporter.Refresh}) + } profiles := configureProfiling(ctx, pool) defer profiles.Stop() for i := range tasks { diff --git a/config.yaml.example b/config.yaml.example index e315c9e8..dce503d0 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -83,6 +83,18 @@ ratelimit: # enabled: false # import_groups: false # refresh_interval: 24h # 24h-168h (one get_geojson.php per region per 23.5h); one request per 15s tick, ETag-conditional +# +# Public hashtag channels (get_channels.php). Every known IATA that MeshMapper +# lists is looked up on its site, the same way as scopes. Each key and hash is +# checked against the name before use; a list with any mismatch is rejected and +# the last good one kept. Imported channels decrypt like configured ones (history +# is backfilled when a key first appears) and are listed under that IATA when +# /channels is filtered by region. Saved in PostgreSQL across restarts; failures +# wait out the 23.5h limit. Turning it off removes the region listing on restart. +# Logged under component=meshmapper.channels. +# channels: +# enabled: false +# refresh_interval: 24h # 24h-168h (one get_channels.php per region per 23.5h) iatas: YVR: @@ -138,8 +150,12 @@ channel_keys: # secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0] # Hashtag names should be provided without the # prefix. # Beacon prepends # automatically before deriving the key. + # A plain name is listed under every region; add region: (a configured + # region) to list it only there when /channels is filtered by region. hashtags: - meshcore + # - name: ottawa-mesh + # region: eastern-canada # Explicit keys: provide the channel hash (hex) and key (hex) directly. # An optional name can be set for display purposes. @@ -147,6 +163,7 @@ channel_keys: "11": key: "8b3387e9c5cdea6ac9e5edbaa115cd72" name: "Public" + # region: eastern-canada # optional, as for hashtags # Regional transport scopes for matching TRANSPORT_FLOOD packets. # Plain names have # prepended automatically (e.g. "bc" → "#bc"). diff --git a/db/channels.go b/db/channels.go index 9082e5ab..a4718b58 100644 --- a/db/channels.go +++ b/db/channels.go @@ -65,6 +65,31 @@ func (s *Store) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ingest.U return packets, nil } +// ListUndecryptedGroupTextPacketsByHash limits the boot backfill to channels that just gained a key. +func (s *Store) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ingest.UndecryptedPacket, error) { + rows, err := s.q.ListUndecryptedGroupTextPacketsByHash(ctx, hashes) + if err != nil { + return nil, err + } + packets := make([]ingest.UndecryptedPacket, 0, len(rows)) + for _, v := range rows { + packets = append(packets, ingest.UndecryptedPacket{PacketHash: v.PacketHash, RawPayload: v.RawPayload}) + } + return packets, nil +} + +// SetChannelConfigScopes replaces every configured channel's region placement; +// an empty region means Beacon-wide. +func (s *Store) SetChannelConfigScopes(ctx context.Context, fingerprints [][]byte, regions []string) error { + if err := s.q.DeleteChannelConfigScopes(ctx); err != nil { + return err + } + if len(fingerprints) == 0 { + return nil + } + return s.q.AddChannelConfigScopes(ctx, sqlc.AddChannelConfigScopesParams{Fingerprints: fingerprints, Regions: regions}) +} + func (s *Store) UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error { return s.q.UpsertChannelIATA(ctx, sqlc.UpsertChannelIATAParams{ ChannelHash: channelHash, diff --git a/db/meshmapper.go b/db/meshmapper.go index 6ff3fa05..09405138 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -176,3 +176,55 @@ func (s *Store) PruneImportedRegions(ctx context.Context, keep []string) ([]stri } return s.q.PruneImportedRegions(ctx, keep) } + +func (s *Store) ListChannelCatalogues(ctx context.Context) ([]meshmapper.Catalogue, error) { + rows, err := s.q.ListChannelCatalogues(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.Catalogue, 0, len(rows)) + for _, row := range rows { + c := meshmapper.Catalogue{IATA: row.Iata, URL: row.Url, Cache: meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, + AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError}} + if row.Etag != nil { + c.ETag = *row.Etag + } + out = append(out, c) + } + return out, nil +} + +// SaveChannelCatalogue records a fetch; a new payload also replaces the IATA's +// channel members, which are written first so a failure never outlives its snapshot. +func (s *Store) SaveChannelCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, fingerprints [][]byte) error { + if cache.Payload != nil { + if err := s.SetChannelMembers(ctx, iata, fingerprints); err != nil { + return err + } + } + params := sqlc.SaveChannelCatalogueParams{ + Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError, + CheckedAt: pgtype.Timestamptz{Time: cache.CheckedAt, Valid: !cache.CheckedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: cache.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: cache.NextAttempt, Valid: true}, + } + if !cache.CheckedAt.IsZero() { + params.Etag = &cache.ETag + } + return s.q.SaveChannelCatalogue(ctx, params) +} + +// SetChannelMembers makes fingerprints the IATA's exact MeshMapper channel list. +func (s *Store) SetChannelMembers(ctx context.Context, iata string, fingerprints [][]byte) error { + if fingerprints == nil { + fingerprints = [][]byte{} + } + if err := s.q.DeleteChannelMembersNotIn(ctx, sqlc.DeleteChannelMembersNotInParams{Iata: iata, Keep: fingerprints}); err != nil { + return err + } + return s.q.AddChannelMembers(ctx, sqlc.AddChannelMembersParams{Iata: iata, Fingerprints: fingerprints}) +} + +func (s *Store) ClearChannelMembers(ctx context.Context) error { + return s.q.DeleteAllChannelMembers(ctx) +} diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go index 7dd8c504..f80c0f7e 100644 --- a/db/meshmapper_integration_test.go +++ b/db/meshmapper_integration_test.go @@ -6,6 +6,7 @@ package db import ( "context" "os" + "slices" "strings" "testing" "time" @@ -328,3 +329,108 @@ func TestMeshMapperRegionsPostgres(t *testing.T) { t.Fatal("prune touched configured regions", pruned, err) } } + +func TestChannelRegionScopingPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"regions", "region_iatas", "channels", "channel_iatas", "channel_config_scopes", + "meshmapper_channel_members", "meshmapper_channel_catalogues"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + for slug, iata := range map[string]string{"east": "YOW", "west": "YYZ"} { + id, err := store.UpsertRegion(ctx, slug, slug, "", 0, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := store.SetRegionIATAs(ctx, id, []string{iata}); err != nil { + t.Fatal(err) + } + } + fp := func(b byte) []byte { return []byte{b, b, b, b, b, b, b, b} } + for name, b := range map[string]byte{"#mm-yow": 1, "#config-east": 2, "#global": 3, "#toronto": 4} { + if _, err := store.UpsertChannel(ctx, []byte{b}, fp(b), name, name[1:]); err != nil { + t.Fatal(err) + } + } + if err := store.UpsertChannelIATA(ctx, []byte{4}, "YOW", time.Now()); err != nil { + t.Fatal(err) + } + if err := store.SetChannelConfigScopes(ctx, [][]byte{fp(2), fp(3)}, []string{"east", ""}); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + for iata, b := range map[string]byte{"YOW": 1, "YYZ": 4} { + update := meshmapper.Cache{Payload: []byte(`{}`), CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(24 * time.Hour)} + if err := store.SaveChannelCatalogue(ctx, iata, "https://x.meshmapper.net/get_channels.php", update, [][]byte{fp(b)}); err != nil { + t.Fatal(err) + } + } + names := func(iatas ...string) string { + t.Helper() + page, err := store.ListChannels(ctx, 50, nil, iatas, 0, nil) + if err != nil { + t.Fatal(err) + } + var out []string + for _, c := range page.Items { + out = append(out, *c.Name) + } + slices.Sort(out) + return strings.Join(out, ",") + } + if got := names("YOW"); got != "#config-east,#global,#mm-yow" { + t.Fatal("YOW channels:", got) + } + if got := names("YYZ"); got != "#global,#toronto" { + t.Fatal("YYZ channels:", got) + } + if got := names(); got != "#config-east,#global,#mm-yow,#toronto" { + t.Fatal("unfiltered channels:", got) + } + // A 304 keeps membership; a new payload replaces it. + if err := store.SaveChannelCatalogue(ctx, "YOW", "https://x.meshmapper.net/get_channels.php", meshmapper.Cache{AttemptedAt: now, NextAttempt: now}, nil); err != nil { + t.Fatal(err) + } + if got := names("YOW"); got != "#config-east,#global,#mm-yow" { + t.Fatal("304 dropped membership:", got) + } + if err := store.SaveChannelCatalogue(ctx, "YOW", "https://x.meshmapper.net/get_channels.php", meshmapper.Cache{Payload: []byte(`{}`), AttemptedAt: now, NextAttempt: now}, [][]byte{}); err != nil { + t.Fatal(err) + } + if got := names("YOW"); got != "#config-east,#global" { + t.Fatal("empty list kept membership:", got) + } + cats, err := store.ListChannelCatalogues(ctx) + if err != nil || len(cats) != 2 { + t.Fatal(cats, err) + } + if err := store.ClearChannelMembers(ctx); err != nil || names("YYZ") != "#global" { + t.Fatal("members not cleared", err) + } +} diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql index 515df569..06e4335e 100644 --- a/db/migrations/001_baseline.sql +++ b/db/migrations/001_baseline.sql @@ -564,6 +564,11 @@ CREATE TABLE analytics_top_talkers_by_iata ( last_sent timestamp with time zone ); +CREATE TABLE channel_config_scopes ( + key_fingerprint bytea NOT NULL, + region_slug text +); + CREATE TABLE channel_iatas ( channel_hash bytea NOT NULL, iata character(3) NOT NULL, @@ -643,6 +648,22 @@ ALTER TABLE known_routes ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY ( CACHE 1 ); +CREATE TABLE meshmapper_channel_catalogues ( + iata text NOT NULL, + url text NOT NULL, + payload jsonb, + etag text, + checked_at timestamp with time zone, + attempted_at timestamp with time zone NOT NULL, + next_attempt timestamp with time zone NOT NULL, + last_error text DEFAULT ''::text NOT NULL +); + +CREATE TABLE meshmapper_channel_members ( + iata text NOT NULL, + key_fingerprint bytea NOT NULL +); + CREATE TABLE meshmapper_scope_catalogues ( iata text NOT NULL, url text NOT NULL, @@ -1027,6 +1048,12 @@ ALTER TABLE ONLY iata_codes ALTER TABLE ONLY known_routes ADD CONSTRAINT known_routes_pkey PRIMARY KEY (iata, path_key); +ALTER TABLE ONLY meshmapper_channel_catalogues + ADD CONSTRAINT meshmapper_channel_catalogues_pkey PRIMARY KEY (iata, url); + +ALTER TABLE ONLY meshmapper_channel_members + ADD CONSTRAINT meshmapper_channel_members_pkey PRIMARY KEY (iata, key_fingerprint); + ALTER TABLE ONLY meshmapper_scope_catalogues ADD CONSTRAINT meshmapper_scope_catalogues_pkey PRIMARY KEY (iata, url); @@ -1142,6 +1169,10 @@ CREATE INDEX idx_channel_messages_channel ON channel_messages USING btree (chann CREATE INDEX idx_channel_messages_sent_brin ON channel_messages USING brin (sent_at); +CREATE INDEX idx_channel_config_scopes_fingerprint ON channel_config_scopes USING btree (key_fingerprint); + +CREATE INDEX idx_meshmapper_channel_members_fingerprint ON meshmapper_channel_members USING btree (key_fingerprint); + CREATE INDEX idx_channels_hash ON channels USING btree (channel_hash); CREATE UNIQUE INDEX idx_channels_hash_no_key ON channels USING btree (channel_hash) WHERE (key_fingerprint IS NULL); diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 2e999336..addcddb1 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -863,6 +863,19 @@ RETURNING id; SELECT packet_hash, raw_payload FROM packets WHERE payload_type = 5 AND decrypted IS NOT TRUE; +-- name: ListUndecryptedGroupTextPacketsByHash :many +-- Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key. +SELECT packet_hash, raw_payload FROM packets +WHERE payload_type = 5 AND decrypted IS NOT TRUE AND channel_hash = ANY(@hashes::bytea[]); + +-- name: DeleteChannelConfigScopes :exec +DELETE FROM channel_config_scopes; + +-- name: AddChannelConfigScopes :exec +-- An empty region places the channel Beacon-wide. +INSERT INTO channel_config_scopes (key_fingerprint, region_slug) +SELECT unnest(@fingerprints::bytea[]), NULLIF(unnest(@regions::text[]), ''); + -- name: UpsertChannelIATA :exec -- Refreshes at most hourly so repeat hears don't churn the row. INSERT INTO channel_iatas (channel_hash, iata, last_heard) @@ -880,15 +893,19 @@ ON CONFLICT (trace_tag, iata) DO UPDATE SET WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour'; -- name: ListChannels :many --- Channels ordered by last seen, optionally filtered by hash and/or IATAs --- (membership via channel_iatas). NULL hash / empty array skip those filters. +-- Channels ordered by last seen, optionally filtered by hash and/or IATAs. +-- A channel belongs to an IATA when MeshMapper lists it there or config scopes it +-- to a region containing it (or Beacon-wide). NULL hash / empty array skip those filters. -- Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). SELECT c.* FROM channels c WHERE (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) - AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci - WHERE ci.iata = ANY(@iatas::bpchar[]) - )) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY(@iatas::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY(@iatas::bpchar[]))))) AND (@cursor_ts::timestamptz IS NULL OR c.last_seen < @cursor_ts) ORDER BY c.last_seen DESC, c.id DESC LIMIT @page_limit; @@ -899,9 +916,13 @@ LIMIT @page_limit; SELECT c.* FROM channels c WHERE (c.last_seen, c.id) < (@cursor_ts::timestamptz, @cursor_id::integer) AND (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) - AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci WHERE ci.iata = ANY(@iatas::bpchar[]) - )) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY(@iatas::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY(@iatas::bpchar[]))))) ORDER BY c.last_seen DESC, c.id DESC LIMIT @page_limit; @@ -1628,6 +1649,33 @@ ON CONFLICT (iata, url) DO UPDATE SET next_attempt = EXCLUDED.next_attempt, last_error = EXCLUDED.last_error; +-- name: ListChannelCatalogues :many +SELECT * FROM meshmapper_channel_catalogues ORDER BY iata, attempted_at; + +-- name: SaveChannelCatalogue :exec +-- NULL payload/etag/checked_at retain the last good list after an error or 304. +INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_channel_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_channel_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_channel_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; + +-- name: DeleteChannelMembersNotIn :exec +DELETE FROM meshmapper_channel_members WHERE iata = @iata AND NOT (key_fingerprint = ANY(@keep::bytea[])); + +-- name: AddChannelMembers :exec +INSERT INTO meshmapper_channel_members (iata, key_fingerprint) +SELECT @iata, unnest(@fingerprints::bytea[]) +ON CONFLICT (iata, key_fingerprint) DO NOTHING; + +-- name: DeleteAllChannelMembers :exec +DELETE FROM meshmapper_channel_members; + -- name: ListZoneBoundaries :many SELECT * FROM meshmapper_zone_boundaries ORDER BY iata; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 81e4ff43..7612b071 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -44,6 +44,34 @@ func (m *MockQuerier) EXPECT() *MockQuerierMockRecorder { return m.recorder } +// AddChannelConfigScopes mocks base method. +func (m *MockQuerier) AddChannelConfigScopes(ctx context.Context, arg db.AddChannelConfigScopesParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddChannelConfigScopes", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddChannelConfigScopes indicates an expected call of AddChannelConfigScopes. +func (mr *MockQuerierMockRecorder) AddChannelConfigScopes(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChannelConfigScopes", reflect.TypeOf((*MockQuerier)(nil).AddChannelConfigScopes), ctx, arg) +} + +// AddChannelMembers mocks base method. +func (m *MockQuerier) AddChannelMembers(ctx context.Context, arg db.AddChannelMembersParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddChannelMembers", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddChannelMembers indicates an expected call of AddChannelMembers. +func (mr *MockQuerierMockRecorder) AddChannelMembers(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChannelMembers", reflect.TypeOf((*MockQuerier)(nil).AddChannelMembers), ctx, arg) +} + // AddRegionIATAs mocks base method. func (m *MockQuerier) AddRegionIATAs(ctx context.Context, arg db.AddRegionIATAsParams) error { m.ctrl.T.Helper() @@ -103,6 +131,48 @@ func (mr *MockQuerierMockRecorder) DeactivateAccount(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeactivateAccount", reflect.TypeOf((*MockQuerier)(nil).DeactivateAccount), ctx, id) } +// DeleteAllChannelMembers mocks base method. +func (m *MockQuerier) DeleteAllChannelMembers(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteAllChannelMembers", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteAllChannelMembers indicates an expected call of DeleteAllChannelMembers. +func (mr *MockQuerierMockRecorder) DeleteAllChannelMembers(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAllChannelMembers", reflect.TypeOf((*MockQuerier)(nil).DeleteAllChannelMembers), ctx) +} + +// DeleteChannelConfigScopes mocks base method. +func (m *MockQuerier) DeleteChannelConfigScopes(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteChannelConfigScopes", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteChannelConfigScopes indicates an expected call of DeleteChannelConfigScopes. +func (mr *MockQuerierMockRecorder) DeleteChannelConfigScopes(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChannelConfigScopes", reflect.TypeOf((*MockQuerier)(nil).DeleteChannelConfigScopes), ctx) +} + +// DeleteChannelMembersNotIn mocks base method. +func (m *MockQuerier) DeleteChannelMembersNotIn(ctx context.Context, arg db.DeleteChannelMembersNotInParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteChannelMembersNotIn", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteChannelMembersNotIn indicates an expected call of DeleteChannelMembersNotIn. +func (mr *MockQuerierMockRecorder) DeleteChannelMembersNotIn(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChannelMembersNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteChannelMembersNotIn), ctx, arg) +} + // DeleteOldChannelIATAs mocks base method. func (m *MockQuerier) DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error { m.ctrl.T.Helper() @@ -1027,6 +1097,21 @@ func (mr *MockQuerierMockRecorder) ListAllChannelMessages(ctx, arg any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAllChannelMessages", reflect.TypeOf((*MockQuerier)(nil).ListAllChannelMessages), ctx, arg) } +// ListChannelCatalogues mocks base method. +func (m *MockQuerier) ListChannelCatalogues(ctx context.Context) ([]db.MeshmapperChannelCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListChannelCatalogues", ctx) + ret0, _ := ret[0].([]db.MeshmapperChannelCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListChannelCatalogues indicates an expected call of ListChannelCatalogues. +func (mr *MockQuerierMockRecorder) ListChannelCatalogues(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannelCatalogues", reflect.TypeOf((*MockQuerier)(nil).ListChannelCatalogues), ctx) +} + // ListChannelMessages mocks base method. func (m *MockQuerier) ListChannelMessages(ctx context.Context, arg db.ListChannelMessagesParams) ([]db.ListChannelMessagesRow, error) { m.ctrl.T.Helper() @@ -1342,6 +1427,21 @@ func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPackets(ctx any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPackets", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPackets), ctx) } +// ListUndecryptedGroupTextPacketsByHash mocks base method. +func (m *MockQuerier) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]db.ListUndecryptedGroupTextPacketsByHashRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListUndecryptedGroupTextPacketsByHash", ctx, hashes) + ret0, _ := ret[0].([]db.ListUndecryptedGroupTextPacketsByHashRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListUndecryptedGroupTextPacketsByHash indicates an expected call of ListUndecryptedGroupTextPacketsByHash. +func (mr *MockQuerierMockRecorder) ListUndecryptedGroupTextPacketsByHash(ctx, hashes any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListUndecryptedGroupTextPacketsByHash", reflect.TypeOf((*MockQuerier)(nil).ListUndecryptedGroupTextPacketsByHash), ctx, hashes) +} + // ListZoneBoundaries mocks base method. func (m *MockQuerier) ListZoneBoundaries(ctx context.Context) ([]db.MeshmapperZoneBoundary, error) { m.ctrl.T.Helper() @@ -1661,6 +1761,20 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } +// SaveChannelCatalogue mocks base method. +func (m *MockQuerier) SaveChannelCatalogue(ctx context.Context, arg db.SaveChannelCatalogueParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveChannelCatalogue", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveChannelCatalogue indicates an expected call of SaveChannelCatalogue. +func (mr *MockQuerierMockRecorder) SaveChannelCatalogue(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveChannelCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveChannelCatalogue), ctx, arg) +} + // SaveScopeCatalogue mocks base method. func (m *MockQuerier) SaveScopeCatalogue(ctx context.Context, arg db.SaveScopeCatalogueParams) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 5ddfc8bf..e08581cd 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -188,6 +188,11 @@ type Channel struct { MessageCount *int64 `json:"message_count"` } +type ChannelConfigScope struct { + KeyFingerprint []byte `json:"key_fingerprint"` + RegionSlug *string `json:"region_slug"` +} + type ChannelIata struct { ChannelHash []byte `json:"channel_hash"` Iata string `json:"iata"` @@ -234,6 +239,22 @@ type KnownRoute struct { LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"` } +type MeshmapperChannelCatalogue struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +type MeshmapperChannelMember struct { + Iata string `json:"iata"` + KeyFingerprint []byte `json:"key_fingerprint"` +} + type MeshmapperScopeCatalogue struct { Iata string `json:"iata"` Url string `json:"url"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 00d07da2..98290d7f 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -13,6 +13,9 @@ import ( ) type Querier interface { + // An empty region places the channel Beacon-wide. + AddChannelConfigScopes(ctx context.Context, arg AddChannelConfigScopesParams) error + AddChannelMembers(ctx context.Context, arg AddChannelMembersParams) error AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error // Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. AmbiguousPrefixes(ctx context.Context) ([]AmbiguousPrefixesRow, error) @@ -20,6 +23,9 @@ type Querier interface { // Lock the current row before deciding the outcome, including when another // deactivation commits while this statement is waiting for its row lock. DeactivateAccount(ctx context.Context, id uuid.UUID) (DeactivateAccountRow, error) + DeleteAllChannelMembers(ctx context.Context) error + DeleteChannelConfigScopes(ctx context.Context) error + DeleteChannelMembersNotIn(ctx context.Context, arg DeleteChannelMembersNotInParams) error // Keeps the channel IATA filter in step with packet retention. DeleteOldChannelIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error // Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- @@ -144,6 +150,7 @@ type Querier interface { // Pass empty string for iata or scope to skip those filters. // Pass cursor=0 to start from the beginning. ListAllChannelMessages(ctx context.Context, arg ListAllChannelMessagesParams) ([]ListAllChannelMessagesRow, error) + ListChannelCatalogues(ctx context.Context) ([]MeshmapperChannelCatalogue, error) // Returns messages for a channel identified by integer ID. // Pass a zero/null timestamp for since to return all messages up to limit. // Pass empty string for iata to skip IATA filtering. @@ -154,8 +161,9 @@ type Querier interface { // Pass empty string for iata or scope to skip those filters. // Pass cursor=0 to start from the beginning. ListChannelMessagesByHash(ctx context.Context, arg ListChannelMessagesByHashParams) ([]ListChannelMessagesByHashRow, error) - // Channels ordered by last seen, optionally filtered by hash and/or IATAs - // (membership via channel_iatas). NULL hash / empty array skip those filters. + // Channels ordered by last seen, optionally filtered by hash and/or IATAs. + // A channel belongs to an IATA when MeshMapper lists it there or config scopes it + // to a region containing it (or Beacon-wide). NULL hash / empty array skip those filters. // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) // Keep the non-null tuple boundary separate from the legacy optional cursor so @@ -224,6 +232,8 @@ type Querier interface { // to the config after they'd already been ingested -- see // internal/ingest.BackfillChannelMessages. ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUndecryptedGroupTextPacketsRow, error) + // Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key. + ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) @@ -265,6 +275,8 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + // NULL payload/etag/checked_at retain the last good list after an error or 304. + SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error // One statement commits the validated snapshot and its lookup identities together. // Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data // after an error or 304. Imported names never replace existing manual metadata. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index d7ee3318..d2264de9 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -13,6 +13,38 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const addChannelConfigScopes = `-- name: AddChannelConfigScopes :exec +INSERT INTO channel_config_scopes (key_fingerprint, region_slug) +SELECT unnest($1::bytea[]), NULLIF(unnest($2::text[]), '') +` + +type AddChannelConfigScopesParams struct { + Fingerprints [][]byte `json:"fingerprints"` + Regions []string `json:"regions"` +} + +// An empty region places the channel Beacon-wide. +func (q *Queries) AddChannelConfigScopes(ctx context.Context, arg AddChannelConfigScopesParams) error { + _, err := q.db.Exec(ctx, addChannelConfigScopes, arg.Fingerprints, arg.Regions) + return err +} + +const addChannelMembers = `-- name: AddChannelMembers :exec +INSERT INTO meshmapper_channel_members (iata, key_fingerprint) +SELECT $1, unnest($2::bytea[]) +ON CONFLICT (iata, key_fingerprint) DO NOTHING +` + +type AddChannelMembersParams struct { + Iata string `json:"iata"` + Fingerprints [][]byte `json:"fingerprints"` +} + +func (q *Queries) AddChannelMembers(ctx context.Context, arg AddChannelMembersParams) error { + _, err := q.db.Exec(ctx, addChannelMembers, arg.Iata, arg.Fingerprints) + return err +} + const addRegionIATAs = `-- name: AddRegionIATAs :exec INSERT INTO region_iatas (region_id, iata) SELECT $1, unnest($2::bpchar[]) @@ -110,6 +142,38 @@ func (q *Queries) DeactivateAccount(ctx context.Context, id uuid.UUID) (Deactiva return i, err } +const deleteAllChannelMembers = `-- name: DeleteAllChannelMembers :exec +DELETE FROM meshmapper_channel_members +` + +func (q *Queries) DeleteAllChannelMembers(ctx context.Context) error { + _, err := q.db.Exec(ctx, deleteAllChannelMembers) + return err +} + +const deleteChannelConfigScopes = `-- name: DeleteChannelConfigScopes :exec +DELETE FROM channel_config_scopes +` + +func (q *Queries) DeleteChannelConfigScopes(ctx context.Context) error { + _, err := q.db.Exec(ctx, deleteChannelConfigScopes) + return err +} + +const deleteChannelMembersNotIn = `-- name: DeleteChannelMembersNotIn :exec +DELETE FROM meshmapper_channel_members WHERE iata = $1 AND NOT (key_fingerprint = ANY($2::bytea[])) +` + +type DeleteChannelMembersNotInParams struct { + Iata string `json:"iata"` + Keep [][]byte `json:"keep"` +} + +func (q *Queries) DeleteChannelMembersNotIn(ctx context.Context, arg DeleteChannelMembersNotInParams) error { + _, err := q.db.Exec(ctx, deleteChannelMembersNotIn, arg.Iata, arg.Keep) + return err +} + const deleteOldChannelIATAs = `-- name: DeleteOldChannelIATAs :exec DELETE FROM channel_iatas WHERE last_heard < $1 ` @@ -2352,6 +2416,39 @@ func (q *Queries) ListAllChannelMessages(ctx context.Context, arg ListAllChannel return items, nil } +const listChannelCatalogues = `-- name: ListChannelCatalogues :many +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_channel_catalogues ORDER BY iata, attempted_at +` + +func (q *Queries) ListChannelCatalogues(ctx context.Context) ([]MeshmapperChannelCatalogue, error) { + rows, err := q.db.Query(ctx, listChannelCatalogues) + if err != nil { + return nil, err + } + defer rows.Close() + items := []MeshmapperChannelCatalogue{} + for rows.Next() { + var i MeshmapperChannelCatalogue + if err := rows.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listChannelMessages = `-- name: ListChannelMessages :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count @@ -2523,10 +2620,13 @@ func (q *Queries) ListChannelMessagesByHash(ctx context.Context, arg ListChannel const listChannels = `-- name: ListChannels :many SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c WHERE ($1::bytea IS NULL OR c.channel_hash = $1) - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci - WHERE ci.iata = ANY($2::bpchar[]) - )) + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY($2::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY($2::bpchar[]))))) AND ($3::timestamptz IS NULL OR c.last_seen < $3) ORDER BY c.last_seen DESC, c.id DESC LIMIT $4 @@ -2539,8 +2639,9 @@ type ListChannelsParams struct { PageLimit int32 `json:"page_limit"` } -// Channels ordered by last seen, optionally filtered by hash and/or IATAs -// (membership via channel_iatas). NULL hash / empty array skip those filters. +// Channels ordered by last seen, optionally filtered by hash and/or IATAs. +// A channel belongs to an IATA when MeshMapper lists it there or config scopes it +// to a region containing it (or Beacon-wide). NULL hash / empty array skip those filters. // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). func (q *Queries) ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) { rows, err := q.db.Query(ctx, listChannels, @@ -2583,9 +2684,13 @@ const listChannelsAfter = `-- name: ListChannelsAfter :many SELECT c.id, c.channel_hash, c.key_fingerprint, c.name, c.hashtag, c.is_hashtag, c.is_public, c.key_known, c.first_seen, c.last_seen, c.message_count FROM channels c WHERE (c.last_seen, c.id) < ($1::timestamptz, $2::integer) AND ($3::bytea IS NULL OR c.channel_hash = $3) - AND (COALESCE(cardinality($4::bpchar[]), 0) = 0 OR c.channel_hash IN ( - SELECT ci.channel_hash FROM channel_iatas ci WHERE ci.iata = ANY($4::bpchar[]) - )) + AND (COALESCE(cardinality($4::bpchar[]), 0) = 0 + OR EXISTS (SELECT 1 FROM meshmapper_channel_members m + WHERE m.key_fingerprint = c.key_fingerprint AND m.iata::bpchar = ANY($4::bpchar[])) + OR EXISTS (SELECT 1 FROM channel_config_scopes s + WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( + SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id + WHERE ri.iata = ANY($4::bpchar[]))))) ORDER BY c.last_seen DESC, c.id DESC LIMIT $5 ` @@ -3967,6 +4072,37 @@ func (q *Queries) ListUndecryptedGroupTextPackets(ctx context.Context) ([]ListUn return items, nil } +const listUndecryptedGroupTextPacketsByHash = `-- name: ListUndecryptedGroupTextPacketsByHash :many +SELECT packet_hash, raw_payload FROM packets +WHERE payload_type = 5 AND decrypted IS NOT TRUE AND channel_hash = ANY($1::bytea[]) +` + +type ListUndecryptedGroupTextPacketsByHashRow struct { + PacketHash []byte `json:"packet_hash"` + RawPayload []byte `json:"raw_payload"` +} + +// Like ListUndecryptedGroupTextPackets, limited to channels that just gained a key. +func (q *Queries) ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error) { + rows, err := q.db.Query(ctx, listUndecryptedGroupTextPacketsByHash, hashes) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListUndecryptedGroupTextPacketsByHashRow{} + for rows.Next() { + var i ListUndecryptedGroupTextPacketsByHashRow + if err := rows.Scan(&i.PacketHash, &i.RawPayload); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listZoneBoundaries = `-- name: ListZoneBoundaries :many SELECT iata, url, feature, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_zone_boundaries ORDER BY iata ` @@ -4574,6 +4710,45 @@ func (q *Queries) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashes return items, nil } +const saveChannelCatalogue = `-- name: SaveChannelCatalogue :exec +INSERT INTO meshmapper_channel_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_channel_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_channel_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_channel_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveChannelCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + +// NULL payload/etag/checked_at retain the last good list after an error or 304. +func (q *Queries) SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error { + _, err := q.db.Exec(ctx, saveChannelCatalogue, + arg.Iata, + arg.Url, + arg.Payload, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + ) + return err +} + const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec WITH inserted AS ( INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) diff --git a/docs/docs.go b/docs/docs.go index 45381385..6e59a488 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -597,13 +597,13 @@ const docTemplate = `{ }, { "type": "string", - "description": "Filter by IATA code", + "description": "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels", "name": "iata", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ", + "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata", "name": "iatas", "in": "query" }, diff --git a/docs/swagger.json b/docs/swagger.json index 4cd76556..843fd6b7 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -595,13 +595,13 @@ }, { "type": "string", - "description": "Filter by IATA code", + "description": "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels", "name": "iata", "in": "query" }, { "type": "string", - "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ", + "description": "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata", "name": "iatas", "in": "query" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 4ed5e382..dda333fc 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1903,11 +1903,13 @@ paths: in: query name: hash type: string - - description: Filter by IATA code + - description: 'Filter by IATA code: channels MeshMapper lists there, config + channels scoped to a region containing it, and Beacon-wide config channels' in: query name: iata type: string - - description: Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ + - description: Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; + same membership rule as iata in: query name: iatas type: string diff --git a/internal/api/handlers/channels.go b/internal/api/handlers/channels.go index 81ee792f..46f312de 100644 --- a/internal/api/handlers/channels.go +++ b/internal/api/handlers/channels.go @@ -35,8 +35,8 @@ func ChannelsRouter(reader api.Reader) http.Handler { // @Tags Channels // @Produce json // @Param hash query string false "Single-byte channel hash (hex)" -// @Param iata query string false "Filter by IATA code" -// @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ" +// @Param iata query string false "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels" +// @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata" // @Param cursor query int false "last_seen epoch ms of last item for pagination; 0 starts from the beginning" // @Param pageCursor query string false "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed." // @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) diff --git a/internal/config/channels.go b/internal/config/channels.go new file mode 100644 index 00000000..09216c30 --- /dev/null +++ b/internal/config/channels.go @@ -0,0 +1,66 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "encoding/hex" + "fmt" + "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "gopkg.in/yaml.v3" +) + +// HashtagConfig is a hashtag channel; a plain string in YAML is a Beacon-wide one. +type HashtagConfig struct { + Name string `yaml:"name"` + Region string `yaml:"region"` // optional region slug +} + +func (h *HashtagConfig) UnmarshalYAML(node *yaml.Node) error { + if node.Kind == yaml.ScalarNode { + return node.Decode(&h.Name) + } + type plain HashtagConfig + return node.Decode((*plain)(h)) +} + +// ChannelScope places a configured channel under a region, or Beacon-wide when Region is empty. +type ChannelScope struct { + Fingerprint []byte + Region string +} + +func (c *Config) validateChannelKeys() error { + for i, h := range c.ChannelKeys.Hashtags { + if strings.TrimSpace(h.Name) == "" { + return fmt.Errorf("channel_keys.hashtags[%d] needs a name", i) + } + if h.Region != "" && c.region(h.Region) == nil { + return fmt.Errorf("channel #%s region %q is not a configured region", h.Name, h.Region) + } + } + for hash, k := range c.ChannelKeys.Keys { + if k.Region != "" && c.region(k.Region) == nil { + return fmt.Errorf("channel key %s region %q is not a configured region", hash, k.Region) + } + } + return nil +} + +// ChannelScopes lists every configured channel's region placement. Keys that +// aren't valid hex are skipped; startup already warns about them. +func (c *Config) ChannelScopes() []ChannelScope { + var scopes []ChannelScope + for _, h := range c.ChannelKeys.Hashtags { + _, _, fingerprint := keystore.DeriveHashtagKey(h.Name) + scopes = append(scopes, ChannelScope{Fingerprint: fingerprint, Region: h.Region}) + } + for _, k := range c.ChannelKeys.Keys { + if key, err := hex.DecodeString(k.Key); err == nil { + scopes = append(scopes, ChannelScope{Fingerprint: keystore.Fingerprint(key), Region: k.Region}) + } + } + return scopes +} diff --git a/internal/config/channels_test.go b/internal/config/channels_test.go new file mode 100644 index 00000000..48656708 --- /dev/null +++ b/internal/config/channels_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" +) + +func loadText(t *testing.T, text string) (*Config, error) { + t.Helper() + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(text), 0600); err != nil { + t.Fatal(err) + } + return Load(p) +} + +func TestChannelKeysRegionScoping(t *testing.T) { + text := `regions: + - slug: east + iatas: [YOW] +channel_keys: + hashtags: + - meshcore + - name: ottawa-mesh + region: east + keys: + "11": + key: "8b3387e9c5cdea6ac9e5edbaa115cd72" + name: Public +` + cfg, err := loadText(t, text) + if err != nil { + t.Fatal(err) + } + if got := cfg.ChannelKeys.Hashtags; len(got) != 2 || got[0].Name != "meshcore" || got[0].Region != "" || got[1].Name != "ottawa-mesh" || got[1].Region != "east" { + t.Fatalf("hashtags parsed as %+v", got) + } + scopes := cfg.ChannelScopes() + if len(scopes) != 3 { + t.Fatalf("scopes: %+v", scopes) + } + _, _, ottawa := keystore.DeriveHashtagKey("ottawa-mesh") + found := false + for _, s := range scopes { + if bytes.Equal(s.Fingerprint, ottawa) { + found = s.Region == "east" + } + } + if !found { + t.Fatal("region-scoped hashtag missing its region", scopes) + } + + for name, bad := range map[string]string{ + "unknown hashtag region": strings.Replace(text, "region: east", "region: west", 1), + "empty hashtag": strings.Replace(text, "- meshcore", `- ""`, 1), + "unknown key region": text + " region: west\n", + } { + if _, err := loadText(t, bad); err == nil { + t.Errorf("%s accepted", name) + } + } +} + +func TestMeshMapperChannelsConfig(t *testing.T) { + valid := "meshmapper:\n channels:\n enabled: true\n" + for text, bad := range map[string]bool{ + valid: false, + valid + " refresh_interval: 24h\n": false, + valid + " refresh_interval: 168h\n": false, + valid + " refresh_interval: 23h\n": true, + valid + " refresh_interval: 169h\n": true, + } { + if _, err := loadText(t, text); (err != nil) != bad { + t.Errorf("%q: error=%v", text, err) + } + } + if (MeshMapperChannelsConfig{}).Interval() != MinChannelsRefresh { + t.Fatal("wrong default") + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 95ebfcbb..65abd77a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -320,9 +320,9 @@ func (d *duration) UnmarshalYAML(value *yaml.Node) error { // channel_hash = SHA256(secret)[0]. Explicit keys are provided as hex strings // keyed by the channel hash hex (e.g. "11" for 0x11). type ChannelKeysConfig struct { - // Hashtags is a list of hashtag names (without the # prefix). + // Hashtags lists hashtag names (without the # prefix), each optionally scoped to a region. // Beacon derives the PSK and channel hash automatically. - Hashtags []string `yaml:"hashtags"` + Hashtags []HashtagConfig `yaml:"hashtags"` // Keys maps channel hash hex → explicit key config. Keys map[string]ExplicitKeyConfig `yaml:"keys"` @@ -330,8 +330,9 @@ type ChannelKeysConfig struct { // ExplicitKeyConfig holds an explicit channel key and optional display name. type ExplicitKeyConfig struct { - Key string `yaml:"key"` // hex-encoded key bytes - Name string `yaml:"name"` // optional display name + Key string `yaml:"key"` // hex-encoded key bytes + Name string `yaml:"name"` // optional display name + Region string `yaml:"region"` // optional region slug; empty lists the channel Beacon-wide } // IATAConfig holds optional overrides for a known IATA code. @@ -414,6 +415,9 @@ func Load(path string) (*Config, error) { if err := cfg.validateMeshMapper(); err != nil { return nil, err } + if err := cfg.validateChannelKeys(); err != nil { + return nil, err + } if err := cfg.validateScopes(); err != nil { return nil, err } diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go index f76d3bde..8f8a0c70 100644 --- a/internal/config/meshmapper.go +++ b/internal/config/meshmapper.go @@ -12,13 +12,28 @@ import ( // MeshMapper rate-limits get_zones/get_geojson to one request per target per // 23.5h and get_scopes to one per region per 55m; minimums round up. const ( - MinZonesRefresh = 24 * time.Hour - MinScopesRefresh = time.Hour + MinZonesRefresh = 24 * time.Hour + MinScopesRefresh = time.Hour + MinChannelsRefresh = 24 * time.Hour // get_channels.php: one per region per 23.5h ) type MeshMapperConfig struct { - Scopes MeshMapperScopesConfig `yaml:"scopes"` - Zones MeshMapperZonesConfig `yaml:"zones"` + Scopes MeshMapperScopesConfig `yaml:"scopes"` + Zones MeshMapperZonesConfig `yaml:"zones"` + Channels MeshMapperChannelsConfig `yaml:"channels"` +} + +// MeshMapperChannelsConfig imports each known IATA's public hashtag channels. +type MeshMapperChannelsConfig struct { + Enabled bool `yaml:"enabled"` + RefreshInterval duration `yaml:"refresh_interval"` +} + +func (c MeshMapperChannelsConfig) Interval() time.Duration { + if c.RefreshInterval.Duration == 0 { + return MinChannelsRefresh + } + return c.RefreshInterval.Duration } // MeshMapperZonesConfig imports each known IATA's published boundary, overriding borderFile. @@ -59,6 +74,9 @@ func (c *Config) validateMeshMapper() error { return fmt.Errorf("meshmapper.zones.refresh_interval must be between 24h and 168h") } } + if ch := c.MeshMapper.Channels; ch.Enabled && (ch.Interval() < MinChannelsRefresh || ch.Interval() > 7*24*time.Hour) { + return fmt.Errorf("meshmapper.channels.refresh_interval must be between 24h and 168h") + } s := c.MeshMapper.Scopes if !s.Enabled { return nil diff --git a/internal/config/seed.go b/internal/config/seed.go index 86d32492..e08f259b 100644 --- a/internal/config/seed.go +++ b/internal/config/seed.go @@ -21,6 +21,7 @@ type Seeder interface { UpsertRegion(ctx context.Context, slug, name, description string, displayOrder int, centerLat, centerLng *float64, zoomLevel *int) (int32, error) SetRegionIATAs(ctx context.Context, regionID int32, iatas []string) error UpsertTransportScope(ctx context.Context, name, displayName string, transportKey, keyFingerprint []byte) error + SetChannelConfigScopes(ctx context.Context, fingerprints [][]byte, regions []string) error } // Seed applies config-defined regions, IATA overrides to the database. @@ -62,6 +63,15 @@ func Seed(ctx context.Context, cfg *Config, db Seeder) error { return err } } + // Channel region placement + var fingerprints [][]byte + var regions []string + for _, scope := range cfg.ChannelScopes() { + fingerprints, regions = append(fingerprints, scope.Fingerprint), append(regions, scope.Region) + } + if err := db.SetChannelConfigScopes(ctx, fingerprints, regions); err != nil { + return err + } // Transport Codes for _, s := range cfg.Scopes { entry := scopestore.FromName(s.Name) diff --git a/internal/config/seed_test.go b/internal/config/seed_test.go index 6241390c..c09aa11f 100644 --- a/internal/config/seed_test.go +++ b/internal/config/seed_test.go @@ -15,12 +15,13 @@ import ( ) type stubSeeder struct { - regions map[string]int32 - iatas []string - regionIATAs map[int32][]string - scopes []string - borders map[string]json.RawMessage - upsertErr error + regions map[string]int32 + iatas []string + regionIATAs map[int32][]string + channelRegions []string + scopes []string + borders map[string]json.RawMessage + upsertErr error } func newStubSeeder() *stubSeeder { @@ -57,6 +58,11 @@ func (s *stubSeeder) SetRegionIATAs(_ context.Context, regionID int32, iatas []s return s.upsertErr } +func (s *stubSeeder) SetChannelConfigScopes(_ context.Context, fingerprints [][]byte, regions []string) error { + s.channelRegions = regions + return s.upsertErr +} + func (s *stubSeeder) UpsertTransportScope(_ context.Context, name, _ string, _, _ []byte) error { s.scopes = append(s.scopes, name) return s.upsertErr @@ -256,3 +262,17 @@ func TestDeriveScopeKey_DifferentInputs(t *testing.T) { t.Error("expected different keys for different inputs") } } + +func TestSeed_ChannelScopes(t *testing.T) { + cfg := &Config{ + Regions: []RegionConfig{{Slug: "bc", IATAs: []string{"YVR"}}}, + ChannelKeys: ChannelKeysConfig{Hashtags: []HashtagConfig{{Name: "meshcore"}, {Name: "vancouver", Region: "bc"}}}, + } + db := newStubSeeder() + if err := Seed(context.Background(), cfg, db); err != nil { + t.Fatal(err) + } + if len(db.channelRegions) != 2 || db.channelRegions[0] != "" || db.channelRegions[1] != "bc" { + t.Fatalf("channel regions = %q", db.channelRegions) + } +} diff --git a/internal/ingest/backfill.go b/internal/ingest/backfill.go index ecb7ebb8..1a412839 100644 --- a/internal/ingest/backfill.go +++ b/internal/ingest/backfill.go @@ -107,6 +107,19 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) ( if err != nil { return 0, err } + return backfill(ctx, db, keys, packets), nil +} + +// BackfillChannelHashes retries only the channels whose keys were just imported at runtime. +func BackfillChannelHashes(ctx context.Context, db DB, keys ChannelKeyStore, hashes [][]byte) (int, error) { + packets, err := db.ListUndecryptedGroupTextPacketsByHash(ctx, hashes) + if err != nil { + return 0, err + } + return backfill(ctx, db, keys, packets), nil +} + +func backfill(ctx context.Context, db DB, keys ChannelKeyStore, packets []UndecryptedPacket) int { decrypted := 0 for _, p := range packets { result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload) @@ -118,5 +131,5 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) ( decrypted++ } } - return decrypted, nil + return decrypted } diff --git a/internal/ingest/backfill_test.go b/internal/ingest/backfill_test.go index 1bbeeef6..c50d7a6a 100644 --- a/internal/ingest/backfill_test.go +++ b/internal/ingest/backfill_test.go @@ -135,6 +135,23 @@ func TestBackfillChannelMessages_DecryptsAndCounts(t *testing.T) { } } +func TestBackfillChannelHashes_OnlyScansNewHashes(t *testing.T) { + psk := make([]byte, 16) + db := &stubDB{ + insertChannelMessageResult: true, + undecryptedByHash: []UndecryptedPacket{{PacketHash: []byte{0x01}, RawPayload: encryptedGroupText(t, 0x44, psk, "ded", "hi")}}, + undecryptedPackets: []UndecryptedPacket{{PacketHash: []byte{0x02}, RawPayload: encryptedGroupText(t, 0x44, psk, "x", "y")}}, + } + keys := &mapKeys{entries: map[byte][]keystore.Entry{0x44: {{Key: psk, Fingerprint: []byte{0xAA}}}}} + n, err := BackfillChannelHashes(context.Background(), db, keys, [][]byte{{0x44}}) + if err != nil || n != 1 { + t.Fatalf("n=%d err=%v", n, err) + } + if len(db.backfillHashes) != 1 || db.backfillHashes[0][0] != 0x44 { + t.Fatalf("scanned hashes %x", db.backfillHashes) + } +} + func TestBackfillChannelMessages_NoUndecryptedPackets(t *testing.T) { db := &stubDB{} keys := &mapKeys{} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 59b9b4e0..c9d0dc7a 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -174,6 +174,9 @@ type DB interface { // used by BackfillChannelMessages to retry them against the current keystore at boot. ListUndecryptedGroupTextPackets(ctx context.Context) ([]UndecryptedPacket, error) + // ListUndecryptedGroupTextPacketsByHash limits that scan to channels that just gained a key. + ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]UndecryptedPacket, error) + // UpsertChannelIATA upserts a channel_iatas row. UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index aa8f892a..14acd2a6 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -184,6 +184,8 @@ type stubDB struct { observationInserted bool insertChannelMessageResult bool // configurable return for InsertChannelMessage; default false undecryptedPackets []UndecryptedPacket + undecryptedByHash []UndecryptedPacket + backfillHashes [][]byte dbHook func() // runs in UpsertChannelIATA, the first DB call after InsertObservation } @@ -279,6 +281,11 @@ func (s *stubDB) ListUndecryptedGroupTextPackets(_ context.Context) ([]Undecrypt return s.undecryptedPackets, nil } +func (s *stubDB) ListUndecryptedGroupTextPacketsByHash(_ context.Context, hashes [][]byte) ([]UndecryptedPacket, error) { + s.backfillHashes = hashes + return s.undecryptedByHash, nil +} + func (s *stubDB) UpsertChannelIATA(_ context.Context, _ []byte, _ string, _ time.Time) error { if s.dbHook != nil { s.dbHook() diff --git a/internal/keystore/keystore.go b/internal/keystore/keystore.go index 92119932..0485442f 100644 --- a/internal/keystore/keystore.go +++ b/internal/keystore/keystore.go @@ -2,14 +2,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // Package keystore provides channel key lookup for the ingest pipeline. -// Keys are loaded from config at startup and never written at runtime. -// Future: add a DB-backed fallback that checks the channel_keys table. +// Config keys are fixed at startup; MeshMapper imports are swapped in at runtime. package keystore import ( "bytes" "crypto/sha256" "encoding/hex" + "slices" + "sync/atomic" ) // Entry holds a resolved channel key along with its metadata. @@ -23,23 +24,51 @@ type Entry struct { // MapKeyStore maps channel hash hex → list of known key entries. // Multiple entries per hash are supported to handle 1-byte hash collisions. type MapKeyStore struct { - entries map[string][]Entry // channel hash hex → entries + config map[string][]Entry + imported map[string][]Entry + entries atomic.Pointer[map[string][]Entry] // config + imported, read lock-free by ingest } -// NewMapKeyStore creates a keystore from a pre-built map of entries. -// Use BuildFromConfig to construct entries from a config.ChannelKeysConfig. +// NewMapKeyStore creates a keystore from the configured entries. func NewMapKeyStore(entries map[string][]Entry) *MapKeyStore { - store := &MapKeyStore{entries: make(map[string][]Entry)} + store := &MapKeyStore{config: make(map[string][]Entry), imported: map[string][]Entry{}} for hash, list := range entries { - store.entries[hash] = append(store.entries[hash], list...) + store.config[hash] = append(store.config[hash], list...) } + store.entries.Store(&store.config) return store } // GetKey returns all known key entries for the given channel hash byte. // Returns nil if no keys are known for this hash. func (s *MapKeyStore) GetKey(channelHash []byte) []Entry { - return s.entries[hex.EncodeToString(channelHash)] + return (*s.entries.Load())[hex.EncodeToString(channelHash)] +} + +// SetImported replaces the imported keys and returns the channel hashes of keys +// that weren't known before. Config keys win; one writer at a time. +func (s *MapKeyStore) SetImported(entries []Entry) [][]byte { + merged := make(map[string][]Entry, len(s.config)) + for hash, list := range s.config { + merged[hash] = list + } + imported := map[string][]Entry{} + var added [][]byte + for _, e := range entries { + hash := sha256.Sum256(e.Key) + hashHex := hex.EncodeToString(hash[:1]) + if EntryExists(merged[hashHex], e) { + continue + } + merged[hashHex] = append(slices.Clone(merged[hashHex]), e) + imported[hashHex] = append(imported[hashHex], e) + if !EntryExists(s.imported[hashHex], e) { + added = append(added, hash[:1]) + } + } + s.imported = imported + s.entries.Store(&merged) + return added } // DeriveHashtagKey derives the channel secret and hash for a hashtag name. diff --git a/internal/keystore/keystore_test.go b/internal/keystore/keystore_test.go index bed3102a..2060c762 100644 --- a/internal/keystore/keystore_test.go +++ b/internal/keystore/keystore_test.go @@ -153,3 +153,33 @@ func TestEntryExists_Empty(t *testing.T) { t.Error("expected false for empty slice") } } + +func TestSetImported_ConfigWinsAndNewHashesReported(t *testing.T) { + secret, hash, fp := DeriveHashtagKey("meshcore") + hashHex := hex.EncodeToString([]byte{hash}) + configured := Entry{Key: secret, Fingerprint: fp, Hashtag: "meshcore", Name: "Configured"} + store := NewMapKeyStore(map[string][]Entry{hashHex: {configured}}) + + dupSecret, _, dupFP := DeriveHashtagKey("meshcore") + newSecret, newHash, newFP := DeriveHashtagKey("ottawa-mesh") + added := store.SetImported([]Entry{ + {Key: dupSecret, Fingerprint: dupFP, Hashtag: "meshcore", Name: "#meshcore"}, + {Key: newSecret, Fingerprint: newFP, Hashtag: "ottawa-mesh", Name: "#ottawa-mesh"}, + }) + if len(added) != 1 || added[0][0] != newHash { + t.Fatalf("new hashes = %x", added) + } + if got := store.GetKey([]byte{hash}); len(got) != 1 || got[0].Name != "Configured" { + t.Fatalf("config entry replaced: %+v", got) + } + if got := store.GetKey([]byte{newHash}); len(got) != 1 || got[0].Hashtag != "ottawa-mesh" { + t.Fatalf("imported entry missing: %+v", got) + } + if added = store.SetImported([]Entry{{Key: newSecret, Fingerprint: newFP, Hashtag: "ottawa-mesh"}}); len(added) != 0 { + t.Fatal("unchanged import reported new hashes", added) + } + store.SetImported(nil) + if store.GetKey([]byte{newHash}) != nil || len(store.GetKey([]byte{hash})) != 1 { + t.Fatal("clearing imports touched config keys or kept the import") + } +} diff --git a/internal/meshmapper/channels.go b/internal/meshmapper/channels.go new file mode 100644 index 00000000..934af8aa --- /dev/null +++ b/internal/meshmapper/channels.go @@ -0,0 +1,307 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/iatadb" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" +) + +const ( + MaxChannels = 256 // per region + MaxChannelBody = 64 << 10 + + channelRetry = config.MinChannelsRefresh // get_channels.php allows one request per region per 23.5h, failed or not +) + +type ChannelStore interface { + ListKnownIATAs(ctx context.Context) ([]string, error) + ListChannelCatalogues(ctx context.Context) ([]Catalogue, error) + // SaveChannelCatalogue replaces the IATA's members when the cache carries a new payload. + SaveChannelCatalogue(ctx context.Context, iata, url string, cache Cache, fingerprints [][]byte) error + ClearChannelMembers(ctx context.Context) error +} + +// ChannelKeys receives every imported key and reports the channel hashes it hadn't seen. +type ChannelKeys interface { + SetImported(entries []keystore.Entry) [][]byte +} + +type channelSource struct { + iata, country, url string + cache Cache + entries []keystore.Entry +} + +// Channels is owned by one background task. +type Channels struct { + store ChannelStore + keys ChannelKeys + dir *Directory + channelsURL func(site string) (string, bool) + client *http.Client + interval time.Duration + seen map[string]bool + sources []channelSource + onNewKeys func(hashes [][]byte) +} + +// NewChannels restores saved channel lists into keys without HTTP requests. +// Disabled, it clears MeshMapper's region membership so config scoping alone applies. +func NewChannels(ctx context.Context, cfg config.MeshMapperChannelsConfig, store ChannelStore, dir *Directory, keys ChannelKeys) (*Channels, error) { + c := &Channels{store: store, keys: keys, dir: dir, channelsURL: channelsEndpoint, interval: cfg.Interval(), seen: map[string]bool{}, + client: &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} + if !cfg.Enabled { + if err := store.ClearChannelMembers(ctx); err != nil { + return nil, fmt.Errorf("clear MeshMapper channel members: %w", err) + } + return c, nil + } + saved, err := store.ListChannelCatalogues(ctx) + if err != nil { + return nil, fmt.Errorf("restore channel catalogues: %w", err) + } + latest := map[string]Catalogue{} + for _, cat := range saved { + if old, ok := latest[cat.IATA]; !ok || cat.AttemptedAt.After(old.AttemptedAt) { + latest[cat.IATA] = cat + } + } + iatas, err := store.ListKnownIATAs(ctx) + if err != nil { + return nil, fmt.Errorf("list IATAs for MeshMapper channels: %w", err) + } + c.track(iatas, latest) + c.publish() + return c, nil +} + +// OnNewKeys is told the channel hashes an import added, for backfill. +func (c *Channels) OnNewKeys(fn func(hashes [][]byte)) { c.onNewKeys = fn } + +func (c *Channels) track(iatas []string, saved map[string]Catalogue) { + for _, iata := range iatas { + if c.seen[iata] { + continue + } + c.seen[iata] = true + country := iatadb.CountryFor(iata) + if country == "" { + continue + } + s := channelSource{iata: iata, country: country} + if cat, ok := saved[iata]; ok { + s.url, s.cache = cat.URL, cat.Cache + if len(cat.Payload) > 0 { + var err error + if s.entries, err = decodeChannels(cat.Payload, iata); err != nil { + s.entries, s.cache.Payload, s.cache.ETag = nil, nil, "" + s.cache.LastError = "invalid saved catalogue; awaiting refresh" + } + } + c.log(s, "restored") + } + c.sources = append(c.sources, s) + } +} + +// Refresh makes at most one request. The scheduler serializes calls every 15s. +func (c *Channels) Refresh(ctx context.Context) (err error) { + parent := ctx + defer func() { + if parent.Err() != nil { + err = nil + } + }() + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + now := time.Now().UTC() + iatas, err := c.store.ListKnownIATAs(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper channels: %w", err) + } + c.track(iatas, nil) + for n := range c.sources { + s := &c.sources[n] + if now.Before(s.cache.NextAttempt) { + continue + } + zones, fetched, err := c.dir.List(ctx, s.country, now) + if err != nil || fetched { + return err + } + if zones == nil { + continue + } + entry, listed := zones[s.iata] + endpoint, valid := "", false + if listed { + endpoint, valid = c.channelsURL(entry.url) + } + if !valid { + // No request to make; channels already imported stay until a list replaces them. + s.cache.NextAttempt, s.cache.LastError = now.Add(c.interval), "not listed" + if listed { + s.cache.LastError = "invalid site URL" + } + c.log(*s, "skipped") + continue + } + if endpoint != s.url { + s.url, s.cache.ETag = endpoint, "" + } + return c.refresh(ctx, s, now) + } + return nil +} + +func (c *Channels) refresh(ctx context.Context, s *channelSource, now time.Time) error { + update := Cache{AttemptedAt: now, NextAttempt: now.Add(c.interval)} + etag := "" + if len(s.cache.Payload) > 0 { + etag = s.cache.ETag + } + status, body, header, err := get(ctx, c.client, "Beacon-MeshMapper-Channels/1", s.url, etag, MaxChannelBody) + if err != nil { + return err + } + var entries []keystore.Entry + var retryAt time.Time + switch status { + case http.StatusOK: + if entries, err = decodeChannels(body, s.iata); err != nil { + update.LastError = "invalid response" + } else { + update.Payload = body + } + case http.StatusNotModified: + if len(s.cache.Payload) == 0 { + update.LastError = "304 without cached catalogue" + } + default: + update.LastError = statusProblem(status, header, now, &retryAt) + } + if update.LastError == "" { + etag := header.Get("ETag") + if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") { + update.LastError, update.Payload, entries = "invalid ETag", nil, nil + } else { + if status == http.StatusNotModified && etag == "" { + etag = s.cache.ETag + } + update.ETag, update.CheckedAt = etag, now + } + } + if update.LastError != "" { + update.NextAttempt = now.Add(min(c.interval, channelRetry)) + if retryAt.After(update.NextAttempt) { + update.NextAttempt = retryAt + } + } + var fingerprints [][]byte + for _, e := range entries { + fingerprints = append(fingerprints, e.Fingerprint) + } + if update.Payload != nil && fingerprints == nil { + fingerprints = [][]byte{} + } + if err := c.store.SaveChannelCatalogue(ctx, s.iata, s.url, update, fingerprints); err != nil { + s.cache.NextAttempt = update.NextAttempt // never retry every tick during a DB outage + return fmt.Errorf("persist channel catalogue %s: %w", s.iata, err) + } + if update.Payload != nil { + s.cache.Payload, s.entries = update.Payload, entries + } + if !update.CheckedAt.IsZero() { + s.cache.CheckedAt, s.cache.ETag = update.CheckedAt, update.ETag + } + s.cache.AttemptedAt, s.cache.NextAttempt, s.cache.LastError = now, update.NextAttempt, update.LastError + if update.Payload != nil { + c.publish() + } + c.log(*s, "checked") + return nil +} + +func (c *Channels) publish() { + var all []keystore.Entry + for _, s := range c.sources { + all = append(all, s.entries...) + } + if added := c.keys.SetImported(all); len(added) > 0 && c.onNewKeys != nil { + c.onNewKeys(added) + } +} + +func (c *Channels) log(s channelSource, action string) { + level := slog.LevelInfo + if s.cache.LastError != "" && s.cache.LastError != "not listed" { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, "MeshMapper channels "+action, "component", "meshmapper.channels", "iata", s.iata, "source", s.url, + "channels", len(s.entries), "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError) +} + +func channelsEndpoint(site string) (string, bool) { return siteEndpoint(site, "get_channels.php") } + +// decodeChannels accepts a single region's list only when every key and hash is +// exactly what Beacon derives from the name, so a bad entry can't plant a wrong key. +func decodeChannels(body []byte, iata string) ([]keystore.Entry, error) { + var document struct { + GeneratedAt time.Time `json:"generated_at"` + Region string `json:"region"` + Zones []string `json:"zones"` + Count *int `json:"count"` + Channels *[]struct { + Name string `json:"name"` + Key string `json:"key"` + Hash string `json:"hash"` + Zones []string `json:"zones"` + } `json:"channels"` + } + if len(body) > MaxChannelBody || !utf8.Valid(body) { + return nil, fmt.Errorf("invalid channel list size/encoding") + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, err + } + if document.GeneratedAt.IsZero() || document.Region != iata || len(document.Zones) != 1 || document.Zones[0] != iata || + document.Channels == nil || document.Count == nil || *document.Count != len(*document.Channels) || len(*document.Channels) > MaxChannels { + return nil, fmt.Errorf("invalid regional channel list") + } + entries := make([]keystore.Entry, 0, len(*document.Channels)) + seen := map[string]bool{} + for _, ch := range *document.Channels { + tag, ok := strings.CutPrefix(ch.Name, "#") + if !ok || tag == "" || len(ch.Name) > 64 || strings.ToLower(ch.Name) != ch.Name || seen[ch.Name] || + strings.ContainsFunc(tag, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) || r == '#' }) || + len(ch.Zones) != 1 || ch.Zones[0] != iata { + return nil, fmt.Errorf("invalid channel entry") + } + secret, hash, fingerprint := keystore.DeriveHashtagKey(tag) + key, err := hex.DecodeString(ch.Key) + if err != nil || !bytes.Equal(key, secret) || !strings.EqualFold(ch.Hash, hex.EncodeToString([]byte{hash})) { + return nil, fmt.Errorf("channel %s key does not match its name", ch.Name) + } + seen[ch.Name] = true + entries = append(entries, keystore.Entry{Key: secret, Fingerprint: fingerprint, Hashtag: tag, Name: ch.Name}) + } + return entries, nil +} diff --git a/internal/meshmapper/channels_test.go b/internal/meshmapper/channels_test.go new file mode 100644 index 00000000..5a3def56 --- /dev/null +++ b/internal/meshmapper/channels_test.go @@ -0,0 +1,234 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" +) + +func channelList(iata string, names ...string) []byte { + rows := []map[string]any{} + for _, name := range names { + secret, hash, _ := keystore.DeriveHashtagKey(strings.TrimPrefix(name, "#")) + rows = append(rows, map[string]any{"name": name, "key": hex.EncodeToString(secret), "hash": hex.EncodeToString([]byte{hash}), "zones": []string{iata}}) + } + b, _ := json.Marshal(map[string]any{"generated_at": "2026-09-30T18:00:00Z", "region": iata, "zones": []string{iata}, "count": len(rows), "channels": rows}) + return b +} + +type channelMemoryStore struct { + iatas []string + rows map[sourceKey]Cache + members map[string][][]byte + fail bool +} + +func newChannelMemoryStore(iatas ...string) *channelMemoryStore { + return &channelMemoryStore{iatas: iatas, rows: map[sourceKey]Cache{}, members: map[string][][]byte{}} +} + +func (s *channelMemoryStore) ListKnownIATAs(context.Context) ([]string, error) { + return slices.Clone(s.iatas), nil +} + +func (s *channelMemoryStore) ListChannelCatalogues(context.Context) ([]Catalogue, error) { + var out []Catalogue + for key, c := range s.rows { + out = append(out, Catalogue{IATA: key.iata, URL: key.url, Cache: c}) + } + return out, nil +} + +func (s *channelMemoryStore) SaveChannelCatalogue(_ context.Context, iata, url string, next Cache, fingerprints [][]byte) error { + if s.fail { + return context.DeadlineExceeded + } + if next.Payload != nil { + s.members[iata] = fingerprints + } + old := s.rows[sourceKey{iata, url}] + if next.Payload == nil { + next.Payload = old.Payload + } + if next.CheckedAt.IsZero() { + next.CheckedAt, next.ETag = old.CheckedAt, old.ETag + } + s.rows[sourceKey{iata, url}] = next + return nil +} + +func (s *channelMemoryStore) ClearChannelMembers(context.Context) error { + s.members = map[string][][]byte{} + return nil +} + +type channelSite struct { + *httptest.Server + list, body string + status int + retryAfter string + listCalls, channels int +} + +func newChannelSite(t *testing.T) *channelSite { + f := &channelSite{status: 200, body: string(channelList("YOW", "#ottawa-mesh"))} + f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/get_zones.php": + f.listCalls++ + _, _ = w.Write([]byte(f.list)) + case "/get_channels.php": + f.channels++ + w.Header().Set("ETag", `"c1"`) + w.Header().Set("Retry-After", f.retryAfter) + w.WriteHeader(f.status) + if f.status == 200 { + _, _ = w.Write([]byte(f.body)) + } + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + f.list = `{"country":"CA","zones":[{"code":"YOW","url":"` + f.URL + `/","has_boundary":true,"group":null}]}` + t.Cleanup(f.Close) + return f +} + +type channelHarness struct { + c *Channels + keys *keystore.MapKeyStore + added [][]byte +} + +func newChannelHarness(t *testing.T, f *channelSite, store *channelMemoryStore, lists *zoneListMemory, enabled bool) *channelHarness { + t.Helper() + dir := NewDirectory(lists) + dir.listURL = f.URL + "/get_zones.php" + if err := dir.Restore(context.Background()); err != nil { + t.Fatal(err) + } + h := &channelHarness{keys: keystore.NewMapKeyStore(nil)} + c, err := NewChannels(context.Background(), config.MeshMapperChannelsConfig{Enabled: enabled}, store, dir, h.keys) + if err != nil { + t.Fatal(err) + } + c.channelsURL = func(site string) (string, bool) { return site + "get_channels.php", strings.HasPrefix(site, f.URL) } + c.OnNewKeys(func(hashes [][]byte) { h.added = append(h.added, hashes...) }) + h.c = c + return h +} + +func (h *channelHarness) refresh(t *testing.T, n int) { + t.Helper() + for range n { + if err := h.c.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } +} + +func TestChannelsImportKeysAndMembers(t *testing.T) { + f := newChannelSite(t) + store := newChannelMemoryStore("YOW") + h := newChannelHarness(t, f, store, newZoneListMemory(), true) + h.refresh(t, 2) + secret, hash, fingerprint := keystore.DeriveHashtagKey("ottawa-mesh") + got := h.keys.GetKey([]byte{hash}) + if len(got) != 1 || !bytes.Equal(got[0].Key, secret) || got[0].Name != "#ottawa-mesh" || got[0].Hashtag != "ottawa-mesh" { + t.Fatalf("key not imported: %+v", got) + } + if len(store.members["YOW"]) != 1 || !bytes.Equal(store.members["YOW"][0], fingerprint) { + t.Fatal("YOW membership not saved", store.members) + } + if len(h.added) != 1 || h.added[0][0] != hash { + t.Fatal("new key not reported for backfill", h.added) + } + if b := h.c.sources[0].cache; b.NextAttempt.Sub(b.AttemptedAt) != 24*time.Hour { + t.Fatal("refresh interval", b.NextAttempt.Sub(b.AttemptedAt)) + } +} + +func TestChannelsRejectMismatchKeepsLastGood(t *testing.T) { + f := newChannelSite(t) + store := newChannelMemoryStore("YOW") + h := newChannelHarness(t, f, store, newZoneListMemory(), true) + h.refresh(t, 2) + good := string(channelList("YOW", "#ottawa-mesh")) + secret, hash, _ := keystore.DeriveHashtagKey("ottawa-mesh") + for name, body := range map[string]string{ + "planted key": strings.Replace(good, hex.EncodeToString(secret), strings.Repeat("0", 32), 1), + "wrong key": strings.Replace(string(channelList("YOW", "#other")), `"#other"`, `"#planted"`, 1), + "wrong hash": strings.Replace(good, `"hash":"`+hex.EncodeToString([]byte{hash})+`"`, `"hash":"zz"`, 1), + "other region": string(channelList("YYZ", "#ottawa-mesh")), + "bad count": strings.Replace(good, `"count":1`, `"count":2`, 1), + "uppercase": string(channelList("YOW", "#Ottawa")), + "group zones": strings.Replace(good, `"zones":["YOW"]}`, `"zones":["YOW","YYZ"]}`, 1), + "missing count": strings.Replace(good, `"count":1,`, ``, 1), + } { + f.body = body + h.c.sources[0].cache.NextAttempt = time.Time{} + h.refresh(t, 1) + s := h.c.sources[0] + if s.cache.LastError != "invalid response" || len(h.keys.GetKey([]byte{hash})) != 1 || len(store.members["YOW"]) != 1 { + t.Fatalf("%s: accepted or dropped last good list: %q", name, s.cache.LastError) + } + if s.cache.NextAttempt.Sub(s.cache.AttemptedAt) != 24*time.Hour { + t.Fatalf("%s: retried before the rate limit", name) + } + } +} + +func TestChannelsRateLimitAndFailuresWait(t *testing.T) { + for _, tc := range []struct { + status int + retryAfter string + want time.Duration + }{{503, "", 24 * time.Hour}, {429, "60", 24 * time.Hour}, {429, "172800", 48 * time.Hour}} { + f := newChannelSite(t) + f.status, f.retryAfter = tc.status, tc.retryAfter + h := newChannelHarness(t, f, newChannelMemoryStore("YOW"), newZoneListMemory(), true) + h.refresh(t, 3) + s := h.c.sources[0] + if f.channels != 1 || s.cache.NextAttempt.Sub(s.cache.AttemptedAt) != tc.want { + t.Fatalf("HTTP %d Retry-After %q: calls=%d wait=%v", tc.status, tc.retryAfter, f.channels, s.cache.NextAttempt.Sub(s.cache.AttemptedAt)) + } + } +} + +func TestChannelsRestoreWithoutRequests(t *testing.T) { + f := newChannelSite(t) + store, lists := newChannelMemoryStore("YOW"), newZoneListMemory() + newChannelHarness(t, f, store, lists, true).refresh(t, 2) + restarted := newChannelHarness(t, f, store, lists, true) + _, hash, _ := keystore.DeriveHashtagKey("ottawa-mesh") + if len(restarted.keys.GetKey([]byte{hash})) != 1 { + t.Fatal("saved channels not restored") + } + restarted.refresh(t, 2) + if f.listCalls != 1 || f.channels != 1 { + t.Fatal("restart made requests", f.listCalls, f.channels) + } +} + +func TestChannelsDisabledClearsMembers(t *testing.T) { + f := newChannelSite(t) + store := newChannelMemoryStore("YOW") + store.members["YOW"] = [][]byte{{1}} + h := newChannelHarness(t, f, store, newZoneListMemory(), false) + if len(store.members) != 0 || len(h.c.sources) != 0 { + t.Fatal("disabled import kept membership or sources") + } +} From 7392fdd2d6b30143ec1a59370d93dc9001a1a622 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Wed, 30 Sep 2026 23:21:44 -0400 Subject: [PATCH 120/141] fix(meshmapper): create group member IATAs and import zone names and centers --- cmd/beacon/main.go | 8 +++ config.yaml.example | 8 ++- db/meshmapper.go | 25 ++++++- db/meshmapper_integration_test.go | 31 ++++++-- db/queries/queries.sql | 13 +++- db/sqlc/mock/querier.go | 14 ++++ db/sqlc/querier.go | 1 + db/sqlc/queries.sql.go | 39 ++++++++-- internal/cache/reader.go | 6 ++ internal/meshmapper/directory.go | 37 +++++++--- internal/meshmapper/groups.go | 110 +++++++++++++++++++++++++---- internal/meshmapper/groups_test.go | 40 ++++++++++- internal/meshmapper/zones.go | 6 +- internal/meshmapper/zones_test.go | 26 +++++++ 14 files changed, 317 insertions(+), 47 deletions(-) diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 4e4cf9a5..3ccbffb6 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -221,9 +221,17 @@ func main() { // Always restored, so disabling the import prunes it and file borders return. zones := meshmapper.NewZones(cfg.MeshMapper.Zones, store, directory) + var detailed []string + for iata, d := range cfg.IATAs { + if d.Name != "" || d.Lat != nil || d.Lng != nil { + detailed = append(detailed, iata) + } + } + zones.SetConfiguredIATAs(detailed) if cr, ok := reader.(*cache.CachedReader); ok { zones.OnChange(cr.InvalidateIATABorder) zones.OnRegionsChange(cr.InvalidateRegions) + zones.OnIATAsChange(cr.InvalidateIATAs) } if localBorders != nil { zones.OnUpdate(func(imported map[string]json.RawMessage) { diff --git a/config.yaml.example b/config.yaml.example index dce503d0..ae3a35ae 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -73,10 +73,14 @@ ratelimit: # Each country's zone list is fetched at most once a day (MeshMapper allows one # get_zones.php per country per 23.5h) and saved, so restarts don't refetch it. # Failed or rate-limited requests also wait out the limit, or Retry-After if longer. +# Listed IATAs also take their display name and approximate location from the +# zone list, unless iatas: below sets a name or lat/lng for them. # import_groups turns MeshMapper zone groups (e.g. ONQC: YOW, YQB, YUL) into # regions, from the zone lists already fetched. A group is imported when at least -# one member is a known IATA, with its full member list; slug = lowercased group -# code, listed after the regions below. A region below with the same slug wins. +# one member is a known IATA, with its full member list (members Beacon hasn't +# heard yet are added as IATAs); slug = lowercased group code, centered on the +# members' average location, listed after the regions below. A region below with +# the same slug wins. # Imported regions are removed when the group disappears or stops qualifying, # and all of them when import_groups is turned off. Requires zones.enabled. # zones: diff --git a/db/meshmapper.go b/db/meshmapper.go index 09405138..8ce90d1b 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -152,7 +152,7 @@ func (s *Store) ListRegionState(ctx context.Context) ([]meshmapper.RegionState, out := make([]meshmapper.RegionState, 0, len(rows)) for _, row := range rows { out = append(out, meshmapper.RegionState{Slug: row.Slug, Name: row.Name, DisplayOrder: int(row.DisplayOrder), - Imported: row.Imported, IATAs: row.Iatas}) + Imported: row.Imported, IATAs: row.Iatas, CenterLat: row.CenterLat, CenterLng: row.CenterLng}) } return out, nil } @@ -160,13 +160,18 @@ func (s *Store) ListRegionState(ctx context.Context) ([]meshmapper.RegionState, // SaveImportedRegion reports false when a hand-written region owns the slug. func (s *Store) SaveImportedRegion(ctx context.Context, r meshmapper.RegionState) (bool, error) { order := int32(r.DisplayOrder) - id, err := s.q.UpsertImportedRegion(ctx, sqlc.UpsertImportedRegionParams{Slug: r.Slug, Name: r.Name, DisplayOrder: &order}) + id, err := s.q.UpsertImportedRegion(ctx, sqlc.UpsertImportedRegionParams{Slug: r.Slug, Name: r.Name, DisplayOrder: &order, + CenterLat: r.CenterLat, CenterLng: r.CenterLng}) if errors.Is(err, pgx.ErrNoRows) { return false, nil } if err != nil { return false, err } + // region_iatas references iata_codes, so members Beacon hasn't heard yet are created first, as Seed does. + if err := s.q.AddIATAs(ctx, r.IATAs); err != nil { + return false, err + } return true, s.SetRegionIATAs(ctx, id, r.IATAs) } @@ -228,3 +233,19 @@ func (s *Store) SetChannelMembers(ctx context.Context, iata string, fingerprints func (s *Store) ClearChannelMembers(ctx context.Context) error { return s.q.DeleteAllChannelMembers(ctx) } + +func (s *Store) ListIATADetails(ctx context.Context) ([]meshmapper.IATADetails, error) { + rows, err := s.q.ListIATAs(ctx) + if err != nil { + return nil, err + } + out := make([]meshmapper.IATADetails, 0, len(rows)) + for _, row := range rows { + d := meshmapper.IATADetails{IATA: row.Iata, Lat: row.ApproxLat, Lng: row.ApproxLng} + if row.DisplayName != nil { + d.Name = *row.DisplayName + } + out = append(out, d) + } + return out, nil +} diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go index f80c0f7e..0df03238 100644 --- a/db/meshmapper_integration_test.go +++ b/db/meshmapper_integration_test.go @@ -268,11 +268,7 @@ func TestMeshMapperRegionsPostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) - for _, table := range []string{"regions", "region_iatas"} { - if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { - t.Fatal(err) - } - } + // Real tables inside the rolled-back tx: TEMP copies drop region_iatas' foreign key to iata_codes. store := &Store{q: sqlc.New(tx)} state := func() map[string]meshmapper.RegionState { t.Helper() @@ -286,6 +282,11 @@ func TestMeshMapperRegionsPostgres(t *testing.T) { } return out } + for _, iata := range []string{"YOW", "YUL"} { // Seed creates config region IATAs first + if err := store.UpsertIATA(ctx, iata); err != nil { + t.Fatal(err) + } + } hand, err := store.UpsertRegion(ctx, "east", "East", "", 3, nil, nil, nil) if err != nil { t.Fatal(err) @@ -299,9 +300,17 @@ func TestMeshMapperRegionsPostgres(t *testing.T) { if got := state()["east"]; got.Imported || got.DisplayOrder != 3 || strings.Join(got.IATAs, ",") != "YOW" { t.Fatal("configured members not replaced", got) } - group := meshmapper.RegionState{Slug: "onqc", Name: "Corridor", DisplayOrder: 4, IATAs: []string{"YOW", "YQB"}} + lat, lng := 45.5, -75.0 + group := meshmapper.RegionState{Slug: "onqc", Name: "Corridor", DisplayOrder: 4, IATAs: []string{"YOW", "YQB"}, CenterLat: &lat, CenterLng: &lng} if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved { - t.Fatal(saved, err) + t.Fatal("members Beacon hasn't heard must be created first", saved, err) + } + if got := state()["onqc"]; got.CenterLat == nil || *got.CenterLat != lat || *got.CenterLng != lng { + t.Fatal("center not saved", got) + } + details, err := store.ListIATADetails(ctx) + if err != nil || !slices.ContainsFunc(details, func(d meshmapper.IATADetails) bool { return d.IATA == "YQB" }) { + t.Fatal("member IATA not created", err) } group.IATAs = []string{"YQB", "YUL"} if saved, err := store.SaveImportedRegion(ctx, group); err != nil || !saved { @@ -325,6 +334,14 @@ func TestMeshMapperRegionsPostgres(t *testing.T) { if _, err := store.SaveImportedRegion(ctx, meshmapper.RegionState{Slug: "golm", Name: "Lakes", IATAs: []string{"YYZ"}}); err != nil { t.Fatal(err) } + if err := store.UpsertIATADetails(ctx, "YQB", "Quebec City", &lat, &lng); err != nil { + t.Fatal(err) + } + if details, _ := store.ListIATADetails(ctx); !slices.ContainsFunc(details, func(d meshmapper.IATADetails) bool { + return d.IATA == "YQB" && d.Name == "Quebec City" && d.Lat != nil && *d.Lat == lat + }) { + t.Fatal("IATA details not saved") + } if pruned, err := store.PruneImportedRegions(ctx, nil); err != nil || strings.Join(pruned, ",") != "golm" { t.Fatal("prune touched configured regions", pruned, err) } diff --git a/db/queries/queries.sql b/db/queries/queries.sql index addcddb1..bae2646a 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -10,6 +10,11 @@ INSERT INTO iata_codes (iata) VALUES ($1) ON CONFLICT (iata) DO NOTHING; +-- name: AddIATAs :exec +INSERT INTO iata_codes (iata) +SELECT unnest(@iatas::bpchar[]) +ON CONFLICT (iata) DO NOTHING; + -- name: GetIATA :one SELECT * FROM iata_codes WHERE iata = $1; @@ -1235,7 +1240,7 @@ ON CONFLICT (region_id, iata) DO NOTHING; -- name: ListRegionState :many -- Every region with its members, for reconciling imported MeshMapper groups. -SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, +SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, r.center_lat, r.center_lng, COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas FROM regions r LEFT JOIN region_iatas ri ON ri.region_id = r.id @@ -1244,11 +1249,13 @@ ORDER BY r.slug; -- name: UpsertImportedRegion :one -- A hand-written region owns its slug: the WHERE turns a clash into no row. -INSERT INTO regions (slug, name, display_order, zoom_level, imported, updated_at) -VALUES (@slug, @name, @display_order, NULL, TRUE, NOW()) +INSERT INTO regions (slug, name, display_order, center_lat, center_lng, zoom_level, imported, updated_at) +VALUES (@slug, @name, @display_order, sqlc.narg(center_lat), sqlc.narg(center_lng), NULL, TRUE, NOW()) ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, display_order = EXCLUDED.display_order, + center_lat = EXCLUDED.center_lat, + center_lng = EXCLUDED.center_lng, updated_at = NOW() WHERE regions.imported RETURNING id; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 7612b071..0626fdae 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -72,6 +72,20 @@ func (mr *MockQuerierMockRecorder) AddChannelMembers(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddChannelMembers", reflect.TypeOf((*MockQuerier)(nil).AddChannelMembers), ctx, arg) } +// AddIATAs mocks base method. +func (m *MockQuerier) AddIATAs(ctx context.Context, iatas []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddIATAs", ctx, iatas) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddIATAs indicates an expected call of AddIATAs. +func (mr *MockQuerierMockRecorder) AddIATAs(ctx, iatas any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddIATAs", reflect.TypeOf((*MockQuerier)(nil).AddIATAs), ctx, iatas) +} + // AddRegionIATAs mocks base method. func (m *MockQuerier) AddRegionIATAs(ctx context.Context, arg db.AddRegionIATAsParams) error { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 98290d7f..94b398a6 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -16,6 +16,7 @@ type Querier interface { // An empty region places the channel Beacon-wide. AddChannelConfigScopes(ctx context.Context, arg AddChannelConfigScopesParams) error AddChannelMembers(ctx context.Context, arg AddChannelMembersParams) error + AddIATAs(ctx context.Context, iatas []string) error AddRegionIATAs(ctx context.Context, arg AddRegionIATAsParams) error // Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. AmbiguousPrefixes(ctx context.Context) ([]AmbiguousPrefixesRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index d2264de9..453be2f8 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -45,6 +45,17 @@ func (q *Queries) AddChannelMembers(ctx context.Context, arg AddChannelMembersPa return err } +const addIATAs = `-- name: AddIATAs :exec +INSERT INTO iata_codes (iata) +SELECT unnest($1::bpchar[]) +ON CONFLICT (iata) DO NOTHING +` + +func (q *Queries) AddIATAs(ctx context.Context, iatas []string) error { + _, err := q.db.Exec(ctx, addIATAs, iatas) + return err +} + const addRegionIATAs = `-- name: AddRegionIATAs :exec INSERT INTO region_iatas (region_id, iata) SELECT $1, unnest($2::bpchar[]) @@ -3823,7 +3834,7 @@ func (q *Queries) ListPacketsByIATAs(ctx context.Context, arg ListPacketsByIATAs } const listRegionState = `-- name: ListRegionState :many -SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, +SELECT r.slug, r.name, COALESCE(r.display_order, 0)::int AS display_order, r.imported, r.center_lat, r.center_lng, COALESCE(array_agg(ri.iata::text ORDER BY ri.iata) FILTER (WHERE ri.iata IS NOT NULL), '{}')::text[] AS iatas FROM regions r LEFT JOIN region_iatas ri ON ri.region_id = r.id @@ -3836,6 +3847,8 @@ type ListRegionStateRow struct { Name string `json:"name"` DisplayOrder int32 `json:"display_order"` Imported bool `json:"imported"` + CenterLat *float64 `json:"center_lat"` + CenterLng *float64 `json:"center_lng"` Iatas []string `json:"iatas"` } @@ -3854,6 +3867,8 @@ func (q *Queries) ListRegionState(ctx context.Context) ([]ListRegionStateRow, er &i.Name, &i.DisplayOrder, &i.Imported, + &i.CenterLat, + &i.CenterLng, &i.Iatas, ); err != nil { return nil, err @@ -5284,25 +5299,35 @@ func (q *Queries) UpsertIATADetails(ctx context.Context, arg UpsertIATADetailsPa } const upsertImportedRegion = `-- name: UpsertImportedRegion :one -INSERT INTO regions (slug, name, display_order, zoom_level, imported, updated_at) -VALUES ($1, $2, $3, NULL, TRUE, NOW()) +INSERT INTO regions (slug, name, display_order, center_lat, center_lng, zoom_level, imported, updated_at) +VALUES ($1, $2, $3, $4, $5, NULL, TRUE, NOW()) ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name, display_order = EXCLUDED.display_order, + center_lat = EXCLUDED.center_lat, + center_lng = EXCLUDED.center_lng, updated_at = NOW() WHERE regions.imported RETURNING id ` type UpsertImportedRegionParams struct { - Slug string `json:"slug"` - Name string `json:"name"` - DisplayOrder *int32 `json:"display_order"` + Slug string `json:"slug"` + Name string `json:"name"` + DisplayOrder *int32 `json:"display_order"` + CenterLat *float64 `json:"center_lat"` + CenterLng *float64 `json:"center_lng"` } // A hand-written region owns its slug: the WHERE turns a clash into no row. func (q *Queries) UpsertImportedRegion(ctx context.Context, arg UpsertImportedRegionParams) (int32, error) { - row := q.db.QueryRow(ctx, upsertImportedRegion, arg.Slug, arg.Name, arg.DisplayOrder) + row := q.db.QueryRow(ctx, upsertImportedRegion, + arg.Slug, + arg.Name, + arg.DisplayOrder, + arg.CenterLat, + arg.CenterLng, + ) var id int32 err := row.Scan(&id) return id, err diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 737cd377..e585f93d 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -90,6 +90,12 @@ func (cr *CachedReader) InvalidateIATABorder(ctx context.Context, iata string) { cr.c.del(ctx, keyIATABorderPrefix+iata) } +// InvalidateIATAs makes imported MeshMapper names and locations visible to every IATA read. +func (cr *CachedReader) InvalidateIATAs(ctx context.Context) { + cr.c.del(ctx, keyIATAs) + cr.c.delPrefix(ctx, keyIATAPrefix) +} + // InvalidateRegions makes imported MeshMapper regions visible to every region read. func (cr *CachedReader) InvalidateRegions(ctx context.Context) { cr.c.del(ctx, keyRegions) diff --git a/internal/meshmapper/directory.go b/internal/meshmapper/directory.go index e429b920..4c1fedd4 100644 --- a/internal/meshmapper/directory.go +++ b/internal/meshmapper/directory.go @@ -4,6 +4,7 @@ package meshmapper import ( + "cmp" "context" "encoding/json" "fmt" @@ -21,6 +22,8 @@ import ( type zoneEntry struct { url string hasBoundary bool + name string // short display name, e.g. "Nanaimo" + lat, lon *float64 // nil when missing or out of range } type zoneGroup struct { @@ -157,16 +160,19 @@ func (d *Directory) fetch(ctx context.Context, country string, now time.Time) er return nil } -// groups merges every loaded list's groups by code. complete is false until each -// country has a list, so a cold start can't look like every group disappearing. -func (d *Directory) groups(countries []string) (merged map[string]zoneGroup, version int, complete bool) { +// snapshot merges every loaded list's zones and groups by code. complete is false until +// each country has a list, so a cold start can't look like every group disappearing. +func (d *Directory) snapshot(countries []string) (merged map[string]zoneGroup, zones map[string]zoneEntry, version int, complete bool) { d.mu.Lock() defer d.mu.Unlock() - merged = map[string]zoneGroup{} + merged, zones = map[string]zoneGroup{}, map[string]zoneEntry{} for _, country := range countries { list := d.lists[country] if list == nil || list.zones == nil { - return nil, d.version, false + return nil, nil, d.version, false + } + for code, z := range list.zones { + zones[code] = z } for code, g := range list.groups { if have, ok := merged[code]; ok { @@ -178,7 +184,7 @@ func (d *Directory) groups(countries []string) (merged map[string]zoneGroup, ver merged[code] = g } } - return merged, d.version, true + return merged, zones, d.version, true } var ( @@ -190,9 +196,13 @@ func decodeZones(body []byte, country string) (map[string]zoneEntry, map[string] var document struct { Country string `json:"country"` Zones *[]struct { - Code string `json:"code"` - URL string `json:"url"` - HasBoundary bool `json:"has_boundary"` + Code string `json:"code"` + Name string `json:"name"` + ShortName string `json:"short_name"` + Lat *float64 `json:"lat"` + Lon *float64 `json:"lon"` + URL string `json:"url"` + HasBoundary bool `json:"has_boundary"` } `json:"zones"` Groups []struct { Code string `json:"code"` @@ -208,7 +218,14 @@ func decodeZones(body []byte, country string) (map[string]zoneEntry, map[string] } zones := make(map[string]zoneEntry, len(*document.Zones)) for _, zone := range *document.Zones { - zones[strings.ToUpper(zone.Code)] = zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary} + entry := zoneEntry{url: zone.URL, hasBoundary: zone.HasBoundary, name: cmp.Or(zone.ShortName, zone.Name)} + if entry.name = strings.TrimSpace(entry.name); len(entry.name) > 128 || strings.ContainsFunc(entry.name, unicode.IsControl) { + entry.name = "" + } + if zone.Lat != nil && zone.Lon != nil && *zone.Lat >= -90 && *zone.Lat <= 90 && *zone.Lon >= -180 && *zone.Lon <= 180 { + entry.lat, entry.lon = zone.Lat, zone.Lon + } + zones[strings.ToUpper(zone.Code)] = entry } groups := map[string]zoneGroup{} for _, g := range document.Groups { diff --git a/internal/meshmapper/groups.go b/internal/meshmapper/groups.go index 464c06f4..588e1ee6 100644 --- a/internal/meshmapper/groups.go +++ b/internal/meshmapper/groups.go @@ -14,12 +14,31 @@ import ( // RegionState is a stored region; imported ones are also what SaveImportedRegion writes. type RegionState struct { - Slug, Name string - DisplayOrder int - Imported bool - IATAs []string + Slug, Name string + DisplayOrder int + Imported bool + IATAs []string + CenterLat, CenterLng *float64 } +// IATADetails is an IATA's display name and approximate location. +type IATADetails struct { + IATA string + Name string + Lat, Lng *float64 +} + +// SetConfiguredIATAs names the IATAs whose config entry sets a name or location; those win. +func (z *Zones) SetConfiguredIATAs(iatas []string) { + z.configured = map[string]bool{} + for _, iata := range iatas { + z.configured[iata] = true + } +} + +// OnIATAsChange is told whenever imported IATA names or locations change. +func (z *Zones) OnIATAsChange(fn func(ctx context.Context)) { z.onIATAs = fn } + // OnRegionsChange is told whenever the imported regions change, for cache invalidation. func (z *Zones) OnRegionsChange(fn func(ctx context.Context)) { z.onRegions = fn } @@ -36,21 +55,63 @@ func (z *Zones) pruneGroups(ctx context.Context) error { return nil } -// syncGroups reconciles imported regions with the loaded zone lists. It only runs -// once every tracked country has a list, and again when a list or the IATA set changes. -func (z *Zones) syncGroups(ctx context.Context, iatas []string) error { - if !z.importGroups { - return nil - } +// syncDirectory applies the loaded zone lists to IATA details and, when enabled, +// imported regions. It only runs once every tracked country has a list, and again +// when a list or the IATA set changes. +func (z *Zones) syncDirectory(ctx context.Context, iatas []string) error { var countries []string for _, r := range z.regions { countries = append(countries, r.country) } slices.Sort(countries) - groups, version, complete := z.dir.groups(slices.Compact(countries)) + groups, zones, version, complete := z.dir.snapshot(slices.Compact(countries)) if !complete || (z.groupsSynced && version == z.groupsVersion && len(iatas) == z.groupsKnown) { return nil } + if err := z.syncIATADetails(ctx, zones); err != nil { + return err + } + if z.importGroups { + if err := z.syncGroups(ctx, iatas, groups, zones); err != nil { + return err + } + } + z.groupsSynced, z.groupsVersion, z.groupsKnown = true, version, len(iatas) + return nil +} + +// syncIATADetails names and locates each listed IATA unless its config entry does. +func (z *Zones) syncIATADetails(ctx context.Context, zones map[string]zoneEntry) error { + stored, err := z.store.ListIATADetails(ctx) + if err != nil { + return fmt.Errorf("list IATAs for MeshMapper details: %w", err) + } + changed := false + for _, cur := range stored { + zone, ok := zones[cur.IATA] + if !ok || z.configured[cur.IATA] || zone.name == "" && zone.lat == nil { + continue + } + name := cmp.Or(zone.name, cur.Name) + lat, lng := cur.Lat, cur.Lng + if zone.lat != nil { + lat, lng = zone.lat, zone.lon + } + if name == cur.Name && sameFloat(lat, cur.Lat) && sameFloat(lng, cur.Lng) { + continue + } + if err := z.store.UpsertIATADetails(ctx, cur.IATA, name, lat, lng); err != nil { + return fmt.Errorf("save MeshMapper details for %s: %w", cur.IATA, err) + } + changed = true + } + if changed && z.onIATAs != nil { + z.onIATAs(ctx) + } + return nil +} + +func (z *Zones) syncGroups(ctx context.Context, iatas []string, groups map[string]zoneGroup, zones map[string]zoneEntry) error { known := make(map[string]bool, len(iatas)) for _, iata := range iatas { known[iata] = true @@ -69,7 +130,8 @@ func (z *Zones) syncGroups(ctx context.Context, iatas []string) error { var want []RegionState for code, g := range groups { if slices.ContainsFunc(g.members, func(m string) bool { return known[m] }) { - want = append(want, RegionState{Slug: strings.ToLower(code), Name: g.name, Imported: true, IATAs: g.members}) + lat, lng := center(g.members, zones) + want = append(want, RegionState{Slug: strings.ToLower(code), Name: g.name, Imported: true, IATAs: g.members, CenterLat: lat, CenterLng: lng}) } } slices.SortFunc(want, func(a, b RegionState) int { return cmp.Or(cmp.Compare(a.Name, b.Name), cmp.Compare(a.Slug, b.Slug)) }) @@ -79,7 +141,8 @@ func (z *Zones) syncGroups(ctx context.Context, iatas []string) error { if cur, ok := stored[r.Slug]; ok && !cur.Imported { slog.Info("MeshMapper group skipped: a configured region uses its slug", "component", "meshmapper.zones", "slug", r.Slug) continue - } else if ok && cur.Name == r.Name && cur.DisplayOrder == r.DisplayOrder && slices.Equal(cur.IATAs, r.IATAs) { + } else if ok && cur.Name == r.Name && cur.DisplayOrder == r.DisplayOrder && slices.Equal(cur.IATAs, r.IATAs) && + sameFloat(cur.CenterLat, r.CenterLat) && sameFloat(cur.CenterLng, r.CenterLng) { keep = append(keep, r.Slug) continue } @@ -102,10 +165,29 @@ func (z *Zones) syncGroups(ctx context.Context, iatas []string) error { if changed || len(pruned) > 0 { z.regionsChanged(ctx) } - z.groupsSynced, z.groupsVersion, z.groupsKnown = true, version, len(iatas) return nil } +// center averages the members' zone locations; nil when none has one. +func center(members []string, zones map[string]zoneEntry) (*float64, *float64) { + var lat, lng float64 + n := 0 + for _, m := range members { + if z := zones[m]; z.lat != nil { + lat, lng, n = lat+*z.lat, lng+*z.lon, n+1 + } + } + if n == 0 { + return nil, nil + } + lat, lng = lat/float64(n), lng/float64(n) + return &lat, &lng +} + +func sameFloat(a, b *float64) bool { + return a == nil && b == nil || a != nil && b != nil && *a == *b +} + func (z *Zones) regionsRemoved(slugs []string) { for _, slug := range slugs { slog.Info("MeshMapper region removed", "component", "meshmapper.zones", "slug", slug) diff --git a/internal/meshmapper/groups_test.go b/internal/meshmapper/groups_test.go index e6328888..4caefa3c 100644 --- a/internal/meshmapper/groups_test.go +++ b/internal/meshmapper/groups_test.go @@ -13,7 +13,10 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/config" ) -const groupedList = `{"country":"CA","zones":[{"code":"YOW","url":"URL/","has_boundary":true,"group":"ONQC"}],"groups":[` + +const groupedList = `{"country":"CA","zones":[` + + `{"code":"YOW","name":"Ottawa, CA","short_name":"Ottawa","lat":45,"lon":-76,"url":"URL/","has_boundary":true,"group":"ONQC"},` + + `{"code":"YUL","name":"Montreal, CA","lat":46,"lon":-74,"url":"URL/","has_boundary":true,"group":"ONQC"},` + + `{"code":"YQB","name":"Quebec City, CA","url":"URL/","has_boundary":true,"group":"ONQC"}],"groups":[` + `{"code":"ONQC","name":"Ottawa-Quebec Corridor","members":["YUL","YOW","YQB"],"url":"https://onqc.meshmapper.net/"},` + `{"code":"GOLM","name":"Great Ontario Lake Mesh","members":["YYZ","YKF"],"url":"https://golm.meshmapper.net/"}]}` @@ -76,6 +79,12 @@ func TestGroupImportedWhenAMemberIsKnown(t *testing.T) { if _, ok := h.store.regions["golm"]; ok { t.Fatal("group with no known member imported") } + if got.CenterLat == nil || *got.CenterLat != 45.5 || *got.CenterLng != -75 { + t.Fatalf("center should average the located members: %v, %v", got.CenterLat, got.CenterLng) + } + if !slices.Contains(h.store.iatas, "YQB") || !slices.Contains(h.store.iatas, "YUL") { + t.Fatal("members not created as IATAs", h.store.iatas) + } if h.invalidated != 1 { t.Fatal("region caches not invalidated", h.invalidated) } @@ -144,3 +153,32 @@ func TestGroupColdStartWaitsForEveryList(t *testing.T) { t.Fatal("imported regions pruned before any list loaded") } } + +func TestZoneListNamesAndLocatesIATAs(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{iatas: []string{"YOW", "YUL", "YQB", "YYZ"}}, false) + h.z.SetConfiguredIATAs([]string{"YUL"}) + changed := 0 + h.z.OnIATAsChange(func(context.Context) { changed++ }) + h.refetch(t) + yow := h.store.details["YOW"] + if yow.Name != "Ottawa" || yow.Lat == nil || *yow.Lat != 45 || *yow.Lng != -76 { + t.Fatalf("YOW not named and located from short_name: %+v", yow) + } + if yqb := h.store.details["YQB"]; yqb.Name != "Quebec City, CA" || yqb.Lat != nil { + t.Fatalf("YQB should fall back to name and stay unlocated: %+v", yqb) + } + if _, ok := h.store.details["YUL"]; ok { + t.Fatal("configured IATA overwritten") + } + if _, ok := h.store.details["YYZ"]; ok { + t.Fatal("unlisted IATA written") + } + if changed != 1 || h.store.writes != 2 { + t.Fatal("writes/invalidations", h.store.writes, changed) + } + h.z.groupsSynced = false + h.refetch(t) + if h.store.writes != 2 || changed != 1 { + t.Fatal("unchanged details rewritten", h.store.writes, changed) + } +} diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go index d14ab4cd..20b99ff1 100644 --- a/internal/meshmapper/zones.go +++ b/internal/meshmapper/zones.go @@ -47,6 +47,8 @@ type ZoneStore interface { ListZoneBoundaries(ctx context.Context) ([]Boundary, error) SaveZoneBoundary(ctx context.Context, b Boundary) error ListRegionState(ctx context.Context) ([]RegionState, error) + ListIATADetails(ctx context.Context) ([]IATADetails, error) + UpsertIATADetails(ctx context.Context, iata, name string, lat, lng *float64) error SaveImportedRegion(ctx context.Context, r RegionState) (bool, error) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) } @@ -73,6 +75,8 @@ type Zones struct { groupsSynced bool groupsVersion, groupsKnown int onRegions func(ctx context.Context) + configured map[string]bool + onIATAs func(ctx context.Context) } func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore, dir *Directory) *Zones { @@ -170,7 +174,7 @@ func (z *Zones) Refresh(ctx context.Context) (err error) { return fmt.Errorf("list IATAs for MeshMapper boundaries: %w", err) } z.track(iatas, nil) - if err := z.syncGroups(ctx, iatas); err != nil { + if err := z.syncDirectory(ctx, iatas); err != nil { return err } for _, r := range z.regions { diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go index bf141208..dc272744 100644 --- a/internal/meshmapper/zones_test.go +++ b/internal/meshmapper/zones_test.go @@ -31,6 +31,27 @@ type zoneMemoryStore struct { fail bool regions map[string]RegionState lists *zoneListMemory + details map[string]IATADetails + writes int +} + +func (s *zoneMemoryStore) ListIATADetails(context.Context) ([]IATADetails, error) { + var out []IATADetails + for _, iata := range s.iatas { + d := s.details[iata] + d.IATA = iata + out = append(out, d) + } + return out, nil +} + +func (s *zoneMemoryStore) UpsertIATADetails(_ context.Context, iata, name string, lat, lng *float64) error { + if s.details == nil { + s.details = map[string]IATADetails{} + } + s.details[iata] = IATADetails{IATA: iata, Name: name, Lat: lat, Lng: lng} + s.writes++ + return nil } func (s *zoneMemoryStore) ListRegionState(context.Context) ([]RegionState, error) { @@ -50,6 +71,11 @@ func (s *zoneMemoryStore) SaveImportedRegion(_ context.Context, r RegionState) ( } r.Imported = true s.regions[r.Slug] = r + for _, m := range r.IATAs { // members become known IATAs, as AddIATAs does + if !slices.Contains(s.iatas, m) { + s.iatas = append(s.iatas, m) + } + } return true, nil } From 27ca84f51eb53fc137303b9244bc7580f57c6a49 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Thu, 1 Oct 2026 06:05:42 -0400 Subject: [PATCH 121/141] fix(meshmapper): allow MeshMapper's 60s request timeout --- internal/meshmapper/channels.go | 4 ++-- internal/meshmapper/directory.go | 2 +- internal/meshmapper/scopes.go | 4 ++-- internal/meshmapper/zones.go | 8 ++++++-- internal/meshmapper/zones_test.go | 27 +++++++++++++++++++++++++++ 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/internal/meshmapper/channels.go b/internal/meshmapper/channels.go index 934af8aa..4ead380d 100644 --- a/internal/meshmapper/channels.go +++ b/internal/meshmapper/channels.go @@ -65,7 +65,7 @@ type Channels struct { func NewChannels(ctx context.Context, cfg config.MeshMapperChannelsConfig, store ChannelStore, dir *Directory, keys ChannelKeys) (*Channels, error) { c := &Channels{store: store, keys: keys, dir: dir, channelsURL: channelsEndpoint, interval: cfg.Interval(), seen: map[string]bool{}, client: &http.Client{ - Timeout: 10 * time.Second, + Timeout: requestTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }} if !cfg.Enabled { @@ -130,7 +130,7 @@ func (c *Channels) Refresh(ctx context.Context) (err error) { err = nil } }() - ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + ctx, cancel := context.WithTimeout(ctx, refreshTimeout) defer cancel() now := time.Now().UTC() iatas, err := c.store.ListKnownIATAs(ctx) diff --git a/internal/meshmapper/directory.go b/internal/meshmapper/directory.go index 4c1fedd4..20f7503f 100644 --- a/internal/meshmapper/directory.go +++ b/internal/meshmapper/directory.go @@ -64,7 +64,7 @@ type Directory struct { func NewDirectory(store DirectoryStore) *Directory { return &Directory{store: store, listURL: ZonesURL, lists: map[string]*zoneList{}, client: &http.Client{ - Timeout: 30 * time.Second, + Timeout: requestTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }} } diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index a4171be9..3ecdcbe1 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -79,7 +79,7 @@ func (i *Importer) SetCacheInvalidator(fn func(context.Context)) { i.onChange = func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, dir *Directory, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) { i := &Importer{store: store, scopes: scopes, manual: manual, dir: dir, scopesURL: scopesEndpoint, seen: map[string]bool{}, interval: cfg.Interval(), client: &http.Client{ - Timeout: 10 * time.Second, + Timeout: requestTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }} if !cfg.Enabled { @@ -142,7 +142,7 @@ func (i *Importer) Refresh(ctx context.Context) (err error) { err = nil } }() - ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + ctx, cancel := context.WithTimeout(ctx, refreshTimeout) defer cancel() now := time.Now().UTC() iatas, err := i.store.ListKnownIATAs(ctx) diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go index 20b99ff1..1310b712 100644 --- a/internal/meshmapper/zones.go +++ b/internal/meshmapper/zones.go @@ -26,6 +26,10 @@ const ( MaxZoneList = 1 << 20 MaxBoundaryBody = 8 << 20 // outlines are never simplified + // MeshMapper asks clients to allow 60-120s; a request given up on may still use the day's call. + requestTimeout = 60 * time.Second + refreshTimeout = requestTimeout + 30*time.Second // room for the DB write after a slow request + zoneListFresh = config.MinZonesRefresh // get_zones.php allows one request per country per 23.5h zoneFailureRetry = config.MinZonesRefresh // failed requests count too ) @@ -82,7 +86,7 @@ type Zones struct { func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore, dir *Directory) *Zones { return &Zones{store: store, enabled: cfg.Enabled, importGroups: cfg.Enabled && cfg.ImportGroups, seen: map[string]bool{}, interval: cfg.Interval(), dir: dir, boundsURL: boundaryEndpoint, client: &http.Client{ - Timeout: 30 * time.Second, + Timeout: requestTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }} } @@ -165,7 +169,7 @@ func (z *Zones) Refresh(ctx context.Context) (err error) { err = nil } }() - ctx, cancel := context.WithTimeout(ctx, time.Minute) + ctx, cancel := context.WithTimeout(ctx, refreshTimeout) defer cancel() now := time.Now().UTC() // IATAs are created from traffic, so pick up new ones every tick. diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go index dc272744..bdf4589e 100644 --- a/internal/meshmapper/zones_test.go +++ b/internal/meshmapper/zones_test.go @@ -15,6 +15,8 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) const square = `{"type":"Polygon","coordinates":[[[-76,45],[-75,45],[-75,46],[-76,46],[-76,45]]]}` @@ -465,3 +467,28 @@ func TestDecodeBoundaryRejectsUnusableOutlines(t *testing.T) { t.Fatal("case-insensitive code rejected", err) } } + +func TestClientsAllowMeshMapperTimeout(t *testing.T) { + ctx := context.Background() + scopes, err := New(ctx, config.MeshMapperScopesConfig{}, nil, NewDirectory(nil), scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + channels, err := NewChannels(ctx, config.MeshMapperChannelsConfig{}, newChannelMemoryStore(), NewDirectory(nil), keystore.NewMapKeyStore(nil)) + if err != nil { + t.Fatal(err) + } + for name, c := range map[string]*http.Client{ + "directory": NewDirectory(nil).client, + "zones": NewZones(config.MeshMapperZonesConfig{}, nil, nil).client, + "scopes": scopes.client, + "channels": channels.client, + } { + if c.Timeout < 60*time.Second { + t.Errorf("%s client gives up after %v; MeshMapper asks for 60-120s", name, c.Timeout) + } + } + if refreshTimeout <= requestTimeout { + t.Fatal("a refresh must outlast its request") + } +} From 119be44c0d8ae87eb34b6153b5a38ae5a10dd563 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Thu, 1 Oct 2026 08:35:11 -0400 Subject: [PATCH 122/141] feat(channels): filter channel list by keyKnown Also seed the cursor integration test with MeshMapper membership; it still used channel_iatas after channel lists moved to region scoping. --- db/channel_cursor_integration_test.go | 20 ++++++--- db/channels.go | 6 +-- db/channels_test.go | 34 ++++++++++++-- db/meshmapper_integration_test.go | 2 +- db/queries/queries.sql | 4 +- db/sqlc/querier.go | 2 +- db/sqlc/queries.sql.go | 14 ++++-- docs/docs.go | 6 +++ docs/swagger.json | 6 +++ docs/swagger.yaml | 5 +++ internal/api/handlers/channels.go | 15 ++++++- internal/api/handlers/channels_test.go | 55 +++++++++++++++++++++-- internal/api/handlers/limits_test.go | 2 +- internal/api/handlers/stub_reader_test.go | 6 +-- internal/api/reader.go | 4 +- internal/cache/cache_test.go | 2 +- internal/cache/reader.go | 4 +- 17 files changed, 153 insertions(+), 34 deletions(-) diff --git a/db/channel_cursor_integration_test.go b/db/channel_cursor_integration_test.go index d748f2aa..a25eb9f7 100644 --- a/db/channel_cursor_integration_test.go +++ b/db/channel_cursor_integration_test.go @@ -57,13 +57,14 @@ func TestChannelCursorPostgres(t *testing.T) { } } applyBaseline(t, ctx, tx) - exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen) VALUES - (1,'\xaa','\x01','2026-09-08 12:00:00+00'),(2,'\xaa','\x02','2026-09-08 12:00:00+00'), - (3,'\xaa','\x03','2026-09-08 12:00:00+00'),(4,'\xbb','\x04','2026-09-08 11:59:59+00'), - (5,'\xcc','\x05','2026-09-08 12:00:00.000321+00'), - (6,'\xcc','\x06','2026-09-08 12:00:00.000111+00'),(7,'\xcc','\x07','2026-09-08 12:00:00.000111+00'); -INSERT INTO channel_iatas(channel_hash,iata,last_heard) VALUES - ('\xaa','YOW',now()),('\xbb','YOW',now()),('\xcc','YYZ',now());`) + // NULL key_known is reported as false, so keyKnown=false must include it. + exec(`INSERT INTO channels(id,channel_hash,key_fingerprint,last_seen,key_known) VALUES + (1,'\xaa','\x01','2026-09-08 12:00:00+00',true),(2,'\xaa','\x02','2026-09-08 12:00:00+00',false), + (3,'\xaa','\x03','2026-09-08 12:00:00+00',NULL),(4,'\xbb','\x04','2026-09-08 11:59:59+00',true), + (5,'\xcc','\x05','2026-09-08 12:00:00.000321+00',true), + (6,'\xcc','\x06','2026-09-08 12:00:00.000111+00',false),(7,'\xcc','\x07','2026-09-08 12:00:00.000111+00',true); +INSERT INTO meshmapper_channel_members(iata,key_fingerprint) VALUES + ('YOW','\x01'),('YOW','\x02'),('YOW','\x03'),('YOW','\x04'),('YYZ','\x05'),('YYZ','\x06'),('YYZ','\x07');`) queries := &channelCursorQueries{DBTX: tx} router := handlers.ChannelsRouter(&Store{q: sqlc.New(queries)}) // Decode the wire contract so the same regression runs against the old server. @@ -97,6 +98,11 @@ INSERT INTO channel_iatas(channel_hash,iata,last_heard) VALUES {"hash and region", "hash=cc&iata=yyz", []int{5, 7, 6}}, {"no regional match", "hash=aa&iata=yyz", nil}, {"missing region", "iata=ZZZ", nil}, + {"key known", "keyKnown=true", []int{5, 7, 1, 4}}, + {"key unknown", "keyKnown=false", []int{6, 3, 2}}, + {"key known in region", "keyKnown=true&iata=yow", []int{1, 4}}, + {"key unknown in region", "keyKnown=false&iata=yow", []int{3, 2}}, + {"key unknown hash", "keyKnown=false&hash=cc", []int{6}}, } { t.Run(tc.name, func(t *testing.T) { params, _ := url.ParseQuery(tc.query) diff --git a/db/channels.go b/db/channels.go index a4718b58..3e07032d 100644 --- a/db/channels.go +++ b/db/channels.go @@ -102,7 +102,7 @@ func (s *Store) DeleteOldChannelIATAs(ctx context.Context, cutoff time.Time) err return s.q.DeleteOldChannelIATAs(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } -func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { +func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { var cursorTS pgtype.Timestamptz if cursor > 0 { cursorTS = pgtype.Timestamptz{Time: time.UnixMilli(cursor), Valid: true} @@ -111,12 +111,12 @@ func (s *Store) ListChannels(ctx context.Context, limit int32, hash []byte, iata var err error if pageCursor != nil { rows, err = s.q.ListChannelsAfter(ctx, sqlc.ListChannelsAfterParams{ - ChannelHash: hash, Iatas: iatas, PageLimit: limit + 1, + ChannelHash: hash, Iatas: iatas, KeyKnown: keyKnown, PageLimit: limit + 1, CursorTs: pgtype.Timestamptz{Time: pageCursor.LastSeen, Valid: true}, CursorID: pageCursor.ID, }) } else { rows, err = s.q.ListChannels(ctx, sqlc.ListChannelsParams{ - ChannelHash: hash, Iatas: iatas, CursorTs: cursorTS, PageLimit: limit + 1, + ChannelHash: hash, Iatas: iatas, KeyKnown: keyKnown, CursorTs: cursorTS, PageLimit: limit + 1, }) } if err != nil { diff --git a/db/channels_test.go b/db/channels_test.go index 0f51c631..9ba475c0 100644 --- a/db/channels_test.go +++ b/db/channels_test.go @@ -11,6 +11,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/jackc/pgx/v5/pgtype" "go.uber.org/mock/gomock" ) @@ -29,7 +30,7 @@ func TestListChannels_Empty(t *testing.T) { Return([]sqlc.Channel{}, nil) store := &Store{q: mock} - page, err := store.ListChannels(context.Background(), 10, nil, nil, 0, nil) + page, err := store.ListChannels(context.Background(), 10, nil, nil, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -71,7 +72,7 @@ func TestListChannels_Pagination(t *testing.T) { Return(rows, nil) store := &Store{q: mock} - page, err := store.ListChannels(context.Background(), 2, nil, nil, 0, nil) + page, err := store.ListChannels(context.Background(), 2, nil, nil, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -95,7 +96,7 @@ func TestListChannels_DBError(t *testing.T) { Return(nil, errors.New("db error")) store := &Store{q: mock} - _, err := store.ListChannels(context.Background(), 10, nil, nil, 0, nil) + _, err := store.ListChannels(context.Background(), 10, nil, nil, nil, 0, nil) if err == nil { t.Fatal("expected error, got nil") } @@ -115,12 +116,37 @@ func TestListChannels_IATAFilter(t *testing.T) { Return([]sqlc.Channel{}, nil) store := &Store{q: mock} - _, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, 0, nil) + _, err := store.ListChannels(context.Background(), 10, nil, []string{"YOW", "YYZ"}, nil, 0, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } } +func TestListChannels_KeyKnownFilter(t *testing.T) { + ctrl := gomock.NewController(t) + mock := mockdb.NewMockQuerier(ctrl) + keyKnown := false + at := time.UnixMilli(1700000000000) + + mock.EXPECT(). + ListChannels(gomock.Any(), sqlc.ListChannelsParams{KeyKnown: &keyKnown, PageLimit: 11}). + Return([]sqlc.Channel{}, nil) + mock.EXPECT(). + ListChannelsAfter(gomock.Any(), sqlc.ListChannelsAfterParams{ + KeyKnown: &keyKnown, PageLimit: 11, + CursorTs: pgtype.Timestamptz{Time: at, Valid: true}, CursorID: 7, + }). + Return([]sqlc.Channel{}, nil) + + store := &Store{q: mock} + if _, err := store.ListChannels(context.Background(), 10, nil, nil, &keyKnown, 0, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, err := store.ListChannels(context.Background(), 10, nil, nil, &keyKnown, 0, &api.ChannelCursor{LastSeen: at, ID: 7}); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + func TestGetChannel_Basic(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go index 0df03238..5b9af836 100644 --- a/db/meshmapper_integration_test.go +++ b/db/meshmapper_integration_test.go @@ -410,7 +410,7 @@ func TestChannelRegionScopingPostgres(t *testing.T) { } names := func(iatas ...string) string { t.Helper() - page, err := store.ListChannels(ctx, 50, nil, iatas, 0, nil) + page, err := store.ListChannels(ctx, 50, nil, iatas, nil, 0, nil) if err != nil { t.Fatal(err) } diff --git a/db/queries/queries.sql b/db/queries/queries.sql index bae2646a..161dc193 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -900,7 +900,7 @@ WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour'; -- name: ListChannels :many -- Channels ordered by last seen, optionally filtered by hash and/or IATAs. -- A channel belongs to an IATA when MeshMapper lists it there or config scopes it --- to a region containing it (or Beacon-wide). NULL hash / empty array skip those filters. +-- to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters. -- Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). SELECT c.* FROM channels c WHERE (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) @@ -911,6 +911,7 @@ WHERE (@channel_hash::bytea IS NULL OR c.channel_hash = @channel_hash) WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id WHERE ri.iata = ANY(@iatas::bpchar[]))))) + AND (sqlc.narg(key_known)::boolean IS NULL OR COALESCE(c.key_known, false) = sqlc.narg(key_known)) AND (@cursor_ts::timestamptz IS NULL OR c.last_seen < @cursor_ts) ORDER BY c.last_seen DESC, c.id DESC LIMIT @page_limit; @@ -928,6 +929,7 @@ WHERE (c.last_seen, c.id) < (@cursor_ts::timestamptz, @cursor_id::integer) WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id WHERE ri.iata = ANY(@iatas::bpchar[]))))) + AND (sqlc.narg(key_known)::boolean IS NULL OR COALESCE(c.key_known, false) = sqlc.narg(key_known)) ORDER BY c.last_seen DESC, c.id DESC LIMIT @page_limit; diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 94b398a6..72e426ca 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -164,7 +164,7 @@ type Querier interface { ListChannelMessagesByHash(ctx context.Context, arg ListChannelMessagesByHashParams) ([]ListChannelMessagesByHashRow, error) // Channels ordered by last seen, optionally filtered by hash and/or IATAs. // A channel belongs to an IATA when MeshMapper lists it there or config scopes it - // to a region containing it (or Beacon-wide). NULL hash / empty array skip those filters. + // to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters. // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) // Keep the non-null tuple boundary separate from the legacy optional cursor so diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 453be2f8..178af5f3 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2638,26 +2638,29 @@ WHERE ($1::bytea IS NULL OR c.channel_hash = $1) WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id WHERE ri.iata = ANY($2::bpchar[]))))) - AND ($3::timestamptz IS NULL OR c.last_seen < $3) + AND ($3::boolean IS NULL OR COALESCE(c.key_known, false) = $3) + AND ($4::timestamptz IS NULL OR c.last_seen < $4) ORDER BY c.last_seen DESC, c.id DESC -LIMIT $4 +LIMIT $5 ` type ListChannelsParams struct { ChannelHash []byte `json:"channel_hash"` Iatas []string `json:"iatas"` + KeyKnown *bool `json:"key_known"` CursorTs pgtype.Timestamptz `json:"cursor_ts"` PageLimit int32 `json:"page_limit"` } // Channels ordered by last seen, optionally filtered by hash and/or IATAs. // A channel belongs to an IATA when MeshMapper lists it there or config scopes it -// to a region containing it (or Beacon-wide). NULL hash / empty array skip those filters. +// to a region containing it (or Beacon-wide). NULL hash / empty array / NULL key_known skip those filters. // Pass cursor=0 to start from the beginning (cursor is last_seen epoch ms). func (q *Queries) ListChannels(ctx context.Context, arg ListChannelsParams) ([]Channel, error) { rows, err := q.db.Query(ctx, listChannels, arg.ChannelHash, arg.Iatas, + arg.KeyKnown, arg.CursorTs, arg.PageLimit, ) @@ -2702,8 +2705,9 @@ WHERE (c.last_seen, c.id) < ($1::timestamptz, $2::integer) WHERE s.key_fingerprint = c.key_fingerprint AND (s.region_slug IS NULL OR s.region_slug IN ( SELECT r.slug FROM regions r JOIN region_iatas ri ON ri.region_id = r.id WHERE ri.iata = ANY($4::bpchar[]))))) + AND ($5::boolean IS NULL OR COALESCE(c.key_known, false) = $5) ORDER BY c.last_seen DESC, c.id DESC -LIMIT $5 +LIMIT $6 ` type ListChannelsAfterParams struct { @@ -2711,6 +2715,7 @@ type ListChannelsAfterParams struct { CursorID int32 `json:"cursor_id"` ChannelHash []byte `json:"channel_hash"` Iatas []string `json:"iatas"` + KeyKnown *bool `json:"key_known"` PageLimit int32 `json:"page_limit"` } @@ -2722,6 +2727,7 @@ func (q *Queries) ListChannelsAfter(ctx context.Context, arg ListChannelsAfterPa arg.CursorID, arg.ChannelHash, arg.Iatas, + arg.KeyKnown, arg.PageLimit, ) if err != nil { diff --git a/docs/docs.go b/docs/docs.go index 6e59a488..15dc6a7e 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -607,6 +607,12 @@ const docTemplate = `{ "name": "iatas", "in": "query" }, + { + "type": "boolean", + "description": "Only channels Beacon can (true) or cannot (false) decrypt; omit for all", + "name": "keyKnown", + "in": "query" + }, { "type": "integer", "description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning", diff --git a/docs/swagger.json b/docs/swagger.json index 843fd6b7..25af0674 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -605,6 +605,12 @@ "name": "iatas", "in": "query" }, + { + "type": "boolean", + "description": "Only channels Beacon can (true) or cannot (false) decrypt; omit for all", + "name": "keyKnown", + "in": "query" + }, { "type": "integer", "description": "last_seen epoch ms of last item for pagination; 0 starts from the beginning", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index dda333fc..3f601a76 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1913,6 +1913,11 @@ paths: in: query name: iatas type: string + - description: Only channels Beacon can (true) or cannot (false) decrypt; omit + for all + in: query + name: keyKnown + type: boolean - description: last_seen epoch ms of last item for pagination; 0 starts from the beginning in: query diff --git a/internal/api/handlers/channels.go b/internal/api/handlers/channels.go index 46f312de..9bf80d59 100644 --- a/internal/api/handlers/channels.go +++ b/internal/api/handlers/channels.go @@ -37,6 +37,7 @@ func ChannelsRouter(reader api.Reader) http.Handler { // @Param hash query string false "Single-byte channel hash (hex)" // @Param iata query string false "Filter by IATA code: channels MeshMapper lists there, config channels scoped to a region containing it, and Beacon-wide config channels" // @Param iatas query string false "Filter by IATA code(s), comma-separated e.g. YOW or YOW,YYZ; same membership rule as iata" +// @Param keyKnown query bool false "Only channels Beacon can (true) or cannot (false) decrypt; omit for all" // @Param cursor query int false "last_seen epoch ms of last item for pagination; 0 starts from the beginning" // @Param pageCursor query string false "Opaque nextPageCursor from a previous response; preserves timestamp ties and precision. Cannot be combined with a positive cursor; cursor=0 is allowed." // @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) @@ -86,7 +87,19 @@ func listChannels(reader api.Reader) http.HandlerFunc { } hashHex = h } - channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, cursor, pageCursor) + var keyKnown *bool + if q := r.URL.Query(); q.Has("keyKnown") { + switch q.Get("keyKnown") { + case "true": + keyKnown = new(true) + case "false": + keyKnown = new(false) + default: + respondError(w, http.StatusBadRequest, "keyKnown must be true or false") + return + } + } + channels, err := reader.ListChannels(r.Context(), limit, hashHex, iatas, keyKnown, cursor, pageCursor) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/channels_test.go b/internal/api/handlers/channels_test.go index 8b964d7f..00bc5d79 100644 --- a/internal/api/handlers/channels_test.go +++ b/internal/api/handlers/channels_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "strings" "testing" "time" @@ -39,7 +40,7 @@ func TestListChannels_InvalidCursor(t *testing.T) { func TestListChannels_PageCursor(t *testing.T) { called := false - r := ChannelsRouter(stubReader{listChannels: func(_ context.Context, limit int32, hash []byte, iatas []string, legacy int64, cursor *api.ChannelCursor) (api.ChannelPage, error) { + r := ChannelsRouter(stubReader{listChannels: func(_ context.Context, limit int32, hash []byte, iatas []string, _ *bool, legacy int64, cursor *api.ChannelCursor) (api.ChannelPage, error) { called = true if limit != 2 || legacy != 0 || cursor == nil || cursor.ID != 9 || !cursor.LastSeen.Equal(time.UnixMicro(1700000000000123)) || !reflect.DeepEqual(hash, []byte{0xaa}) || !reflect.DeepEqual(iatas, []string{"YOW"}) { @@ -145,7 +146,7 @@ func TestListChannelMessages_InvalidCursor(t *testing.T) { func TestListChannels_OK(t *testing.T) { r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{ - listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { return api.ChannelPage{Page: api.Page[api.ChannelSummary]{Items: []api.ChannelSummary{{ID: 1, ChannelHash: "ab"}}}}, nil }, })) @@ -172,7 +173,7 @@ func TestListChannels_IATAParsing(t *testing.T) { var got []string r := chi.NewRouter() r.Get("/channels", listChannels(stubReader{ - listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + listChannels: func(_ context.Context, _ int32, _ []byte, iatas []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { got = iatas return api.ChannelPage{}, nil }, @@ -190,6 +191,54 @@ func TestListChannels_IATAParsing(t *testing.T) { } } +func TestListChannels_KeyKnownParsing(t *testing.T) { + yes, no := true, false + cases := []struct { + name string + query string + code int + want *bool + }{ + {"true", "?keyKnown=true", http.StatusOK, &yes}, + {"false", "?keyKnown=false", http.StatusOK, &no}, + {"absent", "", http.StatusOK, nil}, + {"uppercase", "?keyKnown=TRUE", http.StatusBadRequest, nil}, + {"numeric", "?keyKnown=1", http.StatusBadRequest, nil}, + {"empty", "?keyKnown=", http.StatusBadRequest, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var got *bool + called := false + r := chi.NewRouter() + r.Get("/channels", listChannels(stubReader{ + listChannels: func(_ context.Context, _ int32, _ []byte, _ []string, keyKnown *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { + called = true + got = keyKnown + return api.ChannelPage{}, nil + }, + })) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/channels"+tc.query, nil)) + if w.Code != tc.code { + t.Fatalf("expected %d, got %d", tc.code, w.Code) + } + if tc.code != http.StatusOK { + if called { + t.Error("reader called for invalid keyKnown") + } + if !strings.Contains(w.Body.String(), "keyKnown must be true or false") { + t.Errorf("unexpected error body %s", w.Body.String()) + } + return + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("expected keyKnown %v, got %v", tc.want, got) + } + }) + } +} + func TestGetChannel_OK(t *testing.T) { r := chi.NewRouter() r.Get("/channels/{channelID}", getChannel(stubReader{ diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go index 75fc8cbe..3d7e3a28 100644 --- a/internal/api/handlers/limits_test.go +++ b/internal/api/handlers/limits_test.go @@ -19,7 +19,7 @@ func TestListLimits(t *testing.T) { var got int32 var calls int reader := stubReader{ - listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { + listChannels: func(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { calls++ got = limit return api.ChannelPage{}, nil diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 0c719aca..3d3626cc 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -24,7 +24,7 @@ type stubReader struct { listRegions func(ctx context.Context) ([]api.RegionSummary, error) getRegion func(ctx context.Context, regionID int32) (*api.Region, error) getRegionBySlug func(ctx context.Context, slug string) (*api.Region, error) - listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) + listChannels func(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) getChannel func(ctx context.Context, channelID int32) (*api.Channel, error) listChannelMessages func(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) listChannelMessagesByHash func(ctx context.Context, hash []byte, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) @@ -123,9 +123,9 @@ func (s stubReader) GetRegionBySlug(ctx context.Context, slug string) (*api.Regi return nil, nil } -func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { +func (s stubReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { if s.listChannels != nil { - return s.listChannels(ctx, limit, hash, iatas, cursor, pageCursor) + return s.listChannels(ctx, limit, hash, iatas, keyKnown, cursor, pageCursor) } return api.ChannelPage{}, nil } diff --git a/internal/api/reader.go b/internal/api/reader.go index 21d65403..ba6c7cd5 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -53,11 +53,11 @@ type Reader interface { // ListChannels returns a paginated list of channels ordered by last seen. // Includes both hashtag-derived and explicit key channels. // Pass nil hash to skip hash filtering. Pass empty iatas to return all channels; - // IATAs must be uppercase. + // IATAs must be uppercase. Nil keyKnown skips the decryption-key filter. // cursor is last_seen epoch ms of the last item; pass 0 to start from the beginning. // pageCursor is the precise timestamp/ID boundary; nil preserves the legacy numeric cursor. // When non-nil, pageCursor takes precedence over cursor. - ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *ChannelCursor) (ChannelPage, error) + ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *ChannelCursor) (ChannelPage, error) // GetChannel returns full detail for a single channel by its integer ID. // Returns nil, pgx.ErrNoRows if the channel is not found. diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 41dde518..467cfcf4 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -164,7 +164,7 @@ func (s *stubReader) GetCrossIATANeighbors(_ context.Context, _ uuid.UUID, _ str return nil, nil } -func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { +func (s *stubReader) ListChannels(_ context.Context, _ int32, _ []byte, _ []string, _ *bool, _ int64, _ *api.ChannelCursor) (api.ChannelPage, error) { return api.ChannelPage{}, nil } diff --git a/internal/cache/reader.go b/internal/cache/reader.go index e585f93d..e845e622 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -419,8 +419,8 @@ func (cr *CachedReader) GetCrossIATANeighbors(ctx context.Context, nodeID uuid.U } // ListChannels implements [api.Reader]. -func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { - return cr.inner.ListChannels(ctx, limit, hash, iatas, cursor, pageCursor) +func (cr *CachedReader) ListChannels(ctx context.Context, limit int32, hash []byte, iatas []string, keyKnown *bool, cursor int64, pageCursor *api.ChannelCursor) (api.ChannelPage, error) { + return cr.inner.ListChannels(ctx, limit, hash, iatas, keyKnown, cursor, pageCursor) } // ListChannelMessages implements [api.Reader]. From 6cd03e73daa649a66cf8229d143aac589cd77462 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Thu, 1 Oct 2026 12:11:54 -0400 Subject: [PATCH 123/141] fix(db): refresh saved-route hash prefix on re-hearing --- db/queries/queries.sql | 4 +++- db/route_evidence_integration_test.go | 28 +++++++++++++++++++++++++++ db/sqlc/querier.go | 3 ++- db/sqlc/queries.sql.go | 4 +++- 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 161dc193..65e2ec3f 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1317,10 +1317,12 @@ ORDER BY t.last_heard_at DESC; -- name: UpsertKnownRoute :exec -- Route identity is path_key, an md5 of node_ids computed by the caller. --- On conflict, observation_count and last_seen are bumped. +-- On conflict, observation_count and last_seen are bumped and hash_prefix follows the +-- latest hearing, so evidence matches the hash width the route uses now. INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (iata, path_key) DO UPDATE SET + hash_prefix = EXCLUDED.hash_prefix, last_seen = NOW(), observation_count = known_routes.observation_count + 1; diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go index 138c80ba..a84b534f 100644 --- a/db/route_evidence_integration_test.go +++ b/db/route_evidence_integration_test.go @@ -165,3 +165,31 @@ ANALYZE packet_observations`); err != nil { t.Fatalf("0/1-hop rows entered the route index: %v %v", indexed, err) } } + +func TestRouteEvidenceFollowsPrefixWidthPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) +VALUES (int4send(1),4,0,1,'\x00','\x00',NOW(),NOW()); +INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,path_length_byte) +VALUES (1,int4send(1),'00000000-0000-0000-0000-000000000001','YOW',NOW(),2,2,'\xaa11bb22',4,(1<<6)|2)`); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + ids := []uuid.UUID{uuid.New(), uuid.New()} + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa, 0x11}, {0xbb, 0x22}}, "YOW", 2); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + got, err := store.GetRouteEvidence(ctx, "YOW", hex.EncodeToString(routePathKey(ids)), api.RouteEvidenceQuery{Since: now.Add(-time.Hour), Until: now.Add(time.Minute), Limit: 10}) + if err != nil { + t.Fatal(err) + } + if got.HashSize != 2 || got.PathBytes != "aa11bb22" || len(got.Items) != 1 || got.Route.ObservationCount != 2 { + t.Fatalf("evidence kept the first prefix width: %+v", got) + } +} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 72e426ca..c2adf524 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -329,7 +329,8 @@ type Querier interface { // ROUTES // ============================================================ // Route identity is path_key, an md5 of node_ids computed by the caller. - // On conflict, observation_count and last_seen are bumped. + // On conflict, observation_count and last_seen are bumped and hash_prefix follows the + // latest hearing, so evidence matches the hash width the route uses now. UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error // ============================================================ // NODES diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 178af5f3..30db7f5c 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -5344,6 +5344,7 @@ const upsertKnownRoute = `-- name: UpsertKnownRoute :exec INSERT INTO known_routes (path_key, node_ids, hash_prefix, iata, hop_count) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (iata, path_key) DO UPDATE SET + hash_prefix = EXCLUDED.hash_prefix, last_seen = NOW(), observation_count = known_routes.observation_count + 1 ` @@ -5360,7 +5361,8 @@ type UpsertKnownRouteParams struct { // ROUTES // ============================================================ // Route identity is path_key, an md5 of node_ids computed by the caller. -// On conflict, observation_count and last_seen are bumped. +// On conflict, observation_count and last_seen are bumped and hash_prefix follows the +// latest hearing, so evidence matches the hash width the route uses now. func (q *Queries) UpsertKnownRoute(ctx context.Context, arg UpsertKnownRouteParams) error { _, err := q.db.Exec(ctx, upsertKnownRoute, arg.PathKey, From 98006a93645c9f8b09d2ea441a5776eecb9add02 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 07:41:13 -0400 Subject: [PATCH 124/141] feat(stats): serve historical stats from hourly rollups Stats endpoints read hourly rollup tables filled once per closed hour by one advisory-locked task, instead of materialized views that re-scanned every observation on each refresh. IATA-set tables give exact distinct counts for any IATA set or region. Cleanup holds raw rows for unrolled hours (up to 24h). Adds /stats/series; ingest keeps packet observation counts, observer last IATA, channel message counts and trace tag summaries. 001_baseline.sql is rewritten, so pre-release 2.0 databases must be recreated. See docs/historical-stats.md. --- CONTRIBUTING.md | 2 + README.md | 49 +- SHOULDERS.md | 2 + cmd/beacon/main.go | 17 +- config.yaml.example | 11 +- db/analytics_retention_integration_test.go | 291 ------- db/baseline_integration_test.go | 2 +- db/channel_message_count_integration_test.go | 62 ++ db/channel_scopes_integration_test.go | 1 + db/channels.go | 18 +- db/channels_test.go | 2 +- db/migrations/001_baseline.sql | 819 +++++------------- db/observer_last_iata_integration_test.go | 124 +++ db/observer_metrics_integration_test.go | 6 +- db/observer_retention_integration_test.go | 6 +- db/observers.go | 4 +- db/observers_test.go | 12 +- db/packet_cleanup_integration_test.go | 205 +++++ ...cket_observation_count_integration_test.go | 88 ++ db/packets.go | 15 +- db/packets_integration_test.go | 3 +- db/packets_test.go | 17 +- db/paths.go | 2 - db/paths_integration_test.go | 14 +- db/presence.go | 4 +- db/queries/paths.sql | 8 +- db/queries/queries.sql | 449 +++++----- db/queries/rollup.sql | 279 ++++++ db/queries/series.sql | 91 ++ db/queries/signal.sql | 8 +- db/rollup.go | 202 +++++ db/rollup_integration_test.go | 515 +++++++++++ db/rollup_schema_integration_test.go | 117 +++ db/scope_stats_integration_test.go | 87 +- db/signal.go | 4 - db/signal_integration_test.go | 10 +- db/sqlc/mock/querier.go | 616 +++++++++---- db/sqlc/models.go | 311 +++---- db/sqlc/paths.sql.go | 15 +- db/sqlc/querier.go | 128 ++- db/sqlc/queries.sql.go | 744 ++++++++-------- db/sqlc/rollup.sql.go | 501 +++++++++++ db/sqlc/series.sql.go | 169 ++++ db/sqlc/signal.sql.go | 15 +- db/sqlc_scratch/rollup_obs.sql | 9 + db/stats.go | 167 ++-- db/stats_series.go | 64 ++ db/stats_series_integration_test.go | 126 +++ db/stats_test.go | 72 +- db/store.go | 3 +- db/trace_pagination_integration_test.go | 122 ++- db/traces.go | 15 +- docs/docs.go | 230 ++++- docs/historical-stats.md | 85 ++ docs/swagger.json | 230 ++++- docs/swagger.yaml | 194 ++++- go.mod | 2 +- internal/api/channels.go | 2 +- internal/api/handlers/limits_test.go | 4 +- .../api/handlers/observer_comparison_test.go | 6 +- internal/api/handlers/paths.go | 2 +- internal/api/handlers/paths_test.go | 6 +- internal/api/handlers/regions.go | 9 +- internal/api/handlers/scope_stats_test.go | 7 +- internal/api/handlers/scopes.go | 1 + internal/api/handlers/signal.go | 2 +- internal/api/handlers/signal_test.go | 10 +- internal/api/handlers/stats.go | 142 ++- internal/api/handlers/stats_region_test.go | 90 ++ internal/api/handlers/stats_series.go | 79 ++ internal/api/handlers/stats_series_test.go | 76 ++ internal/api/handlers/stats_test.go | 4 +- internal/api/handlers/stub_reader_test.go | 20 +- internal/api/handlers/traces.go | 4 +- internal/api/reader.go | 7 +- internal/api/router/options.go | 2 + internal/api/router/router.go | 2 +- internal/api/stats.go | 73 +- internal/background/rollup.go | 161 ++++ internal/background/rollup_test.go | 195 +++++ internal/background/tasks.go | 121 ++- internal/background/tasks_test.go | 129 ++- internal/cache/cache.go | 72 +- internal/cache/cache_test.go | 24 +- internal/cache/logging_test.go | 2 +- internal/cache/paths.go | 25 - internal/cache/paths_test.go | 2 + internal/cache/reader.go | 154 +--- internal/cache/rollup_stats.go | 171 ++++ internal/cache/scope_stats_test.go | 6 +- internal/cache/series_test.go | 204 +++++ internal/cache/signal.go | 25 - internal/cache/signal_test.go | 2 + internal/config/config.go | 24 +- internal/config/config_test.go | 65 +- .../ingest/advert_endpoint_update_test.go | 6 +- internal/ingest/backfill.go | 5 +- internal/ingest/backfill_test.go | 29 +- internal/ingest/endpoint_work_test.go | 2 +- internal/ingest/frame_length_test.go | 4 +- internal/ingest/ingest.go | 17 +- internal/ingest/ingest_test.go | 24 +- internal/ingest/neighbors.go | 2 +- internal/ingest/packet.go | 25 +- internal/ingest/packet_endpoints_test.go | 4 +- internal/ingest/repeats_test.go | 27 +- internal/ingest/side_effects.go | 3 +- internal/ingest/side_effects_test.go | 20 +- internal/ingest/status.go | 2 +- internal/presence/coalescer.go | 26 +- internal/presence/coalescer_test.go | 52 +- sqlc.yaml | 10 +- 112 files changed, 6745 insertions(+), 2807 deletions(-) delete mode 100644 db/analytics_retention_integration_test.go create mode 100644 db/channel_message_count_integration_test.go create mode 100644 db/observer_last_iata_integration_test.go create mode 100644 db/packet_cleanup_integration_test.go create mode 100644 db/packet_observation_count_integration_test.go create mode 100644 db/queries/rollup.sql create mode 100644 db/queries/series.sql create mode 100644 db/rollup.go create mode 100644 db/rollup_integration_test.go create mode 100644 db/rollup_schema_integration_test.go create mode 100644 db/sqlc/rollup.sql.go create mode 100644 db/sqlc/series.sql.go create mode 100644 db/sqlc_scratch/rollup_obs.sql create mode 100644 db/stats_series.go create mode 100644 db/stats_series_integration_test.go create mode 100644 docs/historical-stats.md create mode 100644 internal/api/handlers/stats_region_test.go create mode 100644 internal/api/handlers/stats_series.go create mode 100644 internal/api/handlers/stats_series_test.go create mode 100644 internal/background/rollup.go create mode 100644 internal/background/rollup_test.go delete mode 100644 internal/cache/paths.go create mode 100644 internal/cache/rollup_stats.go create mode 100644 internal/cache/series_test.go delete mode 100644 internal/cache/signal.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f859d2fb..3656d682 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,6 +85,8 @@ All schema changes must include a proper migration path: - Re-run `sqlc generate` to regenerate `db/sqlc/` - Update the store layer in `db/` to expose the new functionality - Update `internal/api/reader.go` if the change needs to be exposed via the API +- Historical stats read hourly rollups, never raw observations; follow + [docs/historical-stats.md](docs/historical-stats.md) to add a metric Never edit files under `db/sqlc/` by hand — they are generated by sqlc and will be overwritten. If you need to work around a sqlc limitation, document it diff --git a/README.md b/README.md index 855b9b8a..5e664f69 100644 --- a/README.md +++ b/README.md @@ -258,12 +258,16 @@ scopes: # Observer telemetry storage settings. telemetry: - retention: 672h # how long to keep telemetry snapshots (default: 4 weeks) + retention: 744h # how long to keep telemetry snapshots (default: 31 days) resolution: 1h # snapshot frequency per observer; duplicates within window are dropped (default: 1h) # Packet and observation retention. packets: - retention: 168h # packets, observations, channel messages (default: 7 days) + retention: 168h # packets, observations, channel messages (default: 7 days, minimum 24h) + +# Hourly analytics rollups behind the stats endpoints; they outlive raw packets. +analytics: + rollup_retention: 2160h # also the longest /stats/series window (default: 90 days, minimum 24h) # Presence write coalescing. # Observer last_seen and packet last_heard_at bumps are batched in memory and @@ -302,7 +306,7 @@ observers: cache: ttl: "1h" ttls: - stats: "1h" # stats endpoints (backed by materialized views) + stats: "1h" # stats endpoints (hourly rollups; keys change when a new hour is rolled) reference: "1h" # IATAs, regions, scopes nodes: "1h" # node detail (also explicitly invalidated on upsert) observers: "1h" # observer detail (also explicitly invalidated on upsert) @@ -523,14 +527,15 @@ Not yet implemented — see the Authentication section above. | `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | | `GET` | `/scopes` | List transport scope names; IATA/region filters use configured regions and MeshMapper catalogues | | `GET` | `/scopes/{name}` | Get scope detail | -| `GET` | `/stats/observations` | Hourly observation time series (last 7 days by default) | -| `GET` | `/stats/overview` | Network overview stats | +| `GET` | `/stats/observations` | Hourly observation counts per IATA (last 7 days by default) | +| `GET` | `/stats/overview` | Network overview over the last 24 rolled hours | | `GET` | `/stats/payload-breakdown` | Observation counts by payload type (last 24h by default) | -| `GET` | `/stats/scopes` | Configured region scopes and breakdown of packets, nodes, observers | -| `GET` | `/stats/top-advertisers` | Top N nodes by distinct ADVERT packet count (last 24h by default, from materialized view) | -| `GET` | `/stats/top-nodes` | Top N nodes by observation count (from materialized view) | +| `GET` | `/stats/scopes` | Configured region scopes and breakdown of packets, nodes, observers (last 7 days by default) | +| `GET` | `/stats/series` | Hourly card metrics and window summary from the rollups (`since`/`until` required) | +| `GET` | `/stats/top-advertisers` | Top N nodes by distinct ADVERT packet count (last 24h by default) | +| `GET` | `/stats/top-nodes` | Top N nodes by advert hearings (last 7 days by default) | | `GET` | `/stats/top-observers` | Top N observers by observation count (last 24h by default) | -| `GET` | `/stats/top-talkers` | Top N companion names by decrypted channel message count (last 24h by default, from materialized view) | +| `GET` | `/stats/top-talkers` | Top N companion names by decrypted channel message count (last 24h by default) | | `GET` | `/traces` | List trace tags with filters (optional: ?type=TRACE\|PING) | | `GET` | `/traces/{tag}` | Get full trace detail with resolved routes | @@ -560,20 +565,14 @@ Records include a component field. Ingest workers also include their broker name ### Analytics retention -Hourly traffic, payload, observer activity, talker, advertiser, Signal and Paths -summaries retain 30 days independently of `packets.retention`. Cleanup saves only -aggregates before deleting each packet batch, in the same transaction. Raw -packets, observations and message bodies still expire under packet retention. -The summaries use UTC hourly buckets and appear on the normal view-refresh cycle. -`mv_hourly_iata_stats` now covers 30 days instead of migration 001's seven days; -`/stats/observations?since=` can therefore return retained summaries older than a week. - -Migration 039 starts from data still present; previously deleted history cannot -be reconstructed. Telemetry has its own retention setting. Packet drill-down, -sub-hour observer activity, exact observer comparison and current entity/scope -counts continue to describe retained raw data or current entities, rather than -claiming archived packet detail. Archived summaries expire without waiting for -new packet deletions. A failed archive leaves its entire raw batch intact. +Historical stats (traffic, payload, observer activity, talker, advertiser, node, +scope, Signal and Paths) are read from hourly rollup tables kept for +`analytics.rollup_retention` (default 90 days), independently of `packets.retention`. +One background task rolls each UTC hour about 95 minutes after it closes; +packet cleanup waits for unrolled hours (up to 24h) so they can be rolled in full. +An hour whose raw rows were deleted first is marked partial and reported without values. +Packet drill-down, sub-hour observer activity and observer comparison still read +retained raw data. See [docs/historical-stats.md](docs/historical-stats.md). ### Observer monitoring metrics @@ -590,9 +589,7 @@ The existing observer `observationCount` remains a legacy cumulative presence counter for compatibility, including status/neighbour events. It is not a period packet total. Broker presence and packet-arrival timestamps are now updated separately; this cannot reconstruct previously overwritten timestamps. -The initial analytics archive migration includes unknown-type activity. Existing -development previews that used the earlier draft require a separate operator repair; -radio samples already discarded for those legacy rows cannot be recovered. +Hourly activity comes from the analytics rollups and includes unknown-type activity. ### Saved-route evidence diff --git a/SHOULDERS.md b/SHOULDERS.md index 05076de6..ef2b34e7 100644 --- a/SHOULDERS.md +++ b/SHOULDERS.md @@ -22,6 +22,8 @@ projects that make this possible: - [godotenv](https://github.com/joho/godotenv) — .env file loading - [yaml.v3](https://github.com/go-yaml/yaml) — YAML config parsing - [google/uuid](https://github.com/google/uuid) — UUID generation +- [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) — singleflight for + collapsing concurrent cache misses ## Data diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 3ccbffb6..800c88aa 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -378,9 +378,22 @@ func main() { ingestWorkers.Go(func() { broker1.Start(ctx) }) ingestWorkers.Go(func() { broker2.Start(ctx) }) + rollup := background.NewRollup(store, resolved.PacketRetention, resolved.RollupRetention) + go func() { + if err := rollup.CatchUp(ctx); err != nil { + slog.Error("analytics rollup catch-up failed", "component", "background", "error", err) + } + }() tasks := []background.Task{ + rollup.Task(), background.ViewRefreshTask(store, resolved.ViewRefreshInterval), - background.CleanupTask(store, resolved.TelemetryRetention, resolved.PacketRetention, resolved.NodeDeleteAfter, resolved.CleanupInterval), + background.CleanupTask(store, background.CleanupConfig{ + TelemetryRetention: resolved.TelemetryRetention, + PacketRetention: resolved.PacketRetention, + RollupRetention: resolved.RollupRetention, + NodeDeleteAfter: resolved.NodeDeleteAfter, + Interval: resolved.CleanupInterval, + }), background.ReconfirmTask(store, resolved.RouteRetention, resolved.RouteGrace, int64(resolved.RouteMinObservations), resolved.ReconfirmInterval), } if resolved.ObserverDeleteAfter > 0 { @@ -416,7 +429,7 @@ func main() { MaxConnectsPerMinute: resolved.MaxConnectsPerMinute, WSAllowedOrigins: cfg.WebSocket.AllowedOrigins, CORS: cfg.CORS, Server: cfg.Server, Auth: cfg.Auth, RateLimit: resolved.RateLimit, - Scopes: scopes, + Scopes: scopes, RollupRetention: resolved.RollupRetention, AdminRoutes: map[string]http.Handler{ "/accounts": handlers.AccountsRouter(store), "/backup": handlers.BackupRouter(backupOpts, ctx), diff --git a/config.yaml.example b/config.yaml.example index ae3a35ae..3d91c8ff 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -181,7 +181,7 @@ scopes: region: western-canada telemetry: - retention: 672h # 4 weeks + retention: 744h # 31 days resolution: 1h # store one snapshot per observer per hour # Optional authenticated database/config downloads. Disabled unless explicitly enabled. @@ -194,6 +194,11 @@ telemetry: packets: retention: 168h # 7 days +# Hourly analytics rollups (stats endpoints). Rollups outlive raw packets; this is also the +# longest window /stats/series and the other historical stats endpoints accept. +analytics: + rollup_retention: 2160h # 90 days + # Known-route retention. Routes are distilled packet history (the path a packet # took, hop by hop), so they may outlive packets.retention; there is no required # relationship between the two. @@ -266,7 +271,7 @@ cache: ttl: "1h" ttls: # Aggregated stats endpoints (overview, observations, breakdowns, top nodes/observers, - # radio presets, scope stats). Backed by materialized views refreshed hourly. + # radio presets, scope stats). Backed by hourly rollups; keys change when a new hour rolls. stats: "1h" # Mostly-static reference data (IATAs, regions, scopes). # Only changes when new observers arrive or config is reseeded. @@ -291,6 +296,6 @@ cache: # Shorter intervals are useful during initial deployment to confirm data is # flowing. Back off to 1h or more once stable. #background: -# view_refresh: 1h # default: 1h +# view_refresh: 1h # rebuilds radio presets; default: 1h # reconfirm: 1h # default: 1h # cleanup: 1h # default: 1h diff --git a/db/analytics_retention_integration_test.go b/db/analytics_retention_integration_test.go deleted file mode 100644 index 4306b5a5..00000000 --- a/db/analytics_retention_integration_test.go +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later - -package db - -import ( - "context" - sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" - "os" - "strings" - "testing" - "time" -) - -var retainedViews = []string{"mv_hourly_iata_stats", "mv_payload_breakdown_by_iata", "mv_top_observers_by_iata", "mv_top_talkers_by_iata", "mv_top_advertisers_by_iata", "mv_observer_activity_hourly", "mv_signal_stats_hourly", "mv_path_stats_hourly"} - -func refreshRetainedViews(t *testing.T, ctx context.Context, tx pgx.Tx) { - t.Helper() - for _, view := range retainedViews { - if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW "+view); err != nil { - t.Fatal(err) - } - } -} - -func analyticsSnapshot(t *testing.T, ctx context.Context, tx pgx.Tx) map[string]string { - t.Helper() - refreshRetainedViews(t, ctx, tx) - result := map[string]string{} - for _, view := range retainedViews { - var rows string - if err := tx.QueryRow(ctx, "SELECT COALESCE(jsonb_agg(r ORDER BY r::text),'[]')::text FROM (SELECT to_jsonb(v) r FROM "+view+" v) s").Scan(&rows); err != nil { - t.Fatal(err) - } - result[view] = rows - } - return result -} - -func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { - ctx, tx := retentionTx(t) - applyBaseline(t, ctx, tx) - if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) VALUES ('\x01',4,0,1,'\x00','\x00',NOW()-interval '5 days',NOW()-interval '5 days'); INSERT INTO observers(id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01')`); err != nil { - t.Fatal(err) - } - var schema string - if err := tx.QueryRow(ctx, "SELECT current_schema()").Scan(&schema); err != nil { - t.Fatal(err) - } - if err := tx.Commit(ctx); err != nil { - t.Fatal(err) - } - conn := tx.Conn() - quoted := pgx.Identifier{schema}.Sanitize() - t.Cleanup(func() { - if _, err := conn.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { - t.Error(err) - } - }) - if _, err := conn.Exec(ctx, "SET search_path TO "+quoted); err != nil { - t.Fatal(err) - } - writer, err := pgx.Connect(ctx, os.Getenv("BEACON_TEST_POSTGRES_DSN")) - if err != nil { - t.Fatal(err) - } - defer writer.Close(context.Background()) - if _, err = writer.Exec(ctx, "SET search_path TO "+quoted); err != nil { - t.Fatal(err) - } - ingest, err := writer.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer ingest.Rollback(context.Background()) - // An in-flight FK insert holds KEY SHARE on the packet until it commits. - if _, err = ingest.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES ('\x01','00000000-0000-0000-0000-000000000001','YVR',NOW()-interval '5 days',0,1,0)`); err != nil { - t.Fatal(err) - } - q := sqlc.New(conn) - params := sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: time.Now().Add(-72 * time.Hour), Valid: true}, BatchSize: 1000} - if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { - t.Fatalf("must skip in-flight reception: %d %v", n, err) - } - if err := ingest.Commit(ctx); err != nil { - t.Fatal(err) - } - if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 1 { - t.Fatalf("retry after ingestion: %d %v", n, err) - } - var count int - if err := conn.QueryRow(ctx, "SELECT observation_count FROM analytics_hourly_iata_stats").Scan(&count); err != nil || count != 1 { - t.Fatalf("committed reception not archived: %d %v", count, err) - } - // A second cleanup runner must wait before touching shared aggregate keys. - first, err := writer.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer first.Rollback(context.Background()) - if _, err := sqlc.New(first).DeleteOldPackets(ctx, params); err != nil { - t.Fatal(err) - } - if _, err := conn.Exec(ctx, "SET statement_timeout='300ms'"); err != nil { - t.Fatal(err) - } - if _, err := q.DeleteOldPackets(ctx, params); err == nil { - t.Fatalf("overlapping cleanup did not wait for the first transaction: %v", err) - } - if _, err := conn.Exec(ctx, "SET statement_timeout=0"); err != nil { - t.Fatal(err) - } - if err := first.Commit(ctx); err != nil { - t.Fatal(err) - } - if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { - t.Fatalf("cleanup lock was not released: %d %v", n, err) - } -} - -func TestAnalyticsRetentionPostgres(t *testing.T) { - ctx, tx := retentionTx(t) - applyBaseline(t, ctx, tx) - _, err := tx.Exec(ctx, ` - INSERT INTO channels (id,channel_hash) VALUES (1,'\x01'); - INSERT INTO observers (id,public_key,display_name,observer_type) SELECT ('00000000-0000-0000-0000-00000000000'||i)::uuid,int4send(i),CASE WHEN i=1 THEN NULL ELSE 'Observer '||i END,CASE WHEN i=1 THEN NULL ELSE 'test' END FROM generate_series(1,3) i; - INSERT INTO nodes (id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2,'Repeater'); - INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) - SELECT int4send(i),4,0,CASE WHEN i%2=0 THEN 1 ELSE 2 END,'\x00','\x00','\x02',date_trunc('hour',NOW())-interval '5 days', - CASE WHEN i=4 THEN NOW() ELSE date_trunc('hour',NOW())-interval '5 days' END FROM generate_series(1,4) i; - INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,airtime_ms,payload_type) - SELECT packet_hash,('00000000-0000-0000-0000-00000000000'||n)::uuid,iata,first_heard_at+make_interval(secs=>n),0,1,0, - CASE WHEN n=1 THEN 0 ELSE NULL END,CASE WHEN n=1 THEN -100 ELSE NULL END,1.5,4 - FROM packets CROSS JOIN (VALUES ('YVR',1),('YVR',2),('YYZ',3)) v(iata,n); - INSERT INTO channel_messages (channel_id,packet_hash,sender_name,content,sent_at) - SELECT 1,packet_hash,'Sender','body that must expire',first_heard_at FROM packets; - `) - if err != nil { - t.Fatal(err) - } - for _, name := range []string{"idx_mv_signal_stats_hourly_hour", "idx_mv_path_stats_hourly_hour"} { - var definition string - if err := tx.QueryRow(ctx, "SELECT pg_get_indexdef(to_regclass($1))", name).Scan(&definition); err != nil || !strings.Contains(definition, "(hour)") { - t.Fatalf("missing unfiltered time index %s: %s %v", name, definition, err) - } - } - refreshRetainedViews(t, ctx, tx) - store := &Store{q: sqlc.New(tx)} - hour := time.Now().UTC().Truncate(time.Hour) - signal, err := store.GetSignalStats(ctx, hour.Add(-6*24*time.Hour), hour, nil) - if err != nil { - t.Fatal(err) - } - // 12 observations; only the n=1 rows carry a signal (snr 0, rssi -100). - if signal.Receptions != 12 || signal.SNR.Samples != 4 || signal.RSSI.Samples != 4 { - t.Fatalf("signal stats: %+v", signal) - } - paths, err := store.GetPathStats(ctx, hour.Add(-6*24*time.Hour), hour, nil) - if err != nil { - t.Fatal(err) - } - if paths.Receptions != 12 { - t.Fatalf("path stats: %+v", paths) - } - before := analyticsSnapshot(t, ctx, tx) - q := sqlc.New(tx) - cutoff := time.Now().Add(-72 * time.Hour) - // Failure in one archive must roll back every archive write and the raw deletion. - if _, err := tx.Exec(ctx, `SAVEPOINT archive_failure; ALTER TABLE analytics_signal_stats_hourly ADD CONSTRAINT fail_archive CHECK(receptions < 0) NOT VALID`); err != nil { - t.Fatal(err) - } - if _, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}); err == nil { - t.Fatal("expected archive failure") - } - if _, err := tx.Exec(ctx, "ROLLBACK TO archive_failure"); err != nil { - t.Fatal(err) - } - if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 4 { - t.Fatal("archive failure deleted raw packets") - } - for i := 0; i < 3; i++ { - n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}) - if err != nil || n != 1 { - t.Fatalf("delete batch: %d %v", n, err) - } - } - if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 1 || countRows(t, ctx, tx, "SELECT count(*) FROM channel_messages") != 1 { - t.Fatal("raw packets/messages did not expire") - } - after := analyticsSnapshot(t, ctx, tx) - for _, view := range retainedViews { - if after[view] != before[view] { - t.Errorf("%s lost or double-counted history after raw expiry: before=%s after=%s", view, before[view], after[view]) - } - } - // A retry preserves the exact summaries. - if n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1000}); err != nil || n != 0 { - t.Fatalf("retry: %d %v", n, err) - } - for view, rows := range analyticsSnapshot(t, ctx, tx) { - if rows != before[view] { - t.Errorf("retry changed %s", view) - } - } - // Later receptions on a still-live packet must join its old bucket before expiry. - if _, err := tx.Exec(ctx, `INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000004','\x04'); INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,payload_type) - SELECT packet_hash,'00000000-0000-0000-0000-000000000004','YVR',first_heard_at,0,1,0,'NaN',-80,NULL FROM packets; - UPDATE packets SET last_heard_at=first_heard_at`); err != nil { - t.Fatal(err) - } - late := analyticsSnapshot(t, ctx, tx) - if countRows(t, ctx, tx, "SELECT COALESCE(SUM(observations),0) FROM mv_observer_activity_hourly WHERE payload_type=-1") != 1 { - t.Fatal("unknown-payload observation was omitted") - } - if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { - t.Fatal(err) - } - for view, rows := range analyticsSnapshot(t, ctx, tx) { - if rows != late[view] { - t.Errorf("late reception changed %s: %s != %s", view, rows, late[view]) - } - } - // Archived labels follow a current rename; no entity FK may erase retained history. - if _, err := tx.Exec(ctx, `UPDATE nodes SET name='Renamed'; UPDATE observers SET display_name=NULL,observer_type=NULL;`); err != nil { - t.Fatal(err) - } - analyticsSnapshot(t, ctx, tx) - if countRows(t, ctx, tx, "SELECT count(*) FROM mv_top_advertisers_by_iata WHERE name='Renamed'") != 2 { - t.Fatal("archive name did not follow rename") - } - if _, err := q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{Column1: pgtype.Interval{Microseconds: int64(30 * 24 * time.Hour / time.Microsecond), Valid: true}, Limit: 10}); err != nil { - t.Fatal(err) - } - if _, err := tx.Exec(ctx, "DELETE FROM nodes; DELETE FROM observers"); err != nil { - t.Fatal(err) - } - if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_top_advertisers_by_iata") != 2 { - t.Fatal("entity deletion erased archive") - } - // Independent expiry runs even when there are no packets left to delete. - for _, view := range retainedViews { - table := "analytics_" + view[3:] - at := "bucket" - if view == "mv_hourly_iata_stats" || view == "mv_signal_stats_hourly" || view == "mv_path_stats_hourly" { - at = "hour" - } - if _, err := tx.Exec(ctx, "UPDATE "+table+" SET "+at+"="+at+"-interval '31 days'"); err != nil { - t.Fatal(err) - } - } - if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { - t.Fatal(err) - } - for _, view := range retainedViews { - if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) != 0 { - t.Errorf("%s archive did not expire", view) - } - } - // More than one populated cohort, with shared hourly keys across the boundary. - if _, err := tx.Exec(ctx, ` -INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); -INSERT INTO nodes (id,public_key,node_type) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2); -INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,origin_pubkey,first_heard_at,last_heard_at,raw_payload,raw_header) -SELECT int4send(i),4,0,1,'\x02',date_trunc('hour',NOW())-interval '5 days',date_trunc('hour',NOW())-interval '5 days',decode(repeat('ab',1024),'hex'),'\x00' FROM generate_series(10000,11000) i; -INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) -SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YVR',first_heard_at,0,1,0,4,0,-100 FROM packets; -INSERT INTO channel_messages (channel_id,packet_hash,sender_name,sent_at) SELECT 1,packet_hash,'Sender',first_heard_at FROM packets; -`); err != nil { - t.Fatal(err) - } - populated := analyticsSnapshot(t, ctx, tx) - started := time.Now() - if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { - t.Fatal(err) - } - t.Logf("1001 packets with bodies and observations archived/deleted in %s", time.Since(started)) - for view, rows := range analyticsSnapshot(t, ctx, tx) { - if rows != populated[view] { - t.Errorf("batch boundary changed %s", view) - } - } - var archiveRows int - for _, view := range retainedViews { - archiveRows += countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) - } - if archiveRows > 12 { - t.Fatalf("expected compact rollups, got %d archive rows for 1001 packets", archiveRows) - } -} diff --git a/db/baseline_integration_test.go b/db/baseline_integration_test.go index e879c493..27a8a606 100644 --- a/db/baseline_integration_test.go +++ b/db/baseline_integration_test.go @@ -96,7 +96,7 @@ func TestRunMigrationsBaselinePostgres(t *testing.T) { t.Fatalf("ledger %v, %v", ledger, err) } // Stats refresh uses CONCURRENTLY, which needs populated views. - if _, err := pool.Exec(ctx, "REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hourly_iata_stats"); err != nil { + if _, err := pool.Exec(ctx, "REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets"); err != nil { t.Fatal(err) } } diff --git a/db/channel_message_count_integration_test.go b/db/channel_message_count_integration_test.go new file mode 100644 index 00000000..52558163 --- /dev/null +++ b/db/channel_message_count_integration_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" +) + +func TestChannelMessageCountPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +SELECT decode(lpad(to_hex(i), 2, '0'), 'hex'), 5, 0, 1, '\x00', '\x00', now(), now() FROM generate_series(1, 2) i`); err != nil { + t.Fatal(err) + } + // Re-upserting the channel (as every decrypt does) must not touch the count. + var id int + for range 3 { + if id, err = store.UpsertChannel(ctx, []byte{0x11}, []byte{1, 2, 3, 4, 5, 6, 7, 8}, "public", ""); err != nil { + t.Fatal(err) + } + } + for _, hash := range [][]byte{{1}, {1}, {2}} { // second {1} is a duplicate + if _, err := store.InsertChannelMessage(ctx, ingest.InsertChannelMessageParams{ + ChannelID: id, PacketHash: hash, SenderName: "a", Content: "b", SentAt: time.Now(), + }); err != nil { + t.Fatal(err) + } + } + ch, err := store.GetChannel(ctx, int32(id)) + if err != nil { + t.Fatal(err) + } + if ch.MessageCount != 2 { + t.Errorf("message_count = %d, want 2", ch.MessageCount) + } +} diff --git a/db/channel_scopes_integration_test.go b/db/channel_scopes_integration_test.go index e02f5766..ab9a9c46 100644 --- a/db/channel_scopes_integration_test.go +++ b/db/channel_scopes_integration_test.go @@ -61,6 +61,7 @@ SELECT decode(lpad(to_hex(i),2,'0'),'hex'),5,0,1,transport,scope_id,'\x00','\x15 FROM (VALUES(1,true,1),(2,true,NULL),(3,false,NULL),(4,NULL,NULL),(5,true,2),(6,false,NULL)) v(i,transport,scope_id); INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) SELECT packet_hash,md5(n::text)::uuid,'YOW',NOW(),0,1,0 FROM packets CROSS JOIN generate_series(1,2) n; +UPDATE packets SET observation_count=2; `) if err != nil { t.Fatal(err) diff --git a/db/channels.go b/db/channels.go index 3e07032d..c292b766 100644 --- a/db/channels.go +++ b/db/channels.go @@ -26,12 +26,11 @@ func (s *Store) UpsertChannel(ctx context.Context, channelHash []byte, keyFinger } isHashtag := hashtag != "" row, err := s.q.UpsertChannel(ctx, sqlc.UpsertChannelParams{ - ChannelHash: channelHash, - Column2: keyFingerprint, // key_fingerprint - Name: namePtr, - Hashtag: hashtagPtr, - IsHashtag: &isHashtag, - MessageCount: nil, // message count bumped separately by InsertChannelMessage + ChannelHash: channelHash, + Column2: keyFingerprint, // key_fingerprint + Name: namePtr, + Hashtag: hashtagPtr, + IsHashtag: &isHashtag, }) if err != nil { return 0, err @@ -167,10 +166,7 @@ func (s *Store) GetChannel(ctx context.Context, channelID int32) (*api.Channel, KeyKnown: row.KeyKnown != nil && *row.KeyKnown, }, Hashtag: row.Hashtag, - MessageCount: 0, - } - if row.MessageCount != nil { - channel.MessageCount = *row.MessageCount + MessageCount: row.MessageCount, } if row.IsHashtag != nil && *row.IsHashtag && row.KeyFingerprint != nil { fp := hex.EncodeToString(row.KeyFingerprint) @@ -180,7 +176,7 @@ func (s *Store) GetChannel(ctx context.Context, channelID int32) (*api.Channel, } func (s *Store) InsertChannelMessage(ctx context.Context, m ingest.InsertChannelMessageParams) (*ingest.InsertedChannelMessage, error) { - params := sqlc.InsertChannelMessageParams{ChannelID: int32(m.ChannelID), PacketHash: m.PacketHash, SenderName: &m.SenderName, Content: &m.Content, SentAt: pgtype.Timestamptz{Time: m.SentAt, Valid: true}} + params := sqlc.InsertChannelMessageParams{ChannelID: int32(m.ChannelID), PacketHash: m.PacketHash, SenderName: &m.SenderName, Content: &m.Content, SentAt: pgtype.Timestamptz{Time: m.SentAt, Valid: true}, Column6: m.Historical} row, err := s.q.InsertChannelMessage(ctx, params) if errors.Is(err, pgx.ErrNoRows) { return nil, nil // duplicate diff --git a/db/channels_test.go b/db/channels_test.go index 9ba475c0..be0fd464 100644 --- a/db/channels_test.go +++ b/db/channels_test.go @@ -164,7 +164,7 @@ func TestGetChannel_Basic(t *testing.T) { LastSeen: lastSeen, IsHashtag: &isHashtag, KeyKnown: &keyKnown, - MessageCount: &msgCount, + MessageCount: msgCount, }, nil) store := &Store{q: mock} diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql index 06e4335e..00af4592 100644 --- a/db/migrations/001_baseline.sql +++ b/db/migrations/001_baseline.sql @@ -4,237 +4,6 @@ -- 2.0.0 schema baseline: pg_dump of a fresh database migrated through the 1.x -- chain (001_initial_schema..045). Add new numbered migrations after this file. -CREATE FUNCTION archive_delete_packets(cutoff timestamp with time zone, batch_size integer) RETURNS bigint - LANGUAGE plpgsql - AS $$ -DECLARE hashes bytea[]; deleted bigint; -BEGIN - -- Serialize cleanup runners before overlapping aggregate upserts. - PERFORM pg_advisory_xact_lock(hashtext('beacon.archive_delete_packets')); - IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; - SELECT array_agg(p.packet_hash) INTO hashes FROM ( - SELECT ep.packet_hash FROM packets ep - WHERE ep.last_heard_at < cutoff - ORDER BY ep.last_heard_at, ep.packet_hash - LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED - ) p; - - WITH expired_observations AS MATERIALIZED ( - SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) - ), -archived_hourly_iata_stats AS ( - INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) - SELECT iata, hour, observation_count, unique_packets FROM ( -SELECT - iata, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY iata, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,hour) DO UPDATE SET - observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, - unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets -), -archived_payload_breakdown_by_iata AS ( - INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) - SELECT iata, payload_type, bucket, count FROM ( -SELECT - iata, - payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS count -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET - count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count -), -archived_top_observers_by_iata AS ( - INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM expired_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET - observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, - display_name = EXCLUDED.display_name, - observer_type = EXCLUDED.observer_type -), -archived_top_talkers_by_iata AS ( - INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) - SELECT iata, sender_name, bucket, message_count, last_sent FROM ( -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN expired_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') - ) batch - ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET - message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, - last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) -), -archived_top_advertisers_by_iata AS ( - INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN expired_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,node_id,bucket) DO UPDATE SET - advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, - flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, - direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, - last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), - name = EXCLUDED.name, - node_type = EXCLUDED.node_type -), -archived_observer_activity_hourly AS ( - INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET - observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, - airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, - airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, - snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, - snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, - snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), - rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, - rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n -), -archived_signal_stats_hourly AS ( - INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( -WITH samples AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), binned AS ( - SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples -) -SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(sum(snr), 0)::double precision AS snr_sum, - count(rssi)::bigint AS rssi_samples, - COALESCE(sum(rssi), 0)::double precision AS rssi_sum -FROM binned -GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) - ) batch - ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET - receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, - snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, - snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, - rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, - rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum -), -archived_path_stats_hourly AS ( - INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) - SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( -WITH classified AS ( - SELECT iata, heard_at, hash_size, hop_count, - CASE WHEN payload_type = 9 THEN 2 - -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). - WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 - OR NOT (path_length_byte BETWEEN 0 AND 191 - AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 - AND hash_size = (path_length_byte >> 6) + 1 - AND hop_count = (path_length_byte & 63) - AND hash_size::integer * hop_count::integer <= 64 - AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) - THEN 3 - WHEN hop_count = 0 THEN 1 - ELSE 0 END::integer AS category - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), buckets AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, - CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, - CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries - FROM classified -) -SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions -FROM buckets GROUP BY iata, hour, category, hash_bytes, entries - ) batch - ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET - receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions -) - DELETE FROM packets WHERE packet_hash = ANY(hashes); - GET DIAGNOSTICS deleted = ROW_COUNT; - - -- Once per completed cleanup, including when there were no raw packets to delete. - IF deleted < batch_size THEN - DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - END IF; - RETURN deleted; -END $$; - CREATE TABLE accounts ( id uuid DEFAULT gen_random_uuid() NOT NULL, name text NOT NULL, @@ -243,13 +12,6 @@ CREATE TABLE accounts ( CONSTRAINT accounts_name_check CHECK ((((char_length(name) >= 1) AND (char_length(name) <= 128)) AND (name = btrim(name)))) ); -CREATE TABLE analytics_hourly_iata_stats ( - iata character(3) NOT NULL, - hour timestamp with time zone NOT NULL, - observation_count bigint DEFAULT 0 NOT NULL, - unique_packets bigint DEFAULT 0 NOT NULL -); - CREATE TABLE packet_observations ( id bigint NOT NULL, packet_hash bytea NOT NULL, @@ -273,98 +35,6 @@ CREATE TABLE packet_observations ( ) WITH (autovacuum_vacuum_scale_factor='0.05', autovacuum_vacuum_cost_limit='1000'); -CREATE VIEW analytics_live_hourly_iata_stats AS -SELECT - iata, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' -GROUP BY iata, date_trunc('hour', heard_at, 'UTC'); - -CREATE VIEW analytics_live_observer_activity_hourly AS -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); - -CREATE VIEW analytics_live_path_stats_hourly AS -WITH classified AS ( - SELECT iata, heard_at, hash_size, hop_count, - CASE WHEN payload_type = 9 THEN 2 - -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). - WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 - OR NOT (path_length_byte BETWEEN 0 AND 191 - AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 - AND hash_size = (path_length_byte >> 6) + 1 - AND hop_count = (path_length_byte & 63) - AND hash_size::integer * hop_count::integer <= 64 - AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) - THEN 3 - WHEN hop_count = 0 THEN 1 - ELSE 0 END::integer AS category - FROM packet_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - AND heard_at < date_trunc('hour', NOW(), 'UTC') -), buckets AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, - CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, - CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries - FROM classified -) -SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions -FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; - -CREATE VIEW analytics_live_payload_breakdown_by_iata AS -SELECT - iata, - payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS count -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC'); - -CREATE VIEW analytics_live_signal_stats_hourly AS -WITH samples AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM packet_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - AND heard_at < date_trunc('hour', NOW(), 'UTC') -), binned AS ( - SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples -) -SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(sum(snr), 0)::double precision AS snr_sum, - count(rssi)::bigint AS rssi_samples, - COALESCE(sum(rssi), 0)::double precision AS rssi_sum -FROM binned -GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); - CREATE TABLE nodes ( id uuid DEFAULT gen_random_uuid() NOT NULL, public_key bytea NOT NULL, @@ -410,28 +80,11 @@ CREATE TABLE packets ( channel_hash bytea, trace_tag bytea, first_heard_at timestamp with time zone NOT NULL, - last_heard_at timestamp with time zone NOT NULL + last_heard_at timestamp with time zone NOT NULL, + observation_count bigint DEFAULT 0 NOT NULL ) WITH (autovacuum_vacuum_scale_factor='0.05', autovacuum_vacuum_cost_limit='1000'); -CREATE VIEW analytics_live_top_advertisers_by_iata AS -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN packet_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC'); - CREATE TABLE observers ( id uuid DEFAULT gen_random_uuid() NOT NULL, public_key bytea NOT NULL, @@ -453,22 +106,11 @@ CREATE TABLE observers ( last_seen timestamp with time zone DEFAULT now() NOT NULL, observation_count bigint DEFAULT 0, metadata jsonb, - region_scope text + region_scope text, + last_iata character(3), + last_iata_at timestamp with time zone ); -CREATE VIEW analytics_live_top_observers_by_iata AS -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM packet_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC'); - CREATE TABLE channel_messages ( id bigint NOT NULL, channel_id integer NOT NULL, @@ -479,91 +121,6 @@ CREATE TABLE channel_messages ( sent_at timestamp with time zone NOT NULL ); -CREATE VIEW analytics_live_top_talkers_by_iata AS -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN packet_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at > NOW() - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC'); - -CREATE TABLE analytics_observer_activity_hourly ( - observer_id uuid NOT NULL, - payload_type smallint NOT NULL, - bucket timestamp with time zone NOT NULL, - observations bigint DEFAULT 0 NOT NULL, - airtime_ms real, - airtime_n bigint DEFAULT 0 NOT NULL, - snr_sum real, - snr_n bigint DEFAULT 0 NOT NULL, - snr_min real, - rssi_sum bigint, - rssi_n bigint DEFAULT 0 NOT NULL -); - -CREATE TABLE analytics_path_stats_hourly ( - iata character(3) NOT NULL, - hour timestamp with time zone NOT NULL, - category integer NOT NULL, - hash_bytes integer NOT NULL, - entries integer NOT NULL, - receptions bigint DEFAULT 0 NOT NULL -); - -CREATE TABLE analytics_payload_breakdown_by_iata ( - iata character(3) NOT NULL, - payload_type smallint NOT NULL, - bucket timestamp with time zone NOT NULL, - count bigint DEFAULT 0 NOT NULL -); - -CREATE TABLE analytics_signal_stats_hourly ( - iata character(3) NOT NULL, - hour timestamp with time zone NOT NULL, - kind integer NOT NULL, - snr_bin integer NOT NULL, - rssi_bin integer NOT NULL, - receptions bigint DEFAULT 0 NOT NULL, - snr_samples bigint DEFAULT 0 NOT NULL, - snr_sum double precision, - rssi_samples bigint DEFAULT 0 NOT NULL, - rssi_sum double precision -); - -CREATE TABLE analytics_top_advertisers_by_iata ( - iata character(3) NOT NULL, - node_id uuid NOT NULL, - bucket timestamp with time zone NOT NULL, - advert_count bigint DEFAULT 0 NOT NULL, - flood_advert_count bigint DEFAULT 0 NOT NULL, - direct_advert_count bigint DEFAULT 0 NOT NULL, - last_heard timestamp with time zone, - name text, - node_type smallint -); - -CREATE TABLE analytics_top_observers_by_iata ( - iata character(3) NOT NULL, - observer_id uuid NOT NULL, - bucket timestamp with time zone NOT NULL, - observation_count bigint DEFAULT 0 NOT NULL, - display_name text, - observer_type text -); - -CREATE TABLE analytics_top_talkers_by_iata ( - iata character(3) NOT NULL, - sender_name text NOT NULL, - bucket timestamp with time zone NOT NULL, - message_count bigint DEFAULT 0 NOT NULL, - last_sent timestamp with time zone -); - CREATE TABLE channel_config_scopes ( key_fingerprint bytea NOT NULL, region_slug text @@ -603,7 +160,7 @@ CREATE TABLE channels ( key_known boolean DEFAULT false, first_seen timestamp with time zone DEFAULT now() NOT NULL, last_seen timestamp with time zone DEFAULT now() NOT NULL, - message_count bigint DEFAULT 0 + message_count bigint DEFAULT 0 NOT NULL ); CREATE SEQUENCE channels_id_seq @@ -696,53 +253,6 @@ CREATE TABLE meshmapper_zone_lists ( last_error text DEFAULT ''::text NOT NULL ); -CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS -WITH combined AS ( - SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats - UNION ALL - SELECT iata, hour, observation_count, unique_packets FROM analytics_hourly_iata_stats - WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), observers_by_hour AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, observer_id - FROM packet_observations WHERE heard_at > NOW() - INTERVAL '30 days' - UNION - SELECT iata, bucket AS hour, observer_id FROM analytics_top_observers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), observer_counts AS (SELECT iata, hour, COUNT(*) AS n FROM observers_by_hour GROUP BY iata,hour) -SELECT combined.iata, combined.hour, SUM(observation_count)::bigint AS observation_count, SUM(unique_packets)::bigint AS unique_packets, MAX(o.n)::bigint AS active_observers -FROM combined JOIN observer_counts o USING (iata,hour) GROUP BY combined.iata,combined.hour; - -CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS -WITH combined AS ( - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly - UNION ALL - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM combined GROUP BY observer_id,payload_type,bucket; - -CREATE MATERIALIZED VIEW mv_path_stats_hourly AS -WITH combined AS ( - SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_live_path_stats_hourly - UNION ALL - SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_path_stats_hourly - WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND hour < date_trunc('hour', NOW(), 'UTC') -) -SELECT iata, hour, category, hash_bytes, entries, SUM(receptions)::bigint AS receptions -FROM combined GROUP BY iata,hour,category,hash_bytes,entries; - -CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS -WITH combined AS ( - SELECT iata, payload_type, bucket, count FROM analytics_live_payload_breakdown_by_iata - UNION ALL - SELECT iata, payload_type, bucket, count FROM analytics_payload_breakdown_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT iata, payload_type, bucket, SUM(count)::bigint AS count -FROM combined GROUP BY iata,payload_type,bucket; - CREATE TABLE node_iatas ( node_id uuid NOT NULL, iata character(3) NOT NULL, @@ -755,16 +265,15 @@ WITH (autovacuum_vacuum_scale_factor='0.05'); CREATE MATERIALIZED VIEW mv_radio_presets AS SELECT concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text AS preset, - (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) AS iata, - 'observer' AS source_type, + o.last_iata AS iata, + 'observer'::text AS source_type, COUNT(*) AS count FROM observers o WHERE o.radio_freq_mhz IS NOT NULL AND o.radio_sf IS NOT NULL AND o.radio_bw_khz IS NOT NULL -GROUP BY concat(o.radio_freq_mhz, ',', o.radio_bw_khz, ',', o.radio_sf)::text, - (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) -HAVING (SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1) IS NOT NULL + AND o.last_iata IS NOT NULL +GROUP BY 1, 2 UNION ALL @@ -778,64 +287,7 @@ JOIN node_iatas ni ON ni.node_id = n.id WHERE n.radio_freq_mhz IS NOT NULL AND n.radio_sf IS NOT NULL AND n.radio_bw_khz IS NOT NULL -GROUP BY concat(n.radio_freq_mhz, ',', n.radio_bw_khz, ',', n.radio_sf), ni.iata; - -CREATE MATERIALIZED VIEW mv_signal_stats_hourly AS -WITH combined AS ( - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_live_signal_stats_hourly - UNION ALL - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_signal_stats_hourly - WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND hour < date_trunc('hour', NOW(), 'UTC') -) -SELECT iata, hour, kind, snr_bin, rssi_bin, SUM(receptions)::bigint AS receptions, SUM(snr_samples)::bigint AS snr_samples, SUM(snr_sum)::double precision AS snr_sum, SUM(rssi_samples)::bigint AS rssi_samples, SUM(rssi_sum)::double precision AS rssi_sum -FROM combined GROUP BY iata,hour,kind,snr_bin,rssi_bin; - -CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS -WITH combined AS ( - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_live_top_advertisers_by_iata - UNION ALL - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_top_advertisers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), totals AS (SELECT iata, node_id, bucket, SUM(advert_count)::bigint AS advert_count, SUM(flood_advert_count)::bigint AS flood_advert_count, SUM(direct_advert_count)::bigint AS direct_advert_count, MAX(last_heard) AS last_heard, MAX(name) AS name, MAX(node_type) AS node_type -FROM combined GROUP BY iata,node_id,bucket) -SELECT t.iata, t.node_id, COALESCE(n.name,t.name) AS name, COALESCE(n.node_type,t.node_type) AS node_type, - t.bucket, t.advert_count, t.flood_advert_count, t.direct_advert_count, t.last_heard -FROM totals t LEFT JOIN nodes n ON n.id=t.node_id; - -CREATE MATERIALIZED VIEW mv_top_nodes_by_iata AS -SELECT - ni.iata, - ni.node_id, - n.name, - n.node_type, - ni.observation_count, - ni.last_heard -FROM node_iatas ni -JOIN nodes n ON n.id = ni.node_id -WHERE ni.last_heard > NOW() - INTERVAL '7 days'; - -CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS -WITH combined AS ( - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_live_top_observers_by_iata - UNION ALL - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_top_observers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -), totals AS (SELECT iata, observer_id, bucket, SUM(observation_count)::bigint AS observation_count, MAX(display_name) AS display_name, MAX(observer_type) AS observer_type -FROM combined GROUP BY iata,observer_id,bucket) -SELECT t.iata, t.observer_id, COALESCE(o.display_name,t.display_name) AS display_name, - COALESCE(o.observer_type,t.observer_type) AS observer_type, t.bucket, t.observation_count -FROM totals t LEFT JOIN observers o ON o.id=t.observer_id; - -CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS -WITH combined AS ( - SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_live_top_talkers_by_iata - UNION ALL - SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_top_talkers_by_iata - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT iata, sender_name, bucket, SUM(message_count)::bigint AS message_count, MAX(last_sent) AS last_sent -FROM combined GROUP BY iata,sender_name,bucket; +GROUP BY 1, 2; CREATE TABLE node_neighbors ( node_id uuid NOT NULL, @@ -962,6 +414,168 @@ CREATE TABLE trace_iatas ( last_heard timestamp with time zone DEFAULT now() NOT NULL ); +CREATE TABLE trace_tags ( + trace_tag bytea PRIMARY KEY, + first_heard_at timestamp with time zone NOT NULL, + last_heard_at timestamp with time zone NOT NULL, + packet_count bigint DEFAULT 0 NOT NULL, + trace_type text, + scope_id integer, + best_payload jsonb, + best_path_len integer DEFAULT -1 NOT NULL, + heard boolean DEFAULT false NOT NULL -- false: times are the packet's provisional ones until a hearing is recorded +); + +-- Canonical IATA set: sorted, unique, non-null, non-empty. +CREATE FUNCTION iata_set_is_canonical(s bpchar[]) RETURNS boolean + LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT cardinality(s) > 0 + AND array_position(s, NULL) IS NULL + AND s = (SELECT array_agg(x ORDER BY x) FROM (SELECT DISTINCT unnest(s) AS x) d) +$$; + +-- Hourly rollups. hour = UTC date_trunc('hour', heard_at). No FKs: history outlives deletions. +CREATE TABLE analytics_hourly_iata_observations ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + observation_count bigint NOT NULL, + PRIMARY KEY (hour, iata) +); + +CREATE TABLE analytics_hourly_payload_breakdown ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + payload_type smallint NOT NULL, + count bigint NOT NULL, + PRIMARY KEY (hour, iata, payload_type) +); + +CREATE TABLE analytics_hourly_signal ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + kind integer NOT NULL, + snr_bin integer NOT NULL, + rssi_bin integer NOT NULL, + receptions bigint NOT NULL, + snr_samples bigint NOT NULL, + snr_sum double precision NOT NULL, + rssi_samples bigint NOT NULL, + rssi_sum double precision NOT NULL, + PRIMARY KEY (hour, iata, kind, snr_bin, rssi_bin) +); + +CREATE TABLE analytics_hourly_paths ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + category integer NOT NULL, + hash_bytes integer NOT NULL, + entries integer NOT NULL, + receptions bigint NOT NULL, + PRIMARY KEY (hour, iata, category, hash_bytes, entries) +); + +CREATE TABLE analytics_hourly_observer_activity ( + hour timestamp with time zone NOT NULL, + observer_id uuid NOT NULL, + payload_type smallint NOT NULL, + observations bigint NOT NULL, + airtime_ms real, + airtime_n bigint NOT NULL, + snr_sum real, + snr_n bigint NOT NULL, + snr_min real, + rssi_sum bigint, + rssi_n bigint NOT NULL, + PRIMARY KEY (hour, observer_id, payload_type) +); + +CREATE TABLE analytics_hourly_observer_identity ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + observer_id uuid NOT NULL, + observation_count bigint NOT NULL, + display_name text, + observer_type text, + PRIMARY KEY (hour, iata, observer_id) +); + +-- ADVERT hearings per origin per IATA: top nodes (observations) and advertiser last_heard. +CREATE TABLE analytics_hourly_advert_hearings ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + origin_pubkey bytea NOT NULL, + observations bigint NOT NULL, + last_heard timestamp with time zone NOT NULL, + name text, + node_type smallint, + PRIMARY KEY (hour, iata, origin_pubkey) +); + +-- Distinct ADVERT packets per origin, grouped by the set of IATAs that heard each packet that hour. +CREATE TABLE analytics_hourly_advert_sets ( + hour timestamp with time zone NOT NULL, + origin_pubkey bytea NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + advert_packets bigint NOT NULL, + flood_packets bigint NOT NULL, + direct_packets bigint NOT NULL, + PRIMARY KEY (hour, origin_pubkey, iatas) +); + +-- Distinct channel messages per sender, grouped by the set of IATAs that heard each message that hour. +CREATE TABLE analytics_hourly_talker_sets ( + hour timestamp with time zone NOT NULL, + sender_name text NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + messages bigint NOT NULL, + last_sent timestamp with time zone NOT NULL, + PRIMARY KEY (hour, sender_name, iatas) +); + +CREATE TABLE analytics_hourly_packet_sets ( + hour timestamp with time zone NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + packets bigint NOT NULL, + PRIMARY KEY (hour, iatas) +); + +CREATE TABLE analytics_hourly_scope_sets ( + hour timestamp with time zone NOT NULL, + iatas character(3)[] NOT NULL CHECK (iata_set_is_canonical(iatas)), + scope_id integer NOT NULL, + packets bigint NOT NULL, + PRIMARY KEY (hour, iatas, scope_id) +); + +CREATE TABLE analytics_rollup_hours ( + hour timestamp with time zone PRIMARY KEY, + status text NOT NULL CHECK (status IN ('complete', 'partial', 'missing')), + rolled_at timestamp with time zone, + content_hash text +); + +-- Singletons: the rollup writes revision, cleanup writes raw_deleted_before. +CREATE TABLE analytics_state ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + revision bigint NOT NULL DEFAULT 0 +); + +INSERT INTO analytics_state DEFAULT VALUES; + +-- coverage counts retention deletions of rollup hours; cache keys combine it with revision. +CREATE TABLE analytics_raw_state ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + raw_deleted_before timestamp with time zone, + coverage bigint NOT NULL DEFAULT 0 +); + +INSERT INTO analytics_raw_state DEFAULT VALUES; + +CREATE TABLE analytics_dirty_hours ( + hour timestamp with time zone PRIMARY KEY, + enqueued_at timestamp with time zone NOT NULL DEFAULT now() +); + CREATE TABLE transport_scopes ( id integer NOT NULL, name text NOT NULL, @@ -997,30 +611,6 @@ ALTER TABLE ONLY transport_scopes ALTER COLUMN id SET DEFAULT nextval('transport ALTER TABLE ONLY accounts ADD CONSTRAINT accounts_pkey PRIMARY KEY (id); -ALTER TABLE ONLY analytics_hourly_iata_stats - ADD CONSTRAINT analytics_hourly_iata_stats_pkey PRIMARY KEY (iata, hour); - -ALTER TABLE ONLY analytics_observer_activity_hourly - ADD CONSTRAINT analytics_observer_activity_hourly_pkey PRIMARY KEY (observer_id, payload_type, bucket); - -ALTER TABLE ONLY analytics_path_stats_hourly - ADD CONSTRAINT analytics_path_stats_hourly_pkey PRIMARY KEY (iata, hour, category, hash_bytes, entries); - -ALTER TABLE ONLY analytics_payload_breakdown_by_iata - ADD CONSTRAINT analytics_payload_breakdown_by_iata_pkey PRIMARY KEY (iata, payload_type, bucket); - -ALTER TABLE ONLY analytics_signal_stats_hourly - ADD CONSTRAINT analytics_signal_stats_hourly_pkey PRIMARY KEY (iata, hour, kind, snr_bin, rssi_bin); - -ALTER TABLE ONLY analytics_top_advertisers_by_iata - ADD CONSTRAINT analytics_top_advertisers_by_iata_pkey PRIMARY KEY (iata, node_id, bucket); - -ALTER TABLE ONLY analytics_top_observers_by_iata - ADD CONSTRAINT analytics_top_observers_by_iata_pkey PRIMARY KEY (iata, observer_id, bucket); - -ALTER TABLE ONLY analytics_top_talkers_by_iata - ADD CONSTRAINT analytics_top_talkers_by_iata_pkey PRIMARY KEY (iata, sender_name, bucket); - ALTER TABLE ONLY channel_iatas ADD CONSTRAINT channel_iatas_pkey PRIMARY KEY (channel_hash, iata); @@ -1131,38 +721,6 @@ ALTER TABLE ONLY transport_scopes CREATE UNIQUE INDEX accounts_name_uidx ON accounts USING btree (name) WHERE (deactivated_at IS NULL); -CREATE INDEX idx_analytics_hourly_iata_stats_time ON analytics_hourly_iata_stats USING btree (hour); - -CREATE UNIQUE INDEX idx_analytics_hourly_iata_stats_view ON mv_hourly_iata_stats USING btree (iata, hour); - -CREATE INDEX idx_analytics_observer_activity_hourly_time ON analytics_observer_activity_hourly USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly USING btree (observer_id, payload_type, bucket); - -CREATE INDEX idx_analytics_path_stats_hourly_time ON analytics_path_stats_hourly USING btree (hour); - -CREATE UNIQUE INDEX idx_analytics_path_stats_hourly_view ON mv_path_stats_hourly USING btree (iata, hour, category, hash_bytes, entries); - -CREATE INDEX idx_analytics_payload_breakdown_by_iata_time ON analytics_payload_breakdown_by_iata USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_payload_breakdown_by_iata_view ON mv_payload_breakdown_by_iata USING btree (iata, payload_type, bucket); - -CREATE INDEX idx_analytics_signal_stats_hourly_time ON analytics_signal_stats_hourly USING btree (hour); - -CREATE UNIQUE INDEX idx_analytics_signal_stats_hourly_view ON mv_signal_stats_hourly USING btree (iata, hour, kind, snr_bin, rssi_bin); - -CREATE INDEX idx_analytics_top_advertisers_by_iata_time ON analytics_top_advertisers_by_iata USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_top_advertisers_by_iata_view ON mv_top_advertisers_by_iata USING btree (iata, node_id, bucket); - -CREATE INDEX idx_analytics_top_observers_by_iata_time ON analytics_top_observers_by_iata USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_top_observers_by_iata_view ON mv_top_observers_by_iata USING btree (iata, observer_id, bucket); - -CREATE INDEX idx_analytics_top_talkers_by_iata_time ON analytics_top_talkers_by_iata USING btree (bucket); - -CREATE UNIQUE INDEX idx_analytics_top_talkers_by_iata_view ON mv_top_talkers_by_iata USING btree (iata, sender_name, bucket); - CREATE INDEX idx_channel_iatas_iata ON channel_iatas USING btree (iata); CREATE INDEX idx_channel_messages_channel ON channel_messages USING btree (channel_id, sent_at DESC); @@ -1189,14 +747,8 @@ CREATE INDEX idx_known_routes_last_seen ON known_routes USING btree (last_seen D CREATE INDEX idx_known_routes_reconfirm ON known_routes USING btree (last_reconfirmed_at); -CREATE INDEX idx_mv_path_stats_hourly_hour ON mv_path_stats_hourly USING btree (hour); - CREATE UNIQUE INDEX idx_mv_radio_presets ON mv_radio_presets USING btree (preset, iata, source_type); -CREATE INDEX idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly USING btree (hour); - -CREATE UNIQUE INDEX idx_mv_top_nodes ON mv_top_nodes_by_iata USING btree (iata, node_id); - CREATE INDEX idx_node_iatas_iata ON node_iatas USING btree (iata, last_heard DESC); CREATE INDEX idx_node_neighbors_neighbor ON node_neighbors USING btree (neighbor_id, iata); @@ -1215,12 +767,14 @@ CREATE INDEX idx_nodes_pubkey ON nodes USING btree (public_key); CREATE INDEX idx_nodes_type_last_seen ON nodes USING btree (node_type, last_seen DESC); -CREATE INDEX idx_observations_heard_brin ON packet_observations USING brin (heard_at); +CREATE INDEX idx_observations_heard ON packet_observations USING btree (heard_at); CREATE INDEX idx_observations_iata_heard ON packet_observations USING btree (iata, heard_at DESC); CREATE INDEX idx_observations_observer ON packet_observations USING btree (observer_id, heard_at DESC); +CREATE INDEX idx_observations_observer_id ON packet_observations USING btree (observer_id, id); + CREATE INDEX idx_observations_packet ON packet_observations USING btree (packet_hash); CREATE INDEX idx_observations_route_evidence ON packet_observations USING btree (iata, hash_size, decode(md5(path_bytes), 'hex'::text), heard_at DESC, id DESC) WHERE ((path_bytes IS NOT NULL) AND (payload_type IS NOT NULL) AND (payload_type <> 9) AND (hop_count >= 2)); @@ -1229,6 +783,8 @@ CREATE INDEX idx_observer_locations_recent ON observer_locations USING btree (ob CREATE INDEX idx_observer_owners_node ON observer_owners USING btree (owner_node_id) WHERE (owner_node_id IS NOT NULL); +CREATE INDEX idx_observers_last_iata ON observers USING btree (last_iata) WHERE (last_iata IS NOT NULL); + CREATE INDEX idx_observers_last_seen ON observers USING btree (last_seen DESC); CREATE INDEX idx_observers_pubkey ON observers USING btree (public_key); @@ -1267,6 +823,24 @@ CREATE INDEX idx_telemetry_reported_brin ON observer_telemetry USING brin (repor CREATE INDEX idx_trace_iatas_iata ON trace_iatas USING btree (iata); +CREATE INDEX idx_trace_tags_last_heard ON trace_tags USING btree (last_heard_at DESC); + +CREATE INDEX idx_trace_tags_first_heard ON trace_tags USING btree (first_heard_at); + +CREATE INDEX idx_analytics_observer_activity_observer ON analytics_hourly_observer_activity USING btree (observer_id, hour); + +CREATE INDEX idx_analytics_packet_sets_iatas ON analytics_hourly_packet_sets USING gin (iatas); + +CREATE INDEX idx_analytics_scope_sets_iatas ON analytics_hourly_scope_sets USING gin (iatas); + +CREATE INDEX idx_analytics_scope_sets_scope ON analytics_hourly_scope_sets USING btree (scope_id, hour); + +CREATE INDEX idx_analytics_advert_sets_iatas ON analytics_hourly_advert_sets USING gin (iatas); + +CREATE INDEX idx_analytics_talker_sets_iatas ON analytics_hourly_talker_sets USING gin (iatas); + +CREATE INDEX idx_analytics_rollup_hours_missing ON analytics_rollup_hours USING btree (hour) WHERE (status = 'missing'::text); + CREATE INDEX idx_transport_scopes_fingerprint ON transport_scopes USING btree (key_fingerprint); ALTER TABLE ONLY channel_iatas @@ -1353,7 +927,6 @@ ALTER TABLE ONLY region_iatas ALTER TABLE ONLY trace_iatas ADD CONSTRAINT trace_iatas_iata_fkey FOREIGN KEY (iata) REFERENCES iata_codes(iata) ON DELETE CASCADE; - -- -- diff --git a/db/observer_last_iata_integration_test.go b/db/observer_last_iata_integration_test.go new file mode 100644 index 00000000..ff1c4af3 --- /dev/null +++ b/db/observer_last_iata_integration_test.go @@ -0,0 +1,124 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/MeshCore-Beacon/beacon-server/internal/presence" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestObserverLastIATAPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + c := presence.New(store, time.Hour, time.Hour) + pkA, pkB, pkC := bytes.Repeat([]byte{1}, 32), bytes.Repeat([]byte{2}, 32), bytes.Repeat([]byte{3}, 32) + ids := map[string]uuid.UUID{} + heard := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + // Mirrors ingest: upsert the observer, then record the observation. + packet := func(name string, pk []byte, iata string) { + t.Helper() + id, _, err := c.UpsertObserver(ctx, pk, iata) + if err != nil { + t.Fatal(err) + } + ids[name] = id + heard = heard.Add(time.Second) + hash := []byte(heard.String()) + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ($1, 4, 0, 1, '\x00', '\x00', $2, $2)`, hash, heard); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, `INSERT INTO packet_observations (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +VALUES ($1, $2, $3, $4, 0, 1, 0)`, hash, id, iata, heard); err != nil { + t.Fatal(err) + } + } + status := func(pk []byte) { + t.Helper() + if _, _, err := c.UpsertObserver(ctx, pk, ""); err != nil { + t.Fatal(err) + } + } + + packet("A", pkA, "YVR") + packet("A", pkA, "YYZ") + status(pkA) + packet("B", pkB, "YOW") + if _, err := c.UpdateObserverStatus(ctx, ingest.UpdateObserverStatusParams{PublicKey: pkB, DisplayName: "b"}); err != nil { + t.Fatal(err) + } + packet("B", pkB, "YYJ") // write-through after the identity was dropped + status(pkC) + c.Flush(ctx) + packet("A", pkA, "YVR") // pending in the coalescer, not yet flushed + c.Flush(ctx) + + rows, err := tx.Query(ctx, `SELECT o.id, COALESCE(o.last_iata, ''), COALESCE(( + SELECT po.iata FROM packet_observations po WHERE po.observer_id = o.id ORDER BY po.heard_at DESC LIMIT 1), '') +FROM observers o`) + if err != nil { + t.Fatal(err) + } + n := 0 + for rows.Next() { + var id uuid.UUID + var got, want string + if err := rows.Scan(&id, &got, &want); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("observer %s last_iata = %q, latest observation = %q", id, got, want) + } + n++ + } + if rows.Err() != nil || n != 3 { + t.Fatalf("observers = %d, err %v; want 3", n, rows.Err()) + } + + for name, want := range map[string]string{"A": "YVR", "B": "YYJ"} { + if got, err := store.GetObserverLastIATA(ctx, ids[name]); err != nil || got != want { + t.Errorf("GetObserverLastIATA(%s) = %q, %v; want %q", name, got, err, want) + } + } + var idC uuid.UUID + if err := tx.QueryRow(ctx, "SELECT id FROM observers WHERE public_key = $1", pkC).Scan(&idC); err != nil { + t.Fatal(err) + } + if got, err := store.GetObserverLastIATA(ctx, idC); err != nil || got != "" { + t.Errorf("GetObserverLastIATA(no packets) = %q, %v; want empty", got, err) + } + + page, err := store.ListObservers(ctx, []string{"YYJ", "YYZ"}, "", "", "", "", "", 0, 10) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].IATA != "YYJ" { + t.Errorf("ListObservers(YYJ,YYZ) = %+v; want only B in YYJ", page.Items) + } +} diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 64c82df8..5c0b66bc 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -24,13 +24,11 @@ func TestObserverMetricsPostgres(t *testing.T) { } store := &Store{q: sqlc.New(tx)} id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + // Hourly activity is rolled before cleanup and survives the raw rows. + rollTxHours(t, ctx, tx) if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { t.Fatal(err) } - // The initial archive migration retains unknown payloads without a second rebuild. - if err := store.RefreshObserverActivity(ctx); err != nil { - t.Fatal(err) - } until := time.Now().UTC().Truncate(time.Hour) activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until) if err != nil { diff --git a/db/observer_retention_integration_test.go b/db/observer_retention_integration_test.go index 8e64ea9c..1992c536 100644 --- a/db/observer_retention_integration_test.go +++ b/db/observer_retention_integration_test.go @@ -96,7 +96,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) exec("TRUNCATE pg_temp.observers CASCADE") coalescer := presence.New(store, time.Second, time.Second) pubkey := bytes.Repeat([]byte{42}, 32) - oldID, _, err := coalescer.UpsertObserver(ctx, pubkey) + oldID, _, err := coalescer.UpsertObserver(ctx, pubkey, "YVR") if err != nil { t.Fatal(err) } @@ -108,7 +108,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil || len(ids) != 1 || ids[0] != oldID { t.Fatalf("cached expired observer was not deleted: %v, %v", ids, err) } - newID, _, err := coalescer.UpsertObserver(ctx, pubkey) + newID, _, err := coalescer.UpsertObserver(ctx, pubkey, "YVR") if err != nil || newID == oldID { t.Fatalf("returning observer reused a deleted ID: %v", err) } @@ -122,7 +122,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) // The persisted row looks stale while fresh activity is still coalesced. cutoff = time.Now().Add(-24 * time.Hour) exec("UPDATE observers SET last_seen=$1 WHERE id=$2", cutoff.Add(-time.Hour), newID) - if _, _, err := coalescer.UpsertObserver(ctx, pubkey); err != nil { + if _, _, err := coalescer.UpsertObserver(ctx, pubkey, "YVR"); err != nil { t.Fatal(err) } if ids, err := coalescer.DeleteOldObservers(ctx, cutoff); err != nil || len(ids) != 0 { diff --git a/db/observers.go b/db/observers.go index 566256c3..395e8871 100644 --- a/db/observers.go +++ b/db/observers.go @@ -20,8 +20,8 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -func (s *Store) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) { - row, err := s.q.UpsertObserver(ctx, pubkey) +func (s *Store) UpsertObserver(ctx context.Context, pubkey []byte, iata string) (uuid.UUID, string, error) { + row, err := s.q.UpsertObserver(ctx, sqlc.UpsertObserverParams{PublicKey: pubkey, Iata: iata, IataAt: ts(time.Now())}) if err != nil { return uuid.Nil, "", err } diff --git a/db/observers_test.go b/db/observers_test.go index 20597f8c..c60572d5 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -28,11 +28,13 @@ func TestUpsertObserver_NilDisplayName(t *testing.T) { pubkey := []byte{0x01, 0x02} mock.EXPECT(). - UpsertObserver(gomock.Any(), pubkey). + UpsertObserver(gomock.Any(), gomock.Cond(func(p sqlc.UpsertObserverParams) bool { + return string(p.PublicKey) == string(pubkey) && p.Iata == "YVR" && p.IataAt.Valid + })). Return(sqlc.Observer{ID: observerID, DisplayName: nil}, nil) store := &Store{q: mock} - id, displayName, err := store.UpsertObserver(context.Background(), pubkey) + id, displayName, err := store.UpsertObserver(context.Background(), pubkey, "YVR") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -53,11 +55,13 @@ func TestUpsertObserver_WithDisplayName(t *testing.T) { name := "test-observer" mock.EXPECT(). - UpsertObserver(gomock.Any(), pubkey). + UpsertObserver(gomock.Any(), gomock.Cond(func(p sqlc.UpsertObserverParams) bool { + return string(p.PublicKey) == string(pubkey) && p.Iata == "YVR" && p.IataAt.Valid + })). Return(sqlc.Observer{ID: observerID, DisplayName: &name}, nil) store := &Store{q: mock} - _, displayName, err := store.UpsertObserver(context.Background(), pubkey) + _, displayName, err := store.UpsertObserver(context.Background(), pubkey, "YVR") if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/db/packet_cleanup_integration_test.go b/db/packet_cleanup_integration_test.go new file mode 100644 index 00000000..eb3815f8 --- /dev/null +++ b/db/packet_cleanup_integration_test.go @@ -0,0 +1,205 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +// Raw deletion and the rollup meet at raw_deleted_before: an hour registered but not rolled +// before its raw rows go becomes partial, while hours behind the holdback roll completely. +func TestPacketCleanupRollupHandoffPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + old := h0.Add(-3 * time.Hour).Add(10 * time.Minute) + if _, err := pool.Exec(ctx, ` +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ('\xff', 2, 0, 1, '\x00', '\x00', $1::timestamptz, $1::timestamptz); +INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +VALUES (100, '\xff', '00000000-0000-0000-0000-00000000000a', 'YVR', $1::timestamptz, 0, 1, 0)`, pgx.QueryExecModeSimpleProtocol, old); err != nil { + t.Fatal(err) + } + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0.Add(-10*time.Hour)); err != nil { + t.Fatal(err) + } + oldest, ok, err := store.OldestMissingRollupHour(ctx) + if err != nil || !ok || !oldest.Equal(old.Truncate(time.Hour)) { + t.Fatalf("oldest missing %s %v %v, want %s", oldest, ok, err, old.Truncate(time.Hour)) + } + + // Cleanup held behind h0 (as rawCutoff does when h0 is the oldest unrolled hour) leaves h0 intact. + if err := store.DeleteOldPackets(ctx, h0.Add(-35*time.Minute)); err != nil { + t.Fatal(err) + } + var deletedBefore time.Time + var remaining int + if err := pool.QueryRow(ctx, `SELECT raw_deleted_before, (SELECT count(*) FROM packet_observations) + FROM analytics_raw_state`).Scan(&deletedBefore, &remaining); err != nil { + t.Fatal(err) + } + if !deletedBefore.Equal(old) || remaining != 10 { + t.Fatalf("raw_deleted_before %s, %d observations left; want %s, 10", deletedBefore, remaining, old) + } + + if err := r.MarkPartialHours(ctx); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + var statuses string + if err := pool.QueryRow(ctx, `SELECT string_agg(status, ',' ORDER BY hour) FROM analytics_rollup_hours + WHERE hour BETWEEN $1 AND $2`, old.Truncate(time.Hour), h0.Add(time.Hour)).Scan(&statuses); err != nil { + t.Fatal(err) + } + if statuses != "partial,complete,complete,complete,complete" { + t.Errorf("statuses %q, want the deleted hour partial and the rest complete", statuses) + } + q := rollupOracles["iata observations"] + if got, want := rowsText(t, ctx, pool, q[0]), rowsText(t, ctx, pool, q[1]); got != want { + t.Errorf("held-back hours lost raw rows:\nrollup:\n%s\nraw:\n%s", got, want) + } + + if err := store.DeleteOldRollups(ctx, h0.Add(time.Hour)); err != nil { + t.Fatal(err) + } + var left int + if err := pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM analytics_rollup_hours WHERE hour < $1) + + (SELECT count(*) FROM analytics_hourly_iata_observations WHERE hour < $1)`, h0.Add(time.Hour)).Scan(&left); err != nil || left != 0 { + t.Errorf("rollup rows before the cutoff: %d, %v", left, err) + } +} + +// An in-flight observation insert holds KEY SHARE on its packet; cleanup skips it, and a +// second cleanup runner skips the first one's cohort instead of waiting. +func TestDeleteOldPacketsSkipsLockedPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YVR'); +INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x01'); +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ('\x01', 4, 0, 1, '\x00', '\x00', now() - INTERVAL '5 days', now() - INTERVAL '5 days')`); err != nil { + t.Fatal(err) + } + params := sqlc.DeleteOldPacketsParams{Cutoff: ts(time.Now().Add(-72 * time.Hour)), BatchSize: 1000} + q := sqlc.New(pool) + + ingest, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer ingest.Rollback(context.Background()) + if _, err := ingest.Exec(ctx, `INSERT INTO packet_observations (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count) +VALUES ('\x01', '00000000-0000-0000-0000-000000000001', 'YVR', now() - INTERVAL '5 days', 0, 1, 0)`); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("must skip in-flight reception: %d %v", n, err) + } + if err := ingest.Commit(ctx); err != nil { + t.Fatal(err) + } + + first, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer first.Rollback(context.Background()) + if n, err := sqlc.New(first).DeleteOldPackets(ctx, params); err != nil || n != 1 { + t.Fatalf("first runner: %d %v", n, err) + } + waitCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + if n, err := q.DeleteOldPackets(waitCtx, params); err != nil || n != 0 { + t.Fatalf("second runner should skip the locked cohort: %d %v", n, err) + } + if err := first.Commit(ctx); err != nil { + t.Fatal(err) + } + var left int + if err := pool.QueryRow(ctx, "SELECT (SELECT count(*) FROM packets) + (SELECT count(*) FROM packet_observations)").Scan(&left); err != nil || left != 0 { + t.Errorf("rows left after cleanup: %d %v", left, err) + } +} + +// Registration from an hour-aligned rollup cutoff never recreates hours cleanup just expired. +func TestRollupRetentionCyclePostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + cutoff := h0.Add(30 * time.Minute) // DeleteOldRollups cutoff; registration starts at the next hour + since := h0.Add(time.Hour) + for range 3 { + if err := store.DeleteOldRollups(ctx, cutoff); err != nil { + t.Fatal(err) + } + if err := r.RegisterHours(ctx, since); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + } + var stale int + var first time.Time + if err := pool.QueryRow(ctx, "SELECT count(*) FILTER (WHERE hour < $1), min(hour) FROM analytics_rollup_hours", cutoff).Scan(&stale, &first); err != nil { + t.Fatal(err) + } + if stale != 0 || !first.Equal(since) { + t.Errorf("%d expired hours re-registered, first hour %s; want 0, %s", stale, first, since) + } +} + +// Retention deletes and partial marking change what readers see, so they must move the +// revision cache keys are built from, even with no new roll. +func TestRollupCoverageMovesRevisionPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete { + t.Fatalf("roll: %d %v", outcome, err) + } + step := func(name string, change bool, do func() error) { + t.Helper() + before := revision(t, ctx, store) + if err := do(); err != nil { + t.Fatal(err) + } + if moved := revision(t, ctx, store) != before; moved != change { + t.Errorf("%s: revision moved=%v, want %v", name, moved, change) + } + } + step("no-op cleanup", false, func() error { return store.DeleteOldRollups(ctx, h0) }) + step("partial marking", true, func() error { + if _, err := pool.Exec(ctx, "UPDATE analytics_raw_state SET raw_deleted_before = $1", h0.Add(90*time.Minute)); err != nil { + return err + } + return r.MarkPartialHours(ctx) + }) + step("repeat partial marking", false, func() error { return r.MarkPartialHours(ctx) }) + step("retention delete", true, func() error { return store.DeleteOldRollups(ctx, h0.Add(time.Hour)) }) +} diff --git a/db/packet_observation_count_integration_test.go b/db/packet_observation_count_integration_test.go new file mode 100644 index 00000000..9f5ebad6 --- /dev/null +++ b/db/packet_observation_count_integration_test.go @@ -0,0 +1,88 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" +) + +func TestPacketObservationCountPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + store := &Store{q: sqlc.New(tx)} + + hash := []byte{0xab, 0xcd} + heard := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) +VALUES ($1, 4, 0, 1, '\x00', '\x00', $2, $2)`, hash, heard); err != nil { + t.Fatal(err) + } + a, _, err := store.UpsertObserver(ctx, bytes.Repeat([]byte{1}, 32), "YVR") + if err != nil { + t.Fatal(err) + } + b, _, err := store.UpsertObserver(ctx, bytes.Repeat([]byte{2}, 32), "YYZ") + if err != nil { + t.Fatal(err) + } + + steps := []struct { + obs ingest.InsertObservationParams + wantInserted bool + wantCount int64 + }{ + {ingest.InsertObservationParams{PacketHash: hash, ObserverID: a, IATA: "YVR", HeardAt: heard}, true, 1}, + {ingest.InsertObservationParams{PacketHash: hash, ObserverID: b, IATA: "YYZ", HeardAt: heard}, true, 2}, + {ingest.InsertObservationParams{PacketHash: hash, ObserverID: a, IATA: "YVR", HeardAt: heard}, false, 2}, // broker duplicate + } + for i, s := range steps { + inserted, count, err := store.InsertObservation(ctx, s.obs) + if err != nil { + t.Fatal(err) + } + if inserted != s.wantInserted || count != s.wantCount { + t.Errorf("step %d: inserted=%v count=%d, want %v, %d", i, inserted, count, s.wantInserted, s.wantCount) + } + } + + var stored, raw int64 + if err := tx.QueryRow(ctx, `SELECT p.observation_count, (SELECT count(*) FROM packet_observations po WHERE po.packet_hash = p.packet_hash) +FROM packets p WHERE p.packet_hash = $1`, hash).Scan(&stored, &raw); err != nil { + t.Fatal(err) + } + if stored != raw || stored != 2 { + t.Errorf("packets.observation_count = %d, raw count = %d; want 2", stored, raw) + } + + page, err := store.ListPackets(ctx, nil, nil, nil, nil, time.Time{}, time.Time{}, 0, 10) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].ObservationCount != 2 { + t.Errorf("ListPackets observation counts = %+v, want one packet with 2", page.Items) + } +} diff --git a/db/packets.go b/db/packets.go index 2acc5d42..620acceb 100644 --- a/db/packets.go +++ b/db/packets.go @@ -8,7 +8,6 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" - "errors" "fmt" "log/slog" "time" @@ -17,7 +16,6 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/google/uuid" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/meshcore-go/meshcore-go" ) @@ -514,7 +512,7 @@ func (s *Store) UpsertIATA(ctx context.Context, iata string) error { return s.q.UpsertIATA(ctx, iata) } -func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservationParams) (bool, error) { +func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservationParams) (bool, int64, error) { params := sqlc.InsertObservationParams{ PacketHash: o.PacketHash, ObserverID: o.ObserverID, @@ -536,13 +534,10 @@ func (s *Store) InsertObservation(ctx context.Context, o ingest.InsertObservatio AirtimeMs: o.AirtimeMs, } row, err := s.q.InsertObservation(ctx, params) - if errors.Is(err, pgx.ErrNoRows) { - return false, nil // conflict, not an error - } if err != nil { - return false, err + return false, 0, err } - return row.ID != 0, nil + return row.Inserted, row.ObservationCount, nil } func (s *Store) ListNodeObservations(ctx context.Context, nodeID uuid.UUID, cursor int64, limit int32) (api.Page[api.PacketObservationSummary], error) { @@ -584,10 +579,6 @@ func (s *Store) ListNodeObservations(ctx context.Context, nodeID uuid.UUID, curs }, nil } -func (s *Store) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) { - return s.q.GetPacketObservationCount(ctx, packetHash) -} - // Small because each packet cascades to its observations. const packetDeleteBatch = 1000 diff --git a/db/packets_integration_test.go b/db/packets_integration_test.go index 10521e36..2d019ee8 100644 --- a/db/packets_integration_test.go +++ b/db/packets_integration_test.go @@ -56,7 +56,8 @@ VALUES ('\x01', 4, 0, 1, '\x00', '\x00', NOW(), NOW())`) _, err = tx.Exec(ctx, ` INSERT INTO observers (id, public_key) VALUES ('00000000-0000-0000-0000-000000000001', '\x02'); INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes) -VALUES (1, '\x01', '00000000-0000-0000-0000-000000000001', 'YVR', NOW(), 2, 1, 2, '\x1122');`) +VALUES (1, '\x01', '00000000-0000-0000-0000-000000000001', 'YVR', NOW(), 2, 1, 2, '\x1122'); +UPDATE packets SET observation_count = 1 WHERE packet_hash = '\x01';`) if err != nil { t.Fatal(err) } diff --git a/db/packets_test.go b/db/packets_test.go index d93aac57..c5f06271 100644 --- a/db/packets_test.go +++ b/db/packets_test.go @@ -12,7 +12,6 @@ import ( mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/google/uuid" - "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "go.uber.org/mock/gomock" ) @@ -218,10 +217,10 @@ func TestInsertObservation_Success(t *testing.T) { mock.EXPECT(). InsertObservation(gomock.Any(), gomock.Any()). - Return(sqlc.PacketObservation{ID: 1}, nil) + Return(sqlc.InsertObservationRow{Inserted: true, ObservationCount: 4}, nil) store := &Store{q: mock} - inserted, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ + inserted, count, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ PacketHash: []byte{0xde, 0xad}, ObserverID: observerID, IATA: "YVR", @@ -230,8 +229,8 @@ func TestInsertObservation_Success(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if !inserted { - t.Error("expected inserted true") + if !inserted || count != 4 { + t.Errorf("got inserted=%v count=%d, want true, 4", inserted, count) } } @@ -241,18 +240,18 @@ func TestInsertObservation_Conflict(t *testing.T) { mock.EXPECT(). InsertObservation(gomock.Any(), gomock.Any()). - Return(sqlc.PacketObservation{}, pgx.ErrNoRows) + Return(sqlc.InsertObservationRow{Inserted: false, ObservationCount: 2}, nil) store := &Store{q: mock} - inserted, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ + inserted, count, err := store.InsertObservation(context.Background(), ingest.InsertObservationParams{ PacketHash: []byte{0xde, 0xad}, HeardAt: time.Now(), }) if err != nil { t.Fatalf("expected nil error on conflict, got %v", err) } - if inserted { - t.Error("expected inserted false on conflict") + if inserted || count != 2 { + t.Errorf("got inserted=%v count=%d, want false, 2", inserted, count) } } diff --git a/db/paths.go b/db/paths.go index 9df530d6..19cffd8c 100644 --- a/db/paths.go +++ b/db/paths.go @@ -72,5 +72,3 @@ func (s *Store) GetPathStats(ctx context.Context, since, until time.Time, iatas } return stats, nil } - -func (s *Store) RefreshPathStats(ctx context.Context) error { return s.q.RefreshPathStats(ctx) } diff --git a/db/paths_integration_test.go b/db/paths_integration_test.go index a49f8ccd..656c48f3 100644 --- a/db/paths_integration_test.go +++ b/db/paths_integration_test.go @@ -66,9 +66,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_ if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { t.Fatal(err) } - if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW mv_path_stats_hourly"); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) store := &Store{q: sqlc.New(tx)} until := since.Add(4 * time.Hour) for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { @@ -115,7 +113,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_ } } w := httptest.NewRecorder() - handlers.StatsRouter(store, nil).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) + handlers.StatsRouter(store, handlers.StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) var response api.PathStats if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 14 || response.Hashed != 5 { t.Fatalf("HTTP %d: %s (%v)", w.Code, w.Body.String(), err) @@ -135,9 +133,7 @@ SELECT int4send(1000+i),'00000000-0000-0000-0000-000000000001',$1::timestamptz+i if err != nil || stale.Receptions != 18 { t.Fatalf("snapshot lost: %+v %v", stale, err) } - if err := store.RefreshPathStats(ctx); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) got, err := store.GetPathStats(ctx, since, until, nil) if err != nil { t.Fatal(err) @@ -165,9 +161,7 @@ SELECT int4send(1000+i),'00000000-0000-0000-0000-000000000001',$1::timestamptz+i if _, err = tx.Exec(ctx, "TRUNCATE packet_observations"); err != nil { t.Fatal(err) } - if err := store.RefreshPathStats(ctx); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) got, err = store.GetPathStats(ctx, since, until, nil) if err != nil || got.Receptions != 0 || got.Hourly == nil || got.PathLengths == nil || len(got.HashWidths) != 3 { t.Fatalf("empty: %+v %v", got, err) diff --git a/db/presence.go b/db/presence.go index 41222669..9120d540 100644 --- a/db/presence.go +++ b/db/presence.go @@ -12,11 +12,13 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error { +func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, iatas []string, iataAts []time.Time) error { return s.q.TouchObservers(ctx, sqlc.TouchObserversParams{ Column1: ids, Column2: toTimestamptzs(seen), Column3: counts, + Column4: iatas, + Column5: nullableTimes(iataAts), }) } diff --git a/db/queries/paths.sql b/db/queries/paths.sql index 93703be0..8cdaf51b 100644 --- a/db/queries/paths.sql +++ b/db/queries/paths.sql @@ -1,11 +1,11 @@ -- name: GetPathStats :many --- All HTTP aggregates read the compact classification snapshot. +-- All HTTP aggregates read the hourly rollup, never observations. WITH readings AS ( - SELECT * FROM mv_path_stats_hourly + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 UNION ALL - SELECT * FROM mv_path_stats_hourly + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) ) @@ -17,5 +17,3 @@ FROM readings GROUP BY GROUPING SETS ((category, hash_bytes, entries), (hour, category, hash_bytes)) ORDER BY is_hourly, hour, category, hash_bytes, entries; --- name: RefreshPathStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_path_stats_hourly; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 65e2ec3f..349d621a 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -70,32 +70,39 @@ SELECT id FROM transport_scopes WHERE name = $1; SELECT name FROM transport_scopes ORDER BY name; -- name: GetScopeByName :one +-- Packets and IATAs cover every retained rollup hour; IATAs are those that heard the +-- scope's own packets. Observer and node counts are current memberships. SELECT ts.name, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count, - array_remove(array_agg(DISTINCT po.iata ORDER BY po.iata), NULL)::text[] AS iatas + COALESCE((SELECT SUM(s.packets) FROM analytics_hourly_scope_sets s WHERE s.scope_id = ts.id), 0)::bigint AS packet_count, + (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id)::bigint AS observer_count, + (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id)::bigint AS node_count, + cardinality(i.iatas)::bigint AS iata_count, + i.iatas FROM transport_scopes ts -LEFT JOIN packets p ON p.scope_id = ts.id -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id -WHERE ts.name = $1 -GROUP BY ts.name; +CROSS JOIN LATERAL ( + SELECT COALESCE(array_agg(DISTINCT x.iata::text ORDER BY x.iata::text), '{}')::text[] AS iatas + FROM analytics_hourly_scope_sets s CROSS JOIN LATERAL unnest(s.iatas) AS x(iata) + WHERE s.scope_id = ts.id +) i +WHERE ts.name = $1; -- ============================================================ -- OBSERVERS -- ============================================================ -- name: UpsertObserver :one -INSERT INTO observers (public_key, observer_type, last_seen) -VALUES ($1, 'unknown', NOW()) +-- Empty iata (status/neighbors) leaves last_iata alone; otherwise the newest iata_at wins. +INSERT INTO observers (public_key, observer_type, last_seen, last_iata, last_iata_at) +VALUES ($1, 'unknown', NOW(), NULLIF(sqlc.arg(iata)::text, ''), + CASE WHEN sqlc.arg(iata)::text <> '' THEN sqlc.arg(iata_at)::timestamptz END) ON CONFLICT (public_key) DO UPDATE SET last_seen = NOW(), - observation_count = observers.observation_count + 1 + observation_count = observers.observation_count + 1, + last_iata = CASE WHEN EXCLUDED.last_iata IS NOT NULL + AND (observers.last_iata_at IS NULL OR EXCLUDED.last_iata_at > observers.last_iata_at) + THEN EXCLUDED.last_iata ELSE observers.last_iata END, + last_iata_at = GREATEST(observers.last_iata_at, EXCLUDED.last_iata_at) RETURNING *; -- name: UpdateObserverStatus :one @@ -123,11 +130,16 @@ RETURNING id; -- from regressing a newer write-through (e.g. a status update). UPDATE observers o SET last_seen = GREATEST(o.last_seen, v.seen), - observation_count = COALESCE(o.observation_count, 0) + v.delta + observation_count = COALESCE(o.observation_count, 0) + v.delta, + last_iata = CASE WHEN v.iata <> '' AND (o.last_iata_at IS NULL OR v.iata_at > o.last_iata_at) + THEN v.iata ELSE o.last_iata END, + last_iata_at = GREATEST(o.last_iata_at, v.iata_at) FROM ( SELECT unnest($1::uuid[]) AS id, unnest($2::timestamptz[]) AS seen, - unnest($3::int[]) AS delta + unnest($3::int[]) AS delta, + unnest($4::text[]) AS iata, + unnest($5::timestamptz[]) AS iata_at ) v WHERE o.id = v.id; @@ -171,23 +183,13 @@ COALESCE(CASE WHEN GREATEST(COALESCE(o.last_status_at, o.last_seen), o.last_seen) > NOW() - INTERVAL '5 minutes' THEN 'online' ELSE 'offline' END, 'offline')::text AS status, -COALESCE(( - SELECT po.iata - FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC - LIMIT 1 -), '')::text AS iata +COALESCE(o.last_iata, '')::text AS iata FROM observers o LEFT JOIN observer_brokers ob ON ob.observer_id = o.id LEFT JOIN observer_scopes os ON os.observer_id = o.id LEFT JOIN transport_scopes ts ON ts.id = os.scope_id WHERE - (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ( - SELECT po.iata FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC LIMIT 1 - ) = ANY($1::bpchar[])) + (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR o.last_iata = ANY($1::bpchar[])) AND ($2 = '' OR o.observer_type = $2) AND ($3 = '' OR ob.broker_name = $3) AND ($4 = '' OR CASE @@ -206,10 +208,7 @@ ORDER BY o.last_seen DESC LIMIT $7; -- name: GetObserverLastIATA :one -SELECT iata FROM packet_observations -WHERE observer_id = $1 -ORDER BY heard_at DESC -LIMIT 1; +SELECT COALESCE(last_iata, '')::text FROM observers WHERE id = $1; -- name: GetObserverRadio :one SELECT radio_freq_mhz, radio_bw_khz, radio_sf, radio_cr @@ -265,7 +264,7 @@ ORDER BY count DESC; -- name: GetObserverActivityHourly :many -- Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. SELECT - date_bin($3::interval, bucket, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + date_bin($3::interval, hour, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, SUM(observations)::bigint AS observations, COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, @@ -274,15 +273,15 @@ SELECT COALESCE(MIN(snr_min), 0)::real AS snr_min, COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz +FROM analytics_hourly_observer_activity +WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < @until::timestamptz GROUP BY 1 ORDER BY 1; -- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count -FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz +FROM analytics_hourly_observer_activity +WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -361,6 +360,9 @@ WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name; -- ============================================================ -- name: UpsertPacket :one +-- A new TRACE packet adds itself to its tag summary in the same statement, so the summary +-- can't miss a stored packet. 'TRACE' > 'PING'; the longest path keeps the best payload. +WITH up AS ( INSERT INTO packets ( packet_hash, payload_type, @@ -383,8 +385,25 @@ INSERT INTO packets ( ) ON CONFLICT (packet_hash) DO UPDATE SET last_heard_at = NOW() -RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, (xmax = 0) -AS inserted; +RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, trace_tag, scope_id, (xmax = 0) +AS inserted +), trace AS ( + INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, packet_count, trace_type, scope_id, best_payload, best_path_len) + SELECT u.trace_tag, u.first_heard_at, u.last_heard_at, 1, u.parsed_payload->>'type', u.scope_id, u.parsed_payload, + CASE WHEN jsonb_typeof(u.parsed_payload->'pathHashes') = 'array' + THEN jsonb_array_length(u.parsed_payload->'pathHashes') ELSE 0 END + FROM up u + WHERE u.inserted AND u.trace_tag IS NOT NULL + ON CONFLICT (trace_tag) DO UPDATE SET + packet_count = trace_tags.packet_count + 1, + trace_type = GREATEST(trace_tags.trace_type, EXCLUDED.trace_type), + scope_id = COALESCE(trace_tags.scope_id, EXCLUDED.scope_id), + best_payload = CASE WHEN EXCLUDED.best_path_len > trace_tags.best_path_len + THEN EXCLUDED.best_payload ELSE trace_tags.best_payload END, + best_path_len = GREATEST(trace_tags.best_path_len, EXCLUDED.best_path_len) +) +SELECT packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, inserted +FROM up; -- name: SetPacketDecrypted :exec UPDATE packets SET decrypted = true WHERE packet_hash = $1; @@ -430,9 +449,6 @@ LEFT JOIN transport_scopes ts ON ts.id = p.scope_id WHERE p.trace_tag = decode($1, 'hex') ORDER BY p.first_heard_at ASC; --- name: GetPacketObservationCount :one -SELECT COUNT(*) FROM packet_observations WHERE packet_hash = $1; - -- name: ListPackets :many -- Returns packets with the latest observation rolled in for display. -- Pass cursor=0 to start from the beginning. IATA-filtered requests are @@ -457,7 +473,7 @@ SELECT p.last_heard_at, p.scope_id, ts.name AS scope_name, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, @@ -568,7 +584,7 @@ SELECT sh.site_heard_at, sat.scan_saturated, sat.scan_floor, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, @@ -614,7 +630,7 @@ SELECT p.route_type, p.first_heard_at, p.last_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, po.iata AS latest_observer_iata, @@ -637,10 +653,24 @@ WHERE po.id > $1 ORDER BY po.id ASC LIMIT $6; - -- name: DeleteOldPackets :one --- Locks, archives and cascades one bounded packet cohort in a single transaction. -SELECT archive_delete_packets(@cutoff::timestamptz, @batch_size::integer)::bigint; +-- Deletes one bounded cohort (observations cascade). SKIP LOCKED leaves packets an in-flight +-- observation insert holds. raw_deleted_before tells the rollup which hours lost raw rows. +WITH victims AS MATERIALIZED ( + SELECT vp.packet_hash, vp.last_heard_at FROM packets vp + WHERE vp.last_heard_at < @cutoff::timestamptz + ORDER BY vp.last_heard_at, vp.packet_hash + LIMIT @batch_size::integer + FOR UPDATE OF vp SKIP LOCKED +), deleted AS ( + DELETE FROM packets dp USING victims v WHERE dp.packet_hash = v.packet_hash + RETURNING dp.last_heard_at +), mark AS ( + UPDATE analytics_raw_state + SET raw_deleted_before = GREATEST(raw_deleted_before, (SELECT max(last_heard_at) FROM deleted)) + WHERE EXISTS (SELECT 1 FROM deleted) +) +SELECT count(*) FROM deleted; -- name: DeleteOldNodes :exec -- Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- @@ -682,6 +712,8 @@ DELETE FROM trace_iatas WHERE last_heard < $1; -- ============================================================ -- name: InsertObservation :one +-- Bumps packets.observation_count only for a new row; a duplicate returns the current count. +WITH ins AS ( INSERT INTO packet_observations ( packet_hash, observer_id, @@ -705,7 +737,15 @@ INSERT INTO packet_observations ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING -RETURNING *; +RETURNING packet_hash +), bump AS ( + UPDATE packets p SET observation_count = p.observation_count + 1 + FROM ins WHERE p.packet_hash = ins.packet_hash + RETURNING p.observation_count +) +SELECT EXISTS (SELECT 1 FROM ins) AS inserted, + COALESCE((SELECT b.observation_count FROM bump b), + (SELECT pk.observation_count FROM packets pk WHERE pk.packet_hash = $1), 0)::bigint AS observation_count; -- name: ListObservationsForPacket :many SELECT po.*, o.display_name AS observer_name @@ -849,8 +889,7 @@ INSERT INTO channels (channel_hash, key_fingerprint, name, hashtag, is_hashtag, VALUES ($1, $2::bytea, $3, $4, $5, ($2 IS NOT NULL), NOW()) ON CONFLICT (channel_hash, key_fingerprint) DO UPDATE SET last_seen = NOW(), - name = COALESCE(EXCLUDED.name, channels.name), - message_count = CASE WHEN $6 THEN channels.message_count + 1 ELSE channels.message_count END + name = COALESCE(EXCLUDED.name, channels.name) RETURNING *; -- name: UpsertChannelHashOnly :one @@ -889,13 +928,28 @@ ON CONFLICT (channel_hash, iata) DO UPDATE SET last_heard = EXCLUDED.last_heard WHERE EXCLUDED.last_heard > channel_iatas.last_heard + INTERVAL '1 hour'; --- name: UpsertTraceIATA :exec --- Refreshes at most hourly so repeat hears don't churn the row. -INSERT INTO trace_iatas (trace_tag, iata, last_heard) -VALUES ($1, $2, $3) -ON CONFLICT (trace_tag, iata) DO UPDATE SET - last_heard = EXCLUDED.last_heard -WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour'; +-- name: RecordTrace :exec +-- One hearing of a TRACE packet: trace_iatas refreshes at most hourly and the tag's heard +-- window widens (the first hearing replaces the packet's provisional times). Packet counts +-- and payloads come from UpsertPacket. +WITH iata AS ( + INSERT INTO trace_iatas (trace_tag, iata, last_heard) + VALUES (@trace_tag, @iata, @heard_at) + ON CONFLICT (trace_tag, iata) DO UPDATE SET + last_heard = EXCLUDED.last_heard + WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour' +) +INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, heard) +VALUES (@trace_tag, @heard_at, @heard_at, true) +ON CONFLICT (trace_tag) DO UPDATE SET + first_heard_at = CASE WHEN trace_tags.heard THEN LEAST(trace_tags.first_heard_at, EXCLUDED.first_heard_at) + ELSE EXCLUDED.first_heard_at END, + last_heard_at = CASE WHEN trace_tags.heard THEN GREATEST(trace_tags.last_heard_at, EXCLUDED.last_heard_at) + ELSE EXCLUDED.last_heard_at END, + heard = true; + +-- name: DeleteOldTraceTags :exec +DELETE FROM trace_tags WHERE last_heard_at < $1; -- name: ListChannels :many -- Channels ordered by last seen, optionally filtered by hash and/or IATAs. @@ -936,7 +990,6 @@ LIMIT @page_limit; -- name: GetChannelByID :one SELECT * FROM channels WHERE id = $1; - -- ============================================================ -- CHANNEL MESSAGES -- ============================================================ @@ -944,11 +997,22 @@ SELECT * FROM channels WHERE id = $1; -- name: InsertChannelMessage :one -- Read the immutable first-packet scope in the same statement as insertion. -- A later reception's transport code must not give live and historical messages different tags. +-- message_count is a lifetime count, bumped only for a new message. A historical (backfilled) +-- message queues its observation hours for a re-roll in the same statement, so the two can't diverge. WITH inserted AS ( INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (packet_hash) DO NOTHING - RETURNING id, packet_hash + RETURNING id, packet_hash, channel_id +), bump AS ( + UPDATE channels c SET message_count = c.message_count + 1 + FROM inserted WHERE c.id = inserted.channel_id +), dirty AS ( + INSERT INTO analytics_dirty_hours (hour) + SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC') + FROM packet_observations po JOIN inserted i ON i.packet_hash = po.packet_hash + WHERE $6::boolean + ON CONFLICT (hour) DO UPDATE SET enqueued_at = now() ) SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present FROM inserted @@ -961,7 +1025,7 @@ LEFT JOIN transport_scopes ts ON ts.id = p.scope_id; -- Pass empty string for iata to skip IATA filtering. -- Pass cursor=0 to start from the beginning. SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -980,7 +1044,7 @@ LIMIT $6; -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -999,7 +1063,7 @@ LIMIT $5; -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. SELECT DISTINCT ON (cm.id) cm.*, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count + p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -1017,7 +1081,7 @@ LIMIT $6; -- Returns messages after the given message ID, ordered oldest first. -- Used for WS reconnect backfill. SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -1033,38 +1097,41 @@ LIMIT $4; -- STATS -- ============================================================ --- name: GetStatsOverview :one -SELECT - COUNT(DISTINCT po.packet_hash) AS total_packets, - COUNT(*) AS total_observations, - COUNT(DISTINCT po.observer_id) AS active_observers, - COUNT(DISTINCT po.iata) AS active_iatas -FROM packet_observations po -WHERE po.heard_at > NOW() - INTERVAL '24 hours' - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])); - -- name: GetHourlyStats :many -SELECT iata, hour, observation_count, unique_packets, active_observers -FROM mv_hourly_iata_stats -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND hour >= NOW() - $2::interval +SELECT iata, hour, observation_count +FROM analytics_hourly_iata_observations +WHERE (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + AND hour >= @since::timestamptz ORDER BY iata, hour; -- name: GetTopNodes :many -SELECT * FROM mv_top_nodes_by_iata -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) -ORDER BY observation_count DESC -LIMIT $2; +-- Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; +-- node_id is NULL once the node row is gone. iata is a representative one. +WITH ranked AS ( + SELECT origin_pubkey, SUM(observations)::bigint AS observation_count, MAX(last_heard) AS last_heard, + MAX(iata) AS iata, MAX(name) AS name, MAX(node_type) AS node_type + FROM analytics_hourly_advert_hearings + WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + GROUP BY origin_pubkey + ORDER BY observation_count DESC, origin_pubkey + LIMIT @row_limit +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, r.name, '')::text AS name, COALESCE(n.node_type, r.node_type, 0)::smallint AS node_type, + r.iata::bpchar AS iata, r.observation_count, r.last_heard::timestamptz AS last_heard +FROM ranked r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.observation_count DESC, r.origin_pubkey; -- name: GetStatsPayloadBreakdown :many --- Payload-type counts for the IATA within the window, summed from the --- precomputed hourly buckets. +-- Payload-type observation counts since the given hour, from the hourly rollup. SELECT payload_type, SUM(count)::bigint AS count -FROM mv_payload_breakdown_by_iata -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND bucket >= NOW() - $2::interval +FROM analytics_hourly_payload_breakdown +WHERE (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + AND hour >= @since::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -1080,38 +1147,53 @@ GROUP BY n.node_type ORDER BY count DESC; -- name: GetStatsTopObservers :many --- Top N observers for the IATA within the window, summed from the precomputed --- hourly buckets. Counts sum across matched IATAs; iata is a representative one. -SELECT - observer_id AS id, - display_name, - observer_type, - COALESCE(SUM(observation_count), 0)::bigint AS observation_count, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_observers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY observer_id, display_name, observer_type -ORDER BY observation_count DESC -LIMIT $3; +-- Top N observers since the given hour. Counts sum across matched IATAs; iata is a +-- representative one. Live names win over the rolled snapshot. +WITH ranked AS ( + SELECT observer_id, SUM(observation_count)::bigint AS observation_count, MAX(iata) AS iata, + MAX(display_name) AS display_name, MAX(observer_type) AS observer_type + FROM analytics_hourly_observer_identity + WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iata = ANY(@iatas::bpchar[])) + GROUP BY observer_id + ORDER BY observation_count DESC, observer_id + LIMIT @row_limit +) +SELECT r.observer_id AS id, COALESCE(o.display_name, r.display_name, '')::text AS display_name, + COALESCE(o.observer_type, r.observer_type, '')::text AS observer_type, + r.observation_count, r.iata::bpchar AS iata +FROM ranked r +LEFT JOIN observers o ON o.id = r.observer_id +ORDER BY r.observation_count DESC, r.observer_id; -- name: GetStatsTopAdvertisers :many --- Top N advertisers in the window, summed from the hourly buckets. -SELECT - node_id AS id, - name, - node_type, - COALESCE(SUM(advert_count), 0)::bigint AS advert_count, - COALESCE(SUM(flood_advert_count), 0)::bigint AS flood_advert_count, - COALESCE(SUM(direct_advert_count), 0)::bigint AS direct_advert_count, - MAX(last_heard)::timestamptz AS last_heard, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_advertisers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY node_id, name, node_type -ORDER BY advert_count DESC -LIMIT $3; +-- Top N advertisers since the given hour. Each ADVERT packet counts once per hour heard, +-- however many of the requested IATAs heard it (IATA-set rollup). Live names win. +WITH ranked AS ( + SELECT origin_pubkey, SUM(advert_packets)::bigint AS advert_count, + SUM(flood_packets)::bigint AS flood_advert_count, SUM(direct_packets)::bigint AS direct_advert_count + FROM analytics_hourly_advert_sets + WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iatas && @iatas::bpchar[]) + GROUP BY origin_pubkey + ORDER BY advert_count DESC, origin_pubkey + LIMIT @row_limit +), heard AS ( + SELECT h.origin_pubkey, MAX(h.last_heard) AS last_heard, MAX(h.iata) AS iata, + MAX(h.name) AS name, MAX(h.node_type) AS node_type + FROM analytics_hourly_advert_hearings h + WHERE h.hour >= @since::timestamptz AND h.origin_pubkey IN (SELECT origin_pubkey FROM ranked) + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR h.iata = ANY(@iatas::bpchar[])) + GROUP BY h.origin_pubkey +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, h.name, '')::text AS name, COALESCE(n.node_type, h.node_type, 0)::smallint AS node_type, + r.advert_count, r.flood_advert_count, r.direct_advert_count, + h.last_heard::timestamptz AS last_heard, COALESCE(h.iata, '')::bpchar AS iata +FROM ranked r +LEFT JOIN heard h ON h.origin_pubkey = r.origin_pubkey +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.advert_count DESC, r.origin_pubkey; -- name: GetStatsClockDrift :many -- Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds @@ -1140,17 +1222,18 @@ FROM page n ORDER BY ABS(n.device_clock_drift_seconds) DESC; -- name: GetStatsTopTalkers :many --- Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. +-- Top N senders since the given hour. Each message counts once per hour heard, however many +-- of the requested IATAs heard it (IATA-set rollup). SELECT sender_name, - COALESCE(SUM(message_count), 0)::bigint AS message_count, + SUM(messages)::bigint AS message_count, MAX(last_sent)::timestamptz AS last_sent -FROM mv_top_talkers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) +FROM analytics_hourly_talker_sets +WHERE hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR iatas && @iatas::bpchar[]) GROUP BY sender_name -ORDER BY message_count DESC -LIMIT $3; +ORDER BY message_count DESC, sender_name +LIMIT @row_limit; -- name: GetRadioPresets :many SELECT preset, iata, source_type, count @@ -1160,37 +1243,35 @@ WHERE ($1::text = '' OR preset = $1::text) ORDER BY preset, iata, source_type; -- name: GetScopeStats :many --- Aggregate matching observations once, separately from node memberships to avoid --- a cross-join. Empty IATAs keep the original global counts, including associations --- whose observations have expired; the filtered aggregates are empty in that case. -WITH observation_counts AS ( - SELECT p.scope_id, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count - FROM packet_observations po - JOIN packets p ON p.packet_hash = po.packet_hash - LEFT JOIN observer_scopes os ON os.scope_id = p.scope_id AND os.observer_id = po.observer_id - WHERE po.iata = ANY(sqlc.arg(iatas)::bpchar[]) AND p.scope_id IS NOT NULL - GROUP BY p.scope_id +-- Packets since the given hour come from the IATA-set rollup (counted once per hour heard). +-- Observer and node counts are current memberships; observers filter by their latest IATA. +WITH packet_counts AS ( + SELECT s.scope_id, SUM(s.packets)::bigint AS packet_count + FROM analytics_hourly_scope_sets s + WHERE s.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR s.iatas && @iatas::bpchar[]) + GROUP BY s.scope_id +), observer_counts AS ( + SELECT os.scope_id, COUNT(*)::bigint AS observer_count + FROM observer_scopes os JOIN observers o ON o.id = os.observer_id + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR o.last_iata = ANY(@iatas::bpchar[]) + GROUP BY os.scope_id ), node_counts AS ( - SELECT n.default_scope_id AS scope_id, COUNT(*) AS node_count + SELECT n.default_scope_id AS scope_id, COUNT(*)::bigint AS node_count FROM nodes n - WHERE n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY(sqlc.arg(iatas)::bpchar[])) + WHERE n.default_scope_id IS NOT NULL + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY(@iatas::bpchar[]))) GROUP BY n.default_scope_id ) SELECT ts.name, - CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) - ELSE COALESCE(oc.packet_count, 0) END::bigint AS packet_count, - CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) - ELSE COALESCE(oc.observer_count, 0) END::bigint AS observer_count, - CASE WHEN COALESCE(cardinality(sqlc.arg(iatas)::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) - ELSE COALESCE(nc.node_count, 0) END::bigint AS node_count + COALESCE(pc.packet_count, 0)::bigint AS packet_count, + COALESCE(oc.observer_count, 0)::bigint AS observer_count, + COALESCE(nc.node_count, 0)::bigint AS node_count FROM transport_scopes ts -LEFT JOIN observation_counts oc ON oc.scope_id = ts.id +LEFT JOIN packet_counts pc ON pc.scope_id = ts.id +LEFT JOIN observer_counts oc ON oc.scope_id = ts.id LEFT JOIN node_counts nc ON nc.scope_id = ts.id ORDER BY ts.name; @@ -1270,46 +1351,28 @@ DELETE FROM regions WHERE imported AND NOT (slug = ANY(@keep::text[])) RETURNING -- ============================================================ -- name: ListTraceTags :many --- Returns distinct trace tags with summary info, ordered by most recent first. --- IATA membership comes from trace_iatas (joining observations here spilled the --- hash join). Per-tag details filled in only for the returned page. -WITH tags AS ( - SELECT - p.trace_tag, - MIN(p.first_heard_at) AS first_heard_at, - MAX(p.last_heard_at) AS last_heard_at, - COUNT(*) AS packet_count, - MAX(p.parsed_payload->>'type') AS trace_type - FROM packets p - WHERE p.trace_tag IS NOT NULL - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR p.trace_tag IN ( - SELECT ti.trace_tag FROM trace_iatas ti WHERE ti.iata = ANY($1::bpchar[]))) - AND ($2::text = '' OR p.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) - AND ($3::timestamptz IS NULL OR p.first_heard_at >= $3) - AND ($4::timestamptz IS NULL OR p.first_heard_at <= $4) - AND ($7::text = '' OR p.parsed_payload->>'type' = $7) - GROUP BY p.trace_tag - HAVING ($5::timestamptz IS NULL OR MAX(p.last_heard_at) < $5) - ORDER BY MAX(p.last_heard_at) DESC - LIMIT $6 -) +-- Tags, most recent first. Filters match the tag summary (see RecordTrace). SELECT encode(t.trace_tag, 'hex') AS trace_tag, - t.first_heard_at::timestamptz AS first_heard_at, - t.last_heard_at::timestamptz AS last_heard_at, + t.first_heard_at, + t.last_heard_at, t.packet_count, (SELECT COUNT(*) FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, - t.trace_type::text AS trace_type, - (SELECT p3.parsed_payload - FROM packets p3 - WHERE p3.trace_tag = t.trace_tag - ORDER BY jsonb_array_length(p3.parsed_payload->'pathHashes') DESC - LIMIT 1) AS best_payload -FROM tags t -ORDER BY t.last_heard_at DESC; + COALESCE(t.trace_type, '')::text AS trace_type, + t.best_payload +FROM trace_tags t +WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( + SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) + AND ($2::text = '' OR t.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) + AND ($3::timestamptz IS NULL OR t.first_heard_at >= $3) + AND ($4::timestamptz IS NULL OR t.first_heard_at <= $4) + AND ($5::timestamptz IS NULL OR t.last_heard_at < $5) + AND ($7::text = '' OR t.trace_type = $7) +ORDER BY t.last_heard_at DESC +LIMIT $6; -- ============================================================ -- ROUTES @@ -1487,30 +1550,9 @@ WHERE ns.iata = $1 AND n.node_type IN (2, 3) AND ns.prefix_4 = ANY($2::bytea[]); --- name: RefreshHourlyStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hourly_iata_stats; - --- name: RefreshTopNodes :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_nodes_by_iata; - --- name: RefreshTopObservers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_observers_by_iata; - --- name: RefreshPayloadBreakdown :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_payload_breakdown_by_iata; - --- name: RefreshTopTalkers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_talkers_by_iata; - --- name: RefreshTopAdvertisers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata; - -- name: RefreshRadioPresets :exec REFRESH MATERIALIZED VIEW CONCURRENTLY mv_radio_presets; --- name: RefreshObserverActivity :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_observer_activity_hourly; - -- name: AmbiguousPrefixes :many -- Hop prefixes that match >1 node in an IATA, per width. Computed once per reconfirm run. SELECT iata::text AS iata, 1::int AS len, prefix_1 AS prefix FROM node_short_ids GROUP BY iata, prefix_1 HAVING COUNT(*) > 1 @@ -1631,7 +1673,6 @@ WITH latest AS ( SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, hourly.n AS last_complete_hour FROM hourly; - -- name: GetScopeCatalogue :one SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; diff --git a/db/queries/rollup.sql b/db/queries/rollup.sql new file mode 100644 index 00000000..5b336763 --- /dev/null +++ b/db/queries/rollup.sql @@ -0,0 +1,279 @@ +-- ============================================================ +-- HOURLY ANALYTICS ROLLUPS +-- An hour [H, H+1h) is eligible once now() >= H + 95 min (ingest clamps heard_at to ±30 min). +-- ============================================================ + +-- name: TryRollupLock :one +SELECT pg_try_advisory_lock(25189, 1); + +-- name: RollupUnlock :exec +SELECT pg_advisory_unlock(25189, 1); + +-- name: RegisterRollupHours :exec +-- Adds a 'missing' row for every eligible hour since the earliest retained observation +-- (and not before @since), zero-observation hours included. +INSERT INTO analytics_rollup_hours (hour, status) +SELECT h, 'missing' +-- No observations yet means nothing to register (GREATEST would ignore the NULL MIN). +FROM (SELECT CASE WHEN MIN(heard_at) IS NOT NULL + THEN date_trunc('hour', GREATEST(MIN(heard_at), @since::timestamptz), 'UTC') END AS first_hour + FROM packet_observations) f, + generate_series(f.first_hour, date_trunc('hour', now() - INTERVAL '95 minutes', 'UTC'), INTERVAL '1 hour') h +WHERE f.first_hour IS NOT NULL +ON CONFLICT (hour) DO NOTHING; + +-- name: MarkPartialRollupHours :exec +-- Hours whose raw rows cleanup already started deleting can never be rolled completely. +-- Readers report the new status, so the revision moves too. +WITH marked AS ( + UPDATE analytics_rollup_hours h SET status = 'partial' + FROM analytics_raw_state r + WHERE h.status = 'missing' + AND r.raw_deleted_before IS NOT NULL + AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked); + +-- name: ListMissingRollupHours :many +SELECT hour FROM analytics_rollup_hours +WHERE status = 'missing' +ORDER BY hour +LIMIT $1; + +-- name: ListDirtyRollupHours :many +-- Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours +-- wait: their first roll consumes the entry only if the entry predates its snapshot. +SELECT d.hour FROM analytics_dirty_hours d +JOIN analytics_rollup_hours h ON h.hour = d.hour AND h.status = 'complete' +CROSS JOIN analytics_raw_state r +WHERE r.raw_deleted_before IS NULL OR d.hour > r.raw_deleted_before + INTERVAL '30 minutes' +ORDER BY d.hour +LIMIT $1; + +-- name: DeleteStaleDirtyHours :exec +-- Dirty hours that can no longer be re-rolled. +DELETE FROM analytics_dirty_hours d +WHERE EXISTS (SELECT 1 FROM analytics_rollup_hours h WHERE h.hour = d.hour AND h.status = 'partial') + OR d.hour <= (SELECT raw_deleted_before + INTERVAL '30 minutes' FROM analytics_raw_state); + +-- name: LockRollupHour :one +-- Returns the hour's status, or 'partial' if raw deletion has reached it since registration. +SELECT CASE + WHEN r.raw_deleted_before IS NOT NULL AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + THEN 'partial' + ELSE h.status + END::text AS status +FROM analytics_rollup_hours h CROSS JOIN analytics_raw_state r +WHERE h.hour = $1 +FOR UPDATE OF h; + +-- name: SetRollupHourPartial :exec +WITH marked AS ( + UPDATE analytics_rollup_hours SET status = 'partial' WHERE hour = $1 AND status = 'missing' RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked); + +-- name: FillRollupObs :exec +INSERT INTO rollup_obs (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, + path_len, rssi, snr, airtime_ms, payload_type, route_type, origin_pubkey, scope_id) +SELECT po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, + COALESCE(octet_length(po.path_bytes), 0), po.rssi, po.snr, po.airtime_ms, po.payload_type, + p.route_type, p.origin_pubkey, p.scope_id +FROM packet_observations po +JOIN packets p ON p.packet_hash = po.packet_hash +WHERE po.heard_at >= $1::timestamptz AND po.heard_at < $1::timestamptz + INTERVAL '1 hour'; + +-- name: DeleteRollupHour :exec +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour = @hour::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour = @hour::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour = @hour::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour = @hour::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour = @hour::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour = @hour::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour = @hour::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour = @hour::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour = @hour::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = @hour::timestamptz) +DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = @hour::timestamptz; + +-- name: RollIATAObservations :exec +INSERT INTO analytics_hourly_iata_observations (hour, iata, observation_count) +SELECT @hour::timestamptz, iata, count(*) FROM rollup_obs GROUP BY iata; + +-- name: RollPayloadBreakdown :exec +INSERT INTO analytics_hourly_payload_breakdown (hour, iata, payload_type, count) +SELECT @hour::timestamptz, iata, payload_type, count(*) +FROM rollup_obs WHERE payload_type IS NOT NULL +GROUP BY iata, payload_type; + +-- name: RollSignal :exec +-- kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. +-- rssi=0 AND snr=0 means "not reported". +INSERT INTO analytics_hourly_signal (hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) +SELECT @hour::timestamptz, b.iata, grouping(b.snr_bin, b.rssi_bin)::integer, + COALESCE(b.snr_bin, -1)::integer, COALESCE(b.rssi_bin, -1)::integer, + count(*), count(b.snr), COALESCE(sum(b.snr), 0)::double precision, + count(b.rssi), COALESCE(sum(b.rssi), 0)::double precision +FROM ( + SELECT s.iata, s.snr, s.rssi, + width_bucket(s.snr, -30, 30, 12) AS snr_bin, + width_bucket(s.rssi, -140, 0, 14) AS rssi_bin + FROM ( + SELECT ro.iata, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + AND ro.snr > '-Infinity'::real AND ro.snr < 'Infinity'::real + THEN ro.snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + THEN ro.rssi::double precision END AS rssi + FROM rollup_obs ro + ) s +) b +GROUP BY GROUPING SETS ((b.iata), (b.iata, b.snr_bin), (b.iata, b.rssi_bin)); + +-- name: RollPaths :exec +-- category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). +INSERT INTO analytics_hourly_paths (hour, iata, category, hash_bytes, entries, receptions) +SELECT @hour::timestamptz, c.iata, c.category, c.hash_bytes, c.entries, count(*) +FROM ( + SELECT k.iata, k.category, + CASE WHEN k.category = 0 THEN k.hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN k.category = 0 THEN k.hop_count ELSE 0 END::integer AS entries + FROM ( + SELECT iata, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND path_len = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM rollup_obs + ) k +) c +GROUP BY c.iata, c.category, c.hash_bytes, c.entries; + +-- name: RollObserverActivity :exec +INSERT INTO analytics_hourly_observer_activity (hour, observer_id, payload_type, observations, airtime_ms, airtime_n, + snr_sum, snr_n, snr_min, rssi_sum, rssi_n) +SELECT @hour::timestamptz, observer_id, COALESCE(payload_type, -1)::smallint, + count(*), sum(airtime_ms)::real, count(airtime_ms), + sum(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + count(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), + min(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + sum(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + count(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)) +FROM rollup_obs +GROUP BY observer_id, COALESCE(payload_type, -1); + +-- name: RollObserverIdentity :exec +INSERT INTO analytics_hourly_observer_identity (hour, iata, observer_id, observation_count, display_name, observer_type) +SELECT @hour::timestamptz, r.iata, r.observer_id, r.n, o.display_name, o.observer_type +FROM (SELECT iata, observer_id, count(*) AS n FROM rollup_obs GROUP BY iata, observer_id) r +LEFT JOIN observers o ON o.id = r.observer_id; + +-- name: RollAdvertHearings :exec +INSERT INTO analytics_hourly_advert_hearings (hour, iata, origin_pubkey, observations, last_heard, name, node_type) +SELECT @hour::timestamptz, r.iata, r.origin_pubkey, r.n, r.last_heard, n.name, n.node_type +FROM (SELECT iata, origin_pubkey, count(*) AS n, max(heard_at) AS last_heard + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY iata, origin_pubkey) r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey; + +-- name: RollAdvertSets :exec +-- Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. +INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) +SELECT @hour::timestamptz, origin_pubkey, iatas, count(*), + count(*) FILTER (WHERE route_type IN (0, 1)), + count(*) FILTER (WHERE route_type IN (2, 3)) +FROM (SELECT origin_pubkey, route_type, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY packet_hash, origin_pubkey, route_type) p +GROUP BY origin_pubkey, iatas; + +-- name: RollTalkerSets :exec +INSERT INTO analytics_hourly_talker_sets (hour, sender_name, iatas, messages, last_sent) +SELECT @hour::timestamptz, sender_name, iatas, count(*), max(sent_at) +FROM (SELECT cm.sender_name, max(cm.sent_at) AS sent_at, array_agg(DISTINCT ro.iata ORDER BY ro.iata) AS iatas + FROM rollup_obs ro + JOIN channel_messages cm ON cm.packet_hash = ro.packet_hash + WHERE cm.sender_name IS NOT NULL + GROUP BY cm.id, cm.sender_name) m +GROUP BY sender_name, iatas; + +-- name: RollPacketSets :exec +INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) +SELECT @hour::timestamptz, iatas, count(*) +FROM (SELECT array_agg(DISTINCT iata ORDER BY iata) AS iatas FROM rollup_obs GROUP BY packet_hash) p +GROUP BY iatas; + +-- name: RollScopeSets :exec +INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) +SELECT @hour::timestamptz, iatas, scope_id, count(*) +FROM (SELECT scope_id, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs WHERE scope_id IS NOT NULL + GROUP BY packet_hash, scope_id) p +GROUP BY iatas, scope_id; + +-- name: RollupContentHash :one +-- Order-independent fingerprint of every family row for the hour. +SELECT md5(concat_ws('|', + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_iata_observations t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_payload_breakdown t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_signal t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_paths t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_activity t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_identity t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_hearings t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_talker_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_packet_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = @hour::timestamptz) +))::text AS content_hash; + +-- name: FinishRollupHour :one +-- Marks the hour complete and bumps the revision only when its content changed. +WITH prev AS ( + SELECT ph.content_hash FROM analytics_rollup_hours ph WHERE ph.hour = @hour::timestamptz +), upd AS ( + UPDATE analytics_rollup_hours uh + SET status = 'complete', rolled_at = now(), content_hash = @content_hash::text + WHERE uh.hour = @hour::timestamptz +), undirty AS ( + DELETE FROM analytics_dirty_hours dh WHERE dh.hour = @hour::timestamptz +), rev AS ( + UPDATE analytics_state SET revision = revision + 1 + WHERE (SELECT content_hash FROM prev) IS DISTINCT FROM @content_hash::text + RETURNING revision +) +SELECT EXISTS (SELECT 1 FROM rev)::boolean AS changed; + +-- name: GetAnalyticsRevision :one +-- Both counters only grow, so their sum changes whenever rolled content or coverage does. +SELECT (s.revision + r.coverage)::bigint FROM analytics_state s CROSS JOIN analytics_raw_state r; + +-- name: OldestMissingRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'missing'; + +-- name: DeleteOldRollups :exec +-- Bumps coverage when hours go, so cached responses that included them are not reused. +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour < @cutoff::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour < @cutoff::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour < @cutoff::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour < @cutoff::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour < @cutoff::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour < @cutoff::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour < @cutoff::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour < @cutoff::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour < @cutoff::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour < @cutoff::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour < @cutoff::timestamptz), + l AS (DELETE FROM analytics_dirty_hours tl WHERE tl.hour < @cutoff::timestamptz), + m AS (DELETE FROM analytics_rollup_hours tm WHERE tm.hour < @cutoff::timestamptz RETURNING 1) +UPDATE analytics_raw_state SET coverage = coverage + 1 WHERE EXISTS (SELECT 1 FROM m); diff --git a/db/queries/series.sql b/db/queries/series.sql new file mode 100644 index 00000000..a22ca8e1 --- /dev/null +++ b/db/queries/series.sql @@ -0,0 +1,91 @@ +-- name: GetStatsSeries :many +-- One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours. +-- Counts sum; observers, IATAs and scopes are distinct across the window; packets count +-- once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate. +WITH hours AS ( + SELECT g.h::timestamptz AS hour, COALESCE(r.status, 'missing')::text AS status + FROM generate_series(@since::timestamptz, @until::timestamptz, INTERVAL '1 hour') g(h) + LEFT JOIN analytics_rollup_hours r ON r.hour = g.h + WHERE g.h < @until::timestamptz +), complete AS ( + SELECT hour FROM hours WHERE status = 'complete' +), obs AS ( + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND o.iata = ANY(@iatas::bpchar[]) +), obs_agg AS ( + SELECT hour, sum(observation_count)::bigint AS observations, count(DISTINCT iata)::bigint AS active_iatas + FROM obs GROUP BY GROUPING SETS ((), (hour)) +), pkt AS ( + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND s.iatas && @iatas::bpchar[] +), pkt_agg AS ( + SELECT hour, sum(packets)::bigint AS unique_packets FROM pkt GROUP BY GROUPING SETS ((), (hour)) +), obsr AS ( + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND i.iata = ANY(@iatas::bpchar[]) +), obsr_agg AS ( + SELECT hour, count(DISTINCT observer_id)::bigint AS active_observers FROM obsr GROUP BY GROUPING SETS ((), (hour)) +), scp AS ( + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality(@iatas::bpchar[]) > 0 AND s.iatas && @iatas::bpchar[] +), scp_agg AS ( + SELECT hour, sum(packets)::bigint AS scoped_packets, count(DISTINCT scope_id)::bigint AS active_scopes + FROM scp GROUP BY GROUPING SETS ((), (hour)) +), pth AS ( + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND cardinality(@iatas::bpchar[]) > 0 AND p.iata = ANY(@iatas::bpchar[]) +), pth_agg AS ( + SELECT hour, max(entries)::integer AS max_path_entries FROM pth GROUP BY GROUPING SETS ((), (hour)) +), sig AS ( + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND cardinality(@iatas::bpchar[]) > 0 AND s.iata = ANY(@iatas::bpchar[]) +), sig_agg AS ( + SELECT hour, sum(snr_sum)::double precision AS snr_sum, sum(snr_samples)::bigint AS snr_samples, + sum(rssi_sum)::double precision AS rssi_sum, sum(rssi_samples)::bigint AS rssi_samples + FROM sig GROUP BY GROUPING SETS ((), (hour)) +), keys AS ( + SELECT hour, status FROM hours + UNION ALL + SELECT NULL::timestamptz, 'summary'::text +) +SELECT k.hour, k.status, + COALESCE(o.observations, 0)::bigint AS observations, + COALESCE(p.unique_packets, 0)::bigint AS unique_packets, + COALESCE(ob.active_observers, 0)::bigint AS active_observers, + COALESCE(o.active_iatas, 0)::bigint AS active_iatas, + COALESCE(s.scoped_packets, 0)::bigint AS scoped_packets, + COALESCE(s.active_scopes, 0)::bigint AS active_scopes, + COALESCE(pa.max_path_entries, 0)::integer AS max_path_entries, + COALESCE(sg.snr_sum, 0)::double precision AS snr_sum, + COALESCE(sg.snr_samples, 0)::bigint AS snr_samples, + COALESCE(sg.rssi_sum, 0)::double precision AS rssi_sum, + COALESCE(sg.rssi_samples, 0)::bigint AS rssi_samples +FROM keys k +LEFT JOIN obs_agg o ON o.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pkt_agg p ON p.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN obsr_agg ob ON ob.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN scp_agg s ON s.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pth_agg pa ON pa.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN sig_agg sg ON sg.hour IS NOT DISTINCT FROM k.hour +ORDER BY k.hour NULLS LAST; + +-- name: GetEarliestCompleteRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete'; diff --git a/db/queries/signal.sql b/db/queries/signal.sql index 24bb800c..ecd20cf1 100644 --- a/db/queries/signal.sql +++ b/db/queries/signal.sql @@ -1,12 +1,12 @@ -- name: GetSignalStats :many --- Read compact hourly snapshots, never observations on an HTTP request. +-- Read the hourly rollup, never observations on an HTTP request. -- Weight averages by sample counts instead of averaging regional/hourly means. WITH readings AS ( - SELECT * FROM mv_signal_stats_hourly + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 UNION ALL - SELECT * FROM mv_signal_stats_hourly + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal WHERE hour >= @since::timestamptz AND hour < @until::timestamptz AND cardinality(@iatas::bpchar[]) > 0 AND iata = ANY(@iatas::bpchar[]) ) @@ -27,5 +27,3 @@ FROM readings WHERE kind IN (1, 2) GROUP BY kind, snr_bin, rssi_bin ORDER BY kind, hour, snr_bin, rssi_bin; --- name: RefreshSignalStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_signal_stats_hourly; diff --git a/db/rollup.go b/db/rollup.go new file mode 100644 index 00000000..3c2cab32 --- /dev/null +++ b/db/rollup.go @@ -0,0 +1,202 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + "fmt" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" +) + +// Must match db/sqlc_scratch/rollup_obs.sql, which only exists so sqlc can type-check rollup.sql. +const createRollupObs = `CREATE TEMP TABLE rollup_obs ( + packet_hash bytea NOT NULL, observer_id uuid NOT NULL, iata character(3) NOT NULL, + heard_at timestamptz NOT NULL, path_length_byte smallint NOT NULL, hash_size smallint NOT NULL, + hop_count smallint NOT NULL, path_len integer NOT NULL, rssi smallint, snr real, airtime_ms real, + payload_type smallint, route_type smallint NOT NULL, origin_pubkey bytea, scope_id integer +) ON COMMIT DROP` + +// rollupDelay is how long after an hour closes it becomes eligible to roll (ingest clamps +// heard_at to ±30 min; RegisterRollupHours uses the same bound). +const rollupDelay = 95 * time.Minute + +// RollOutcome is what RollHour did with an hour. +type RollOutcome int + +const ( + RollSkipped RollOutcome = iota // unregistered, or lost a serialization race; retried next pass + RollPartial // raw rows already deleted; never rolled + RollComplete // rolled and marked complete +) + +// RollupSession holds the rollup advisory lock on one dedicated connection. +type RollupSession struct { + conn *pgxpool.Conn + q *sqlc.Queries +} + +// BeginRollup takes the rollup lock. ok is false when another process holds it. +func (s *Store) BeginRollup(ctx context.Context) (*RollupSession, bool, error) { + if s.pool == nil { + return nil, false, errors.New("rollup: store has no connection pool") + } + conn, err := s.pool.Acquire(ctx) + if err != nil { + return nil, false, err + } + q := sqlc.New(conn) + ok, err := q.TryRollupLock(ctx) + if err != nil || !ok { + conn.Release() + return nil, false, err + } + return &RollupSession{conn: conn, q: q}, true, nil +} + +// Close releases the lock. If unlocking fails the connection is closed so the lock can't leak. +func (r *RollupSession) Close(ctx context.Context) { + if err := r.q.RollupUnlock(ctx); err != nil { + _ = r.conn.Hijack().Close(context.Background()) + return + } + r.conn.Release() +} + +// RegisterHours adds a 'missing' row for every eligible hour since max(since, earliest observation). +func (r *RollupSession) RegisterHours(ctx context.Context, since time.Time) error { + return r.q.RegisterRollupHours(ctx, ts(since)) +} + +func (r *RollupSession) MarkPartialHours(ctx context.Context) error { + return r.q.MarkPartialRollupHours(ctx) +} + +// MissingHours returns up to limit unrolled hours, oldest first. +func (r *RollupSession) MissingHours(ctx context.Context, limit int32) ([]time.Time, error) { + rows, err := r.q.ListMissingRollupHours(ctx, limit) + return times(rows), err +} + +// DirtyHours drops dirty hours that can't be re-rolled and returns up to limit of the rest. +func (r *RollupSession) DirtyHours(ctx context.Context, limit int32) ([]time.Time, error) { + if err := r.q.DeleteStaleDirtyHours(ctx); err != nil { + return nil, err + } + rows, err := r.q.ListDirtyRollupHours(ctx, limit) + return times(rows), err +} + +// RollHour rebuilds every rollup family for one hour in a single REPEATABLE READ transaction. +// changed reports whether the hour's content differs from its previous roll. +func (r *RollupSession) RollHour(ctx context.Context, hour time.Time) (outcome RollOutcome, changed bool, err error) { + tx, err := r.conn.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + if err != nil { + return RollSkipped, false, err + } + defer func() { + _ = tx.Rollback(context.Background()) + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "40001" { + outcome, changed, err = RollSkipped, false, nil + } + }() + outcome, changed, err = rollHour(ctx, tx, hour) + if err != nil { + return RollSkipped, false, err + } + return outcome, changed, tx.Commit(ctx) +} + +// rollHour does RollHour's work inside tx; the caller commits. +func rollHour(ctx context.Context, tx pgx.Tx, hour time.Time) (RollOutcome, bool, error) { + q := sqlc.New(tx) + h := ts(hour) + status, err := q.LockRollupHour(ctx, h) + if errors.Is(err, pgx.ErrNoRows) { + return RollSkipped, false, nil + } + if err != nil { + return RollSkipped, false, err + } + if status == "partial" { + return RollPartial, false, q.SetRollupHourPartial(ctx, h) + } + if _, err := tx.Exec(ctx, createRollupObs); err != nil { + return RollSkipped, false, err + } + if err := q.FillRollupObs(ctx, h); err != nil { + return RollSkipped, false, err + } + if _, err := tx.Exec(ctx, "ANALYZE rollup_obs"); err != nil { + return RollSkipped, false, err + } + if err := q.DeleteRollupHour(ctx, h); err != nil { + return RollSkipped, false, err + } + for name, roll := range map[string]func(context.Context, pgtype.Timestamptz) error{ + "iata observations": q.RollIATAObservations, + "payload breakdown": q.RollPayloadBreakdown, + "signal": q.RollSignal, + "paths": q.RollPaths, + "observer activity": q.RollObserverActivity, + "observer identity": q.RollObserverIdentity, + "advert hearings": q.RollAdvertHearings, + "advert sets": q.RollAdvertSets, + "talker sets": q.RollTalkerSets, + "packet sets": q.RollPacketSets, + "scope sets": q.RollScopeSets, + } { + if err := roll(ctx, h); err != nil { + return RollSkipped, false, fmt.Errorf("roll %s: %w", name, err) + } + } + // Dropped here as well as ON COMMIT so a caller can roll several hours in one transaction. + if _, err := tx.Exec(ctx, "DROP TABLE rollup_obs"); err != nil { + return RollSkipped, false, err + } + hash, err := q.RollupContentHash(ctx, h) + if err != nil { + return RollSkipped, false, err + } + changed, err := q.FinishRollupHour(ctx, sqlc.FinishRollupHourParams{Hour: h, ContentHash: hash}) + if err != nil { + return RollSkipped, false, err + } + return RollComplete, changed, nil +} + +// AnalyticsRevision changes whenever rolled content, an hour's status or rollup coverage changes. +func (s *Store) AnalyticsRevision(ctx context.Context) (int64, error) { + return s.q.GetAnalyticsRevision(ctx) +} + +func ts(t time.Time) pgtype.Timestamptz { + return pgtype.Timestamptz{Time: t, Valid: true} +} + +func times(rows []pgtype.Timestamptz) []time.Time { + out := make([]time.Time, len(rows)) + for i, r := range rows { + out[i] = r.Time + } + return out +} + +// OldestMissingRollupHour returns the earliest hour still waiting to be rolled; ok is false if none. +func (s *Store) OldestMissingRollupHour(ctx context.Context) (hour time.Time, ok bool, err error) { + h, err := s.q.OldestMissingRollupHour(ctx) + return h.Time, h.Valid, err +} + +// DeleteOldRollups drops rolled hours, their bookkeeping and queued re-rolls before cutoff. +func (s *Store) DeleteOldRollups(ctx context.Context, cutoff time.Time) error { + return s.q.DeleteOldRollups(ctx, ts(cutoff)) +} diff --git a/db/rollup_integration_test.go b/db/rollup_integration_test.go new file mode 100644 index 00000000..a5cace6f --- /dev/null +++ b/db/rollup_integration_test.go @@ -0,0 +1,515 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// rollupFixture loads observations across two hours, h0 and h0+1h, and returns h0. +// h0+2h is left empty so a zero-observation hour gets registered too. +func rollupFixture(t *testing.T, ctx context.Context, pool *pgxpool.Pool) time.Time { + t.Helper() + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + var h0 time.Time + if err := pool.QueryRow(ctx, "SELECT date_trunc('hour', now(), 'UTC') - INTERVAL '4 hours'").Scan(&h0); err != nil { + t.Fatal(err) + } + _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YVR'), ('YYZ'), ('YOW'); +INSERT INTO transport_scopes (id, name, transport_key, key_fingerprint) +VALUES (1, 'scope-one', decode(repeat('01', 16), 'hex'), decode(repeat('01', 8), 'hex')); +INSERT INTO observers (id, public_key, display_name, observer_type) VALUES + ('00000000-0000-0000-0000-00000000000a', '\x0a', 'obs a', 'repeater'), + ('00000000-0000-0000-0000-00000000000b', '\x0b', 'obs b', NULL); +INSERT INTO nodes (public_key, node_type, name) VALUES ('\xee', 2, 'node e'); +INSERT INTO channels (id, channel_hash) VALUES (1, '\x11'); +-- hash, payload type, route type, origin, scope +INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, origin_pubkey, scope_id, raw_payload, raw_header, first_heard_at, last_heard_at) +SELECT hash, pt, rt, 0, origin, scope, '\x00', '\x00', $1::timestamptz, $1::timestamptz FROM (VALUES + ('\x01'::bytea, 4, 1, '\xee'::bytea, NULL::int), -- flood advert, two IATAs + ('\x02', 4, 2, '\xee', NULL), -- direct advert, one IATA + ('\x03', 5, 1, NULL, 1), -- scoped channel message, two IATAs + ('\x04', 5, 1, NULL, 1), -- scoped channel message, one IATA + ('\x05', 2, 1, NULL, NULL), -- straddles h0 and h0+1h + ('\x06', 9, 1, NULL, NULL), -- TRACE + ('\x07', 2, 1, NULL, NULL) -- invalid path +) v(hash, pt, rt, origin, scope); +INSERT INTO channel_messages (id, channel_id, packet_hash, sender_name, sent_at) VALUES + (1, 1, '\x03', 'alice', $1::timestamptz + INTERVAL '5 minutes'), + (2, 1, '\x04', 'alice', $1::timestamptz + INTERVAL '6 minutes'); +SELECT setval(pg_get_serial_sequence('channel_messages', 'id'), 100); +INSERT INTO packet_observations (id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, airtime_ms, payload_type) +SELECT n, hash, obs::uuid, iata, $1::timestamptz + mins * INTERVAL '1 minute', plb, hs, hc, pb, rssi, snr, air, pt FROM (VALUES + (1, '\x01'::bytea, '00000000-0000-0000-0000-00000000000a', 'YVR', 1, 65, 2, 1, '\xaabb'::bytea, -90, 5.5::real, 12.5::real, 4), + (2, '\x01', '00000000-0000-0000-0000-00000000000b', 'YYZ', 2, 0, 1, 0, NULL, -100, -3.25, NULL, 4), + (3, '\x02', '00000000-0000-0000-0000-00000000000a', 'YVR', 3, 2, 1, 2, '\xaabb', 0, 0, 10, 4), + (4, '\x03', '00000000-0000-0000-0000-00000000000a', 'YVR', 4, 1, 1, 1, '\xaa', -80, 8, 20, 5), + (5, '\x03', '00000000-0000-0000-0000-00000000000b', 'YYZ', 5, 1, 1, 1, '\xbb', -85, 7, NULL, 5), + (6, '\x04', '00000000-0000-0000-0000-00000000000b', 'YYZ', 6, 0, 1, 0, NULL, NULL, NULL, NULL, 5), + (7, '\x05', '00000000-0000-0000-0000-00000000000a', 'YVR', 59, 0, 1, 0, NULL, -120, -12, 5, 2), + (8, '\x05', '00000000-0000-0000-0000-00000000000b', 'YYZ', 61, 0, 1, 0, NULL, -121, -13, 5, 2), + (9, '\x06', '00000000-0000-0000-0000-00000000000a', 'YVR', 7, 3, 1, 3, '\x010203', -95, 2, 30, 9), + (10, '\x07', '00000000-0000-0000-0000-00000000000a', 'YVR', 8, 3, 1, 3, '\xaa', -95, 2, 30, 2) +) v(n, hash, obs, iata, mins, plb, hs, hc, pb, rssi, snr, air, pt);`, pgx.QueryExecModeSimpleProtocol, h0) + if err != nil { + t.Fatal(err) + } + return h0 +} + +// Raw-scan oracles: the SQL the old live views ran, grouped per hour. +var rollupOracles = map[string][2]string{ + "iata observations": { + `SELECT hour, iata, observation_count FROM analytics_hourly_iata_observations`, + `SELECT date_trunc('hour', heard_at, 'UTC'), iata, count(*) FROM packet_observations GROUP BY 1, 2`}, + "payload breakdown": { + `SELECT hour, iata, payload_type, count FROM analytics_hourly_payload_breakdown`, + `SELECT date_trunc('hour', heard_at, 'UTC'), iata, payload_type, count(*) FROM packet_observations + WHERE payload_type IS NOT NULL GROUP BY 1, 2, 3`}, + "signal": { + `SELECT hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal`, + `WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) THEN rssi::double precision END AS rssi + FROM packet_observations + ), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, width_bucket(rssi, -140, 0, 14) AS rssi_bin FROM samples + ) + SELECT hour, iata, grouping(snr_bin, rssi_bin)::integer, COALESCE(snr_bin, -1)::integer, COALESCE(rssi_bin, -1)::integer, + count(*)::bigint, count(snr)::bigint, COALESCE(sum(snr), 0)::double precision, + count(rssi)::bigint, COALESCE(sum(rssi), 0)::double precision + FROM binned GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin))`}, + "paths": { + `SELECT hour, iata, category, hash_bytes, entries, receptions FROM analytics_hourly_paths`, + `WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM packet_observations + ), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified + ) + SELECT hour, iata, category, hash_bytes, entries, count(*)::bigint FROM buckets GROUP BY 1, 2, 3, 4, 5`}, + "observer activity": { + `SELECT hour, observer_id, payload_type, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n + FROM analytics_hourly_observer_activity`, + `SELECT date_trunc('hour', heard_at, 'UTC'), observer_id, COALESCE(payload_type, -1)::smallint, + COUNT(*)::bigint, SUM(airtime_ms)::real, COUNT(airtime_ms)::bigint, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint + FROM packet_observations GROUP BY 1, 2, 3`}, + "observer identity": { + `SELECT hour, iata, observer_id, observation_count, display_name, observer_type FROM analytics_hourly_observer_identity`, + `SELECT date_trunc('hour', po.heard_at, 'UTC'), po.iata, po.observer_id, count(*), o.display_name, o.observer_type + FROM packet_observations po JOIN observers o ON o.id = po.observer_id GROUP BY 1, 2, 3, 5, 6`}, + "advert hearings": { + `SELECT hour, iata, origin_pubkey, observations, last_heard, name, node_type FROM analytics_hourly_advert_hearings`, + `SELECT date_trunc('hour', po.heard_at, 'UTC'), po.iata, p.origin_pubkey, count(*), max(po.heard_at), n.name, n.node_type + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + LEFT JOIN nodes n ON n.public_key = p.origin_pubkey + WHERE po.payload_type = 4 AND p.origin_pubkey IS NOT NULL GROUP BY 1, 2, 3, 6, 7`}, +} + +// Set families must give exact distinct counts for any IATA set, summed per hour. +var rollupSetOracles = map[string][2]string{ + "packets": { + `SELECT COALESCE(sum(packets), 0) FROM analytics_hourly_packet_sets WHERE iatas && $1::bpchar[]`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', heard_at, 'UTC'), packet_hash + FROM packet_observations WHERE iata = ANY($1::bpchar[])) d`}, + "scoped packets": { + `SELECT COALESCE(sum(packets), 0) FROM analytics_hourly_scope_sets WHERE iatas && $1::bpchar[] AND scope_id = 1`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.packet_hash + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND p.scope_id = 1) d`}, + "adverts": { + `SELECT COALESCE(sum(advert_packets), 0) FROM analytics_hourly_advert_sets WHERE iatas && $1::bpchar[]`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.packet_hash + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND po.payload_type = 4) d`}, + "flood adverts": { + `SELECT COALESCE(sum(flood_packets), 0) FROM analytics_hourly_advert_sets WHERE iatas && $1::bpchar[]`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.packet_hash + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND po.payload_type = 4 AND p.route_type IN (0, 1)) d`}, + "messages": { + `SELECT COALESCE(sum(messages), 0) FROM analytics_hourly_talker_sets WHERE iatas && $1::bpchar[] AND sender_name = 'alice'`, + `SELECT count(*) FROM (SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), cm.id + FROM packet_observations po JOIN channel_messages cm ON cm.packet_hash = po.packet_hash + WHERE po.iata = ANY($1::bpchar[]) AND cm.sender_name = 'alice') d`}, +} + +// rollTxHours registers every hour holding observations (eligible or not) and rolls all +// registered hours inside tx, so tx-scoped fixtures can read rollup-backed stats. +func rollTxHours(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + if _, err := tx.Exec(ctx, `INSERT INTO analytics_rollup_hours (hour, status) + SELECT DISTINCT date_trunc('hour', heard_at, 'UTC'), 'missing' FROM packet_observations + ON CONFLICT (hour) DO NOTHING`); err != nil { + t.Fatal(err) + } + rows, err := tx.Query(ctx, "SELECT hour FROM analytics_rollup_hours ORDER BY hour") + if err != nil { + t.Fatal(err) + } + hours, err := pgx.CollectRows(rows, pgx.RowTo[time.Time]) + if err != nil { + t.Fatal(err) + } + for _, h := range hours { + if outcome, _, err := rollHour(ctx, tx, h); err != nil || outcome != RollComplete { + t.Fatalf("roll %s: outcome %d err %v", h, outcome, err) + } + } +} + +// backfillMessage stores a decrypted-late message the way channel backfill does. +func backfillMessage(t *testing.T, ctx context.Context, store *Store, hash byte, sender string, at time.Time) { + t.Helper() + m, err := store.InsertChannelMessage(ctx, ingest.InsertChannelMessageParams{ + ChannelID: 1, PacketHash: []byte{hash}, SenderName: sender, Content: "late", SentAt: at, Historical: true, + }) + if err != nil || m == nil { + t.Fatalf("backfill message %x: %v %v", hash, m, err) + } +} + +func rowsText(t *testing.T, ctx context.Context, pool *pgxpool.Pool, query string) string { + t.Helper() + var s *string + if err := pool.QueryRow(ctx, "SELECT string_agg(x::text, E'\\n' ORDER BY x::text) FROM ("+query+") x").Scan(&s); err != nil { + t.Fatalf("%v\n%s", err, query) + } + if s == nil { + return "" + } + return *s +} + +func rollAll(t *testing.T, ctx context.Context, r *RollupSession) { + t.Helper() + hours, err := r.MissingHours(ctx, 100) + if err != nil { + t.Fatal(err) + } + for _, h := range hours { + if outcome, _, err := r.RollHour(ctx, h); err != nil || outcome != RollComplete { + t.Fatalf("roll %s: outcome %d, err %v", h, outcome, err) + } + } +} + +func revision(t *testing.T, ctx context.Context, s *Store) int64 { + t.Helper() + rev, err := s.AnalyticsRevision(ctx) + if err != nil { + t.Fatal(err) + } + return rev +} + +func TestRollupHourPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if other, ok, err := store.BeginRollup(ctx); err != nil || ok { + if other != nil { + other.Close(ctx) + } + t.Fatalf("second BeginRollup: ok=%v err=%v, want lock held", ok, err) + } + + if err := r.RegisterHours(ctx, h0.Add(-10*time.Hour)); err != nil { + t.Fatal(err) + } + var first, last time.Time + var eligibleLast bool + if err := pool.QueryRow(ctx, `SELECT min(hour), max(hour), max(hour) = date_trunc('hour', now() - INTERVAL '95 minutes', 'UTC') + FROM analytics_rollup_hours WHERE status = 'missing'`).Scan(&first, &last, &eligibleLast); err != nil { + t.Fatal(err) + } + if !first.Equal(h0) || !eligibleLast { + t.Fatalf("registered hours %s..%s, want %s..now-95m", first, last, h0) + } + + rev0 := revision(t, ctx, store) + rollAll(t, ctx, r) + rev1 := revision(t, ctx, store) + if rev1 == rev0 { + t.Error("first roll did not bump the revision") + } + + for name, q := range rollupOracles { + if got, want := rowsText(t, ctx, pool, q[0]), rowsText(t, ctx, pool, q[1]); got != want { + t.Errorf("%s differs from raw:\nrollup:\n%s\nraw:\n%s", name, got, want) + } + } + for name, q := range rollupSetOracles { + for _, set := range [][]string{{"YVR"}, {"YYZ"}, {"YVR", "YYZ"}, {"YOW"}, {"YOW", "YVR", "YYZ"}} { + var got, want int64 + if err := pool.QueryRow(ctx, q[0], set).Scan(&got); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, q[1], set).Scan(&want); err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("%s for %v: rollup %d, raw %d", name, set, got, want) + } + } + } + + // Empty hour: complete, no family rows. + var emptyStatus string + var emptyRows int + if err := pool.QueryRow(ctx, `SELECT status, (SELECT count(*) FROM analytics_hourly_iata_observations WHERE hour = $1) + FROM analytics_rollup_hours WHERE hour = $1`, h0.Add(2*time.Hour)).Scan(&emptyStatus, &emptyRows); err != nil { + t.Fatal(err) + } + if emptyStatus != "complete" || emptyRows != 0 { + t.Errorf("empty hour status %q rows %d, want complete, 0", emptyStatus, emptyRows) + } + + // Re-rolling unchanged data is a no-op for readers. + if outcome, changed, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete || changed { + t.Fatalf("idempotent re-roll: outcome %d changed %v err %v", outcome, changed, err) + } + if revision(t, ctx, store) != rev1 { + t.Error("idempotent re-roll bumped the revision") + } + + // A backfilled message dirties every hour its packet was heard in; re-rolling picks it up. + backfillMessage(t, ctx, store, 0x05, "bob", h0) + dirty, err := r.DirtyHours(ctx, 10) + if err != nil || len(dirty) != 2 || !dirty[0].Equal(h0) || !dirty[1].Equal(h0.Add(time.Hour)) { + t.Fatalf("dirty hours %v, %v; want h0 and h0+1h", dirty, err) + } + for _, h := range dirty { + if outcome, changed, err := r.RollHour(ctx, h); err != nil || outcome != RollComplete || !changed { + t.Fatalf("dirty re-roll %s: outcome %d changed %v err %v", h, outcome, changed, err) + } + } + if revision(t, ctx, store) <= rev1 { + t.Error("changed re-roll did not bump the revision") + } + if dirty, err := r.DirtyHours(ctx, 10); err != nil || len(dirty) != 0 { + t.Errorf("dirty hours after re-roll %v, %v; want none", dirty, err) + } + var bob int64 + if err := pool.QueryRow(ctx, `SELECT COALESCE(sum(messages), 0) FROM analytics_hourly_talker_sets WHERE sender_name = 'bob'`).Scan(&bob); err != nil || bob != 2 { + t.Errorf("bob messages across the straddled hours = %d, %v; want 2", bob, err) + } +} + +func TestRollupPartialHoursPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0.Add(-10*time.Hour)); err != nil { + t.Fatal(err) + } + + // Rolling h0+1h first leaves h0 missing when cleanup reaches it. + if outcome, _, err := r.RollHour(ctx, h0.Add(time.Hour)); err != nil || outcome != RollComplete { + t.Fatalf("roll h0+1h: %d %v", outcome, err) + } + if _, err := pool.Exec(ctx, "UPDATE analytics_raw_state SET raw_deleted_before = $1", h0.Add(20*time.Minute)); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollPartial { + t.Fatalf("roll h0 after raw deletion: outcome %d err %v, want partial", outcome, err) + } + if err := r.MarkPartialHours(ctx); err != nil { + t.Fatal(err) + } + var statuses string + if err := pool.QueryRow(ctx, `SELECT string_agg(status, ',' ORDER BY hour) FROM analytics_rollup_hours WHERE hour IN ($1, $2)`, + h0, h0.Add(time.Hour)).Scan(&statuses); err != nil { + t.Fatal(err) + } + if statuses != "partial,complete" { + t.Errorf("statuses %q, want partial,complete", statuses) + } + + // Only complete hours with intact raw rows are re-rolled; h0's entry is dropped. + backfillMessage(t, ctx, store, 0x01, "carol", h0) + backfillMessage(t, ctx, store, 0x05, "bob", h0) + if dirty, err := r.DirtyHours(ctx, 10); err != nil || len(dirty) != 1 || !dirty[0].Equal(h0.Add(time.Hour)) { + t.Errorf("dirty hours %v, %v; want only h0+1h", dirty, err) + } + if missing, err := r.MissingHours(ctx, 100); err != nil || len(missing) == 0 || !missing[0].After(h0.Add(time.Hour)) { + t.Errorf("missing hours %v, %v; want only hours after h0+1h", missing, err) + } +} + +// A roll that dies mid-transaction leaves the hour missing for the next pass. +func TestRollupInterruptedHourPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + // Breaking a family table makes the roll fail after earlier families were written. + if _, err := pool.Exec(ctx, "ALTER TABLE analytics_hourly_scope_sets ADD CONSTRAINT boom CHECK (packets < 0) NOT VALID"); err != nil { + t.Fatal(err) + } + if _, _, err := r.RollHour(ctx, h0); err == nil { + t.Fatal("roll succeeded despite a failing family") + } + var status string + var rows int + if err := pool.QueryRow(ctx, `SELECT status, (SELECT count(*) FROM analytics_hourly_iata_observations WHERE hour = $1) + FROM analytics_rollup_hours WHERE hour = $1`, h0).Scan(&status, &rows); err != nil { + t.Fatal(err) + } + if status != "missing" || rows != 0 { + t.Errorf("after failed roll: status %q, %d family rows; want missing, 0", status, rows) + } + if _, err := pool.Exec(ctx, "ALTER TABLE analytics_hourly_scope_sets DROP CONSTRAINT boom"); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete { + t.Errorf("retry: outcome %d err %v", outcome, err) + } +} + +// A message backfilled after a first roll's snapshot (but before it commits) is not lost: +// its dirty entry is invisible to that roll, survives it, and triggers a re-roll. +func TestRollupBackfillDuringFirstRollPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + + roll, err := pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + if err != nil { + t.Fatal(err) + } + defer roll.Rollback(ctx) + if _, err := roll.Exec(ctx, "SELECT count(*) FROM channel_messages"); err != nil { // fixes the snapshot + t.Fatal(err) + } + backfillMessage(t, ctx, store, 0x05, "bob", h0) // commits on another connection + if outcome, _, err := rollHour(ctx, roll, h0); err != nil || outcome != RollComplete { + t.Fatalf("first roll: %d %v", outcome, err) + } + if err := roll.Commit(ctx); err != nil { + t.Fatal(err) + } + talkers := func() int64 { + t.Helper() + var n int64 + if err := pool.QueryRow(ctx, "SELECT COALESCE(sum(messages), 0) FROM analytics_hourly_talker_sets WHERE hour = $1 AND sender_name = 'bob'", h0).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + if talkers() != 0 { + t.Fatal("first roll saw the message; the race was not reproduced") + } + dirty, err := r.DirtyHours(ctx, 10) + if err != nil || len(dirty) == 0 || !dirty[0].Equal(h0) { + t.Fatalf("dirty hours %v, %v; want h0 still queued", dirty, err) + } + if outcome, _, err := r.RollHour(ctx, h0); err != nil || outcome != RollComplete { + t.Fatalf("re-roll: %d %v", outcome, err) + } + if talkers() != 1 { + t.Error("re-roll did not pick up the backfilled message") + } +} + +// With no observations there is nothing to roll, so no empty "complete" hours. +func TestRollupRegisterEmptyDatabasePostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + r, ok, err := New(pool, 0, 0).BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, time.Now().Add(-7*24*time.Hour)); err != nil { + t.Fatal(err) + } + var n int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM analytics_rollup_hours").Scan(&n); err != nil || n != 0 { + t.Errorf("registered %d hours on an empty database, err %v; want 0", n, err) + } +} + +// An observation may be heard up to 30 minutes after its packet's last_heard_at, so an hour +// starting exactly raw_deleted_before + 30 min may have lost rows too. +func TestRollupPartialBoundaryPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + r, ok, err := New(pool, 0, 0).BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + defer r.Close(ctx) + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + h1 := h0.Add(time.Hour) + if _, err := pool.Exec(ctx, "UPDATE analytics_raw_state SET raw_deleted_before = $1", h1.Add(-30*time.Minute)); err != nil { + t.Fatal(err) + } + if outcome, _, err := r.RollHour(ctx, h1); err != nil || outcome != RollPartial { + t.Errorf("hour at the boundary: outcome %d err %v, want partial", outcome, err) + } + if outcome, _, err := r.RollHour(ctx, h1.Add(time.Hour)); err != nil || outcome != RollComplete { + t.Errorf("next hour: outcome %d err %v, want complete", outcome, err) + } +} diff --git a/db/rollup_schema_integration_test.go b/db/rollup_schema_integration_test.go new file mode 100644 index 00000000..169e07fe --- /dev/null +++ b/db/rollup_schema_integration_test.go @@ -0,0 +1,117 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" +) + +func TestRollupSchemaPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(ctx) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + applyBaseline(t, ctx, tx) + + // Each failing insert runs in a savepoint so the tx stays usable. + bad := map[string]string{ + "duplicate": "'{YVR,YVR}'", + "unsorted": "'{YYZ,YVR}'", + "empty": "'{}'", + "null member": "'{YVR,NULL}'", + } + inserts := map[string]string{ + "packet_sets": "INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) VALUES (now(), %s, 1)", + "scope_sets": "INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) VALUES (now(), %s, 1, 1)", + "advert_sets": "INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) VALUES (now(), '\\x01', %s, 1, 1, 0)", + "talker_sets": "INSERT INTO analytics_hourly_talker_sets (hour, sender_name, iatas, messages, last_sent) VALUES (now(), 'a', %s, 1, now())", + } + for tname, tmpl := range inserts { + for name, lit := range bad { + sql := fmt.Sprintf(tmpl, lit) + if _, err := tx.Exec(ctx, "SAVEPOINT s"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, sql); err == nil { + t.Errorf("%s %s: insert succeeded, want CHECK violation", tname, name) + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT s"); err != nil { + t.Fatal(err) + } + } + if _, err := tx.Exec(ctx, fmt.Sprintf(tmpl, "'{YVR,YYZ}'")); err != nil { + t.Errorf("%s canonical set: %v", tname, err) + } + } + + for _, tbl := range []string{"analytics_state", "analytics_raw_state"} { + var n int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM "+tbl).Scan(&n); err != nil || n != 1 { + t.Errorf("%s rows = %d, err %v; want 1", tbl, n, err) + } + if _, err := tx.Exec(ctx, "SAVEPOINT s"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, "INSERT INTO "+tbl+" (singleton) VALUES (false)"); err == nil { + t.Errorf("%s accepted a second row", tbl) + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT s"); err != nil { + t.Fatal(err) + } + } + var rev int64 + if err := tx.QueryRow(ctx, "SELECT revision FROM analytics_state").Scan(&rev); err != nil || rev != 0 { + t.Errorf("revision = %d, err %v; want 0", rev, err) + } + var rawDeleted *time.Time + if err := tx.QueryRow(ctx, "SELECT raw_deleted_before FROM analytics_raw_state").Scan(&rawDeleted); err != nil || rawDeleted != nil { + t.Errorf("raw_deleted_before = %v, err %v; want NULL", rawDeleted, err) + } + + var heard, brin, obsID, lastIATA *string + if err := tx.QueryRow(ctx, `SELECT to_regclass('idx_observations_heard')::text, to_regclass('idx_observations_heard_brin')::text, + to_regclass('idx_observations_observer_id')::text, to_regclass('idx_observers_last_iata')::text`).Scan(&heard, &brin, &obsID, &lastIATA); err != nil { + t.Fatal(err) + } + if heard == nil || obsID == nil || lastIATA == nil { + t.Errorf("missing index: heard=%v observer_id=%v last_iata=%v", heard, obsID, lastIATA) + } + if brin != nil { + t.Errorf("idx_observations_heard_brin still exists") + } + + var nullable string + if err := tx.QueryRow(ctx, `SELECT is_nullable FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = 'channels' AND column_name = 'message_count'`).Scan(&nullable); err != nil || nullable != "NO" { + t.Errorf("channels.message_count nullable = %q, err %v; want NO", nullable, err) + } + for _, c := range [][2]string{{"packets", "observation_count"}, {"observers", "last_iata"}} { + var n int + if err := tx.QueryRow(ctx, `SELECT count(*) FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = $1 AND column_name = $2`, c[0], c[1]).Scan(&n); err != nil || n != 1 { + t.Errorf("column %s.%s missing (err %v)", c[0], c[1], err) + } + } + if _, err := tx.Exec(ctx, "INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at) VALUES ('\\x01', now(), now())"); err != nil { + t.Errorf("trace_tags: %v", err) + } +} diff --git a/db/scope_stats_integration_test.go b/db/scope_stats_integration_test.go index 94e1845e..207e09d7 100644 --- a/db/scope_stats_integration_test.go +++ b/db/scope_stats_integration_test.go @@ -4,52 +4,42 @@ package db import ( - "context" - "os" "reflect" "testing" "time" - sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/jackc/pgx/v5" ) -// Run on a migrated private database. All fixture tables and writes are rolled back. +// Packets come from the rolled IATA sets (each packet once per hour, however many requested +// IATAs heard it); observers are current memberships filtered by their latest IATA. func TestScopeStatsPostgres(t *testing.T) { - dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") - if dsn == "" { - t.Skip("set BEACON_TEST_POSTGRES_DSN for PostgreSQL regression") - } - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - conn, err := pgx.Connect(ctx, dsn) - if err != nil { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { t.Fatal(err) } - defer conn.Close(context.Background()) - tx, err := conn.Begin(ctx) - if err != nil { + var h time.Time + if err := pool.QueryRow(ctx, "SELECT date_trunc('hour', now(), 'UTC') - INTERVAL '4 hours'").Scan(&h); err != nil { t.Fatal(err) } - defer tx.Rollback(context.Background()) - for _, table := range []string{"transport_scopes", "packets", "packet_observations", "observer_scopes", "nodes", "node_iatas"} { - if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { - t.Fatal(err) - } - } - _, err = tx.Exec(ctx, ` + if _, err := pool.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YVR'), ('YYJ'), ('YYZ'); INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) SELECT i,name,decode(repeat('00',16),'hex'),decode(lpad(to_hex(i),16,'0'),'hex') FROM (VALUES (1,'#a'),(2,'#b'),(3,'#unused')) v(i,name); +INSERT INTO observers (id,public_key,last_iata) +SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),2,'0'),'hex'),iata +FROM (VALUES (1,'YVR'),(2,'YYJ'),(3,'YVR'),(4,'YVR'),(5,'YYZ')) v(i,iata); INSERT INTO packets (packet_hash,scope_id,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) -SELECT decode(lpad(to_hex(i),2,'0'),'hex'),scope_id,4,0,1,'\x00','\x00',NOW(),NOW() +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),scope_id,4,0,1,'\x00','\x00',$1::timestamptz,$1::timestamptz FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,2),(8,NULL)) v(i,scope_id); INSERT INTO observer_scopes (observer_id,scope_id) SELECT md5(i::text)::uuid,scope_id FROM (VALUES (1,1),(2,1),(4,1),(5,1),(1,2),(2,2),(3,2)) v(i,scope_id); +-- packet 4 is never heard, so it never reaches the rollup INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) SELECT id,decode(lpad(to_hex(packet),2,'0'),'hex'),md5(observer::text)::uuid,iata, - '2026-01-01'::timestamptz+id*interval '1 second',0,1,0 + $1::timestamptz+id*interval '1 second',0,1,0 FROM (VALUES (1,1,1,'YVR'),(2,1,4,'YVR'),(3,1,2,'YYJ'),(4,2,2,'YVR'), (5,3,1,'YYZ'),(6,5,3,'YVR'),(7,6,1,'YYJ'),(8,7,2,'YYZ'),(9,8,1,'YVR')) v(id,packet,observer,iata); INSERT INTO nodes (id,public_key,node_type,default_scope_id) @@ -57,27 +47,38 @@ SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),2,scope_id FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,NULL)) v(i,scope_id); INSERT INTO node_iatas (node_id,iata) SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YYZ'),(5,'YYJ'),(6,'YVR'),(6,'YYJ'),(7,'YVR')) v(i,iata); -`) - if err != nil { +`, pgx.QueryExecModeSimpleProtocol, h); err != nil { + t.Fatal(err) + } + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + if err := r.RegisterHours(ctx, h); err != nil { t.Fatal(err) } - store := &Store{q: sqlc.New(tx)} + rollAll(t, ctx, r) + r.Close(ctx) + for _, tc := range []struct { name string iatas []string + since time.Time a, b [3]int64 // packets, observers, nodes }{ - {"global", nil, [3]int64{4, 4, 4}, [3]int64{3, 3, 2}}, - {"empty filter", []string{}, [3]int64{4, 4, 4}, [3]int64{3, 3, 2}}, - {"YVR", []string{"YVR"}, [3]int64{2, 3, 2}, [3]int64{1, 1, 1}}, - {"YYJ", []string{"YYJ"}, [3]int64{1, 1, 1}, [3]int64{1, 1, 2}}, - {"YYZ", []string{"YYZ"}, [3]int64{1, 1, 1}, [3]int64{1, 1, 0}}, - {"overlapping IATAs", []string{"YVR", "YYJ"}, [3]int64{2, 3, 2}, [3]int64{2, 2, 2}}, - {"duplicate IATAs", []string{"YYJ", "YVR", "YYJ"}, [3]int64{2, 3, 2}, [3]int64{2, 2, 2}}, - {"unknown IATA", []string{"ZZZ"}, [3]int64{}, [3]int64{}}, + {"global", nil, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}}, + {"empty filter", []string{}, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}}, + {"YVR", []string{"YVR"}, time.Time{}, [3]int64{2, 2, 2}, [3]int64{1, 2, 1}}, + {"YYJ", []string{"YYJ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 1, 2}}, + {"YYZ", []string{"YYZ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 0, 0}}, + {"overlapping IATAs", []string{"YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}}, + {"duplicate IATAs", []string{"YYJ", "YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}}, + {"unknown IATA", []string{"ZZZ"}, time.Time{}, [3]int64{}, [3]int64{}}, + {"window after the traffic", nil, h.Add(time.Hour), [3]int64{0, 4, 4}, [3]int64{0, 3, 2}}, } { t.Run(tc.name, func(t *testing.T) { - rows, err := store.GetScopeStats(ctx, tc.iatas) + rows, err := store.GetScopeStats(ctx, tc.iatas, tc.since) if err != nil { t.Fatal(err) } @@ -91,10 +92,20 @@ SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YY } }) } - if _, err := tx.Exec(ctx, "DELETE FROM transport_scopes"); err != nil { + + detail, err := store.GetScopeByName(ctx, "#a") + if err != nil { + t.Fatal(err) + } + want := &api.ScopeDetail{Name: "#a", PacketCount: 3, ObserverCount: 4, NodeCount: 4, IATACount: 3, IATAs: []string{"YVR", "YYJ", "YYZ"}} + if !reflect.DeepEqual(detail, want) { + t.Errorf("scope detail %+v, want %+v", detail, want) + } + + if _, err := pool.Exec(ctx, "TRUNCATE transport_scopes CASCADE"); err != nil { t.Fatal(err) } - rows, err := store.GetScopeStats(ctx, nil) + rows, err := store.GetScopeStats(ctx, nil, time.Time{}) if err != nil || rows == nil || len(rows) != 0 { t.Fatalf("empty roster = %+v, %v", rows, err) } diff --git a/db/signal.go b/db/signal.go index 9a7b1a66..0edd4cf1 100644 --- a/db/signal.go +++ b/db/signal.go @@ -45,10 +45,6 @@ func (s *Store) GetSignalStats(ctx context.Context, since, until time.Time, iata return stats, nil } -func (s *Store) RefreshSignalStats(ctx context.Context) error { - return s.q.RefreshSignalStats(ctx) -} - func signalAverage(value float64, count int64) *float64 { if count == 0 { return nil diff --git a/db/signal_integration_test.go b/db/signal_integration_test.go index 73bc8252..06555032 100644 --- a/db/signal_integration_test.go +++ b/db/signal_integration_test.go @@ -64,9 +64,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi if _, err := tx.Exec(ctx, "UPDATE packet_observations SET heard_at=to_timestamp(extract(epoch FROM $1::timestamptz)+extract(epoch FROM heard_at)-extract(epoch FROM '2026-01-01 00:00+00'::timestamptz))", since); err != nil { t.Fatal(err) } - if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW mv_signal_stats_hourly"); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) store := &Store{q: sqlc.New(tx)} until := since.Add(4 * time.Hour) for _, tc := range []struct { @@ -138,7 +136,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi } w := httptest.NewRecorder() request := httptest.NewRequest("GET", fmt.Sprintf("/signal?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx) - handlers.StatsRouter(store, nil).ServeHTTP(w, request) + handlers.StatsRouter(store, handlers.StatsOptions{}).ServeHTTP(w, request) var response api.SignalStats if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || w.Code != 200 || response.Receptions != 13 || response.SNR.Samples != 5 { t.Fatalf("PostgreSQL HTTP response: status=%d body=%s error=%v", w.Code, w.Body.String(), err) @@ -150,9 +148,7 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,snr,rssi if err != nil || stale.Receptions != 15 { t.Fatalf("snapshot lost: %+v %v", stale, err) } - if err := store.RefreshSignalStats(ctx); err != nil { - t.Fatal(err) - } + rollTxHours(t, ctx, tx) got, err := store.GetSignalStats(ctx, since, until, nil) if err != nil || got.Receptions != 0 || got.Hourly == nil || len(got.Hourly) != 0 || got.SNR.Average != nil { t.Fatalf("empty: %+v %v", got, err) diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 0626fdae..2b436d9e 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -245,6 +245,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldPackets(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldPackets", reflect.TypeOf((*MockQuerier)(nil).DeleteOldPackets), ctx, arg) } +// DeleteOldRollups mocks base method. +func (m *MockQuerier) DeleteOldRollups(ctx context.Context, cutoff pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldRollups", ctx, cutoff) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteOldRollups indicates an expected call of DeleteOldRollups. +func (mr *MockQuerierMockRecorder) DeleteOldRollups(ctx, cutoff any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldRollups", reflect.TypeOf((*MockQuerier)(nil).DeleteOldRollups), ctx, cutoff) +} + // DeleteOldRoutes mocks base method. func (m *MockQuerier) DeleteOldRoutes(ctx context.Context, arg db.DeleteOldRoutesParams) (int64, error) { m.ctrl.T.Helper() @@ -288,6 +302,20 @@ func (mr *MockQuerierMockRecorder) DeleteOldTraceIATAs(ctx, lastHeard any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceIATAs", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceIATAs), ctx, lastHeard) } +// DeleteOldTraceTags mocks base method. +func (m *MockQuerier) DeleteOldTraceTags(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteOldTraceTags", ctx, lastHeardAt) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteOldTraceTags indicates an expected call of DeleteOldTraceTags. +func (mr *MockQuerierMockRecorder) DeleteOldTraceTags(ctx, lastHeardAt any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldTraceTags", reflect.TypeOf((*MockQuerier)(nil).DeleteOldTraceTags), ctx, lastHeardAt) +} + // DeleteRegionIATAsNotIn mocks base method. func (m *MockQuerier) DeleteRegionIATAsNotIn(ctx context.Context, arg db.DeleteRegionIATAsNotInParams) error { m.ctrl.T.Helper() @@ -302,6 +330,63 @@ func (mr *MockQuerierMockRecorder) DeleteRegionIATAsNotIn(ctx, arg any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRegionIATAsNotIn", reflect.TypeOf((*MockQuerier)(nil).DeleteRegionIATAsNotIn), ctx, arg) } +// DeleteRollupHour mocks base method. +func (m *MockQuerier) DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRollupHour", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteRollupHour indicates an expected call of DeleteRollupHour. +func (mr *MockQuerierMockRecorder) DeleteRollupHour(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRollupHour", reflect.TypeOf((*MockQuerier)(nil).DeleteRollupHour), ctx, hour) +} + +// DeleteStaleDirtyHours mocks base method. +func (m *MockQuerier) DeleteStaleDirtyHours(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteStaleDirtyHours", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteStaleDirtyHours indicates an expected call of DeleteStaleDirtyHours. +func (mr *MockQuerierMockRecorder) DeleteStaleDirtyHours(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteStaleDirtyHours", reflect.TypeOf((*MockQuerier)(nil).DeleteStaleDirtyHours), ctx) +} + +// FillRollupObs mocks base method. +func (m *MockQuerier) FillRollupObs(ctx context.Context, dollar_1 pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FillRollupObs", ctx, dollar_1) + ret0, _ := ret[0].(error) + return ret0 +} + +// FillRollupObs indicates an expected call of FillRollupObs. +func (mr *MockQuerierMockRecorder) FillRollupObs(ctx, dollar_1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FillRollupObs", reflect.TypeOf((*MockQuerier)(nil).FillRollupObs), ctx, dollar_1) +} + +// FinishRollupHour mocks base method. +func (m *MockQuerier) FinishRollupHour(ctx context.Context, arg db.FinishRollupHourParams) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinishRollupHour", ctx, arg) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FinishRollupHour indicates an expected call of FinishRollupHour. +func (mr *MockQuerierMockRecorder) FinishRollupHour(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinishRollupHour", reflect.TypeOf((*MockQuerier)(nil).FinishRollupHour), ctx, arg) +} + // GetAccount mocks base method. func (m *MockQuerier) GetAccount(ctx context.Context, id uuid.UUID) (db.Account, error) { m.ctrl.T.Helper() @@ -317,6 +402,21 @@ func (mr *MockQuerierMockRecorder) GetAccount(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccount", reflect.TypeOf((*MockQuerier)(nil).GetAccount), ctx, id) } +// GetAnalyticsRevision mocks base method. +func (m *MockQuerier) GetAnalyticsRevision(ctx context.Context) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAnalyticsRevision", ctx) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAnalyticsRevision indicates an expected call of GetAnalyticsRevision. +func (mr *MockQuerierMockRecorder) GetAnalyticsRevision(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAnalyticsRevision", reflect.TypeOf((*MockQuerier)(nil).GetAnalyticsRevision), ctx) +} + // GetChannelByID mocks base method. func (m *MockQuerier) GetChannelByID(ctx context.Context, id int32) (db.Channel, error) { m.ctrl.T.Helper() @@ -347,11 +447,26 @@ func (mr *MockQuerierMockRecorder) GetCrossIATANeighbors(ctx, arg any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCrossIATANeighbors", reflect.TypeOf((*MockQuerier)(nil).GetCrossIATANeighbors), ctx, arg) } +// GetEarliestCompleteRollupHour mocks base method. +func (m *MockQuerier) GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetEarliestCompleteRollupHour", ctx) + ret0, _ := ret[0].(pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetEarliestCompleteRollupHour indicates an expected call of GetEarliestCompleteRollupHour. +func (mr *MockQuerierMockRecorder) GetEarliestCompleteRollupHour(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEarliestCompleteRollupHour", reflect.TypeOf((*MockQuerier)(nil).GetEarliestCompleteRollupHour), ctx) +} + // GetHourlyStats mocks base method. -func (m *MockQuerier) GetHourlyStats(ctx context.Context, arg db.GetHourlyStatsParams) ([]db.MvHourlyIataStat, error) { +func (m *MockQuerier) GetHourlyStats(ctx context.Context, arg db.GetHourlyStatsParams) ([]db.GetHourlyStatsRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetHourlyStats", ctx, arg) - ret0, _ := ret[0].([]db.MvHourlyIataStat) + ret0, _ := ret[0].([]db.GetHourlyStatsRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -618,18 +733,18 @@ func (mr *MockQuerierMockRecorder) GetObserverComparison(ctx, arg any) *gomock.C } // GetObserverLastIATA mocks base method. -func (m *MockQuerier) GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) { +func (m *MockQuerier) GetObserverLastIATA(ctx context.Context, id uuid.UUID) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetObserverLastIATA", ctx, observerID) + ret := m.ctrl.Call(m, "GetObserverLastIATA", ctx, id) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetObserverLastIATA indicates an expected call of GetObserverLastIATA. -func (mr *MockQuerierMockRecorder) GetObserverLastIATA(ctx, observerID any) *gomock.Call { +func (mr *MockQuerierMockRecorder) GetObserverLastIATA(ctx, id any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverLastIATA", reflect.TypeOf((*MockQuerier)(nil).GetObserverLastIATA), ctx, observerID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverLastIATA", reflect.TypeOf((*MockQuerier)(nil).GetObserverLastIATA), ctx, id) } // GetObserverRadio mocks base method. @@ -692,21 +807,6 @@ func (mr *MockQuerierMockRecorder) GetPacketByHash(ctx, packetHash any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPacketByHash", reflect.TypeOf((*MockQuerier)(nil).GetPacketByHash), ctx, packetHash) } -// GetPacketObservationCount mocks base method. -func (m *MockQuerier) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPacketObservationCount", ctx, packetHash) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetPacketObservationCount indicates an expected call of GetPacketObservationCount. -func (mr *MockQuerierMockRecorder) GetPacketObservationCount(ctx, packetHash any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPacketObservationCount", reflect.TypeOf((*MockQuerier)(nil).GetPacketObservationCount), ctx, packetHash) -} - // GetPacketsByTraceTag mocks base method. func (m *MockQuerier) GetPacketsByTraceTag(ctx context.Context, decode string) ([]db.GetPacketsByTraceTagRow, error) { m.ctrl.T.Helper() @@ -858,18 +958,18 @@ func (mr *MockQuerierMockRecorder) GetScopeNames(ctx any) *gomock.Call { } // GetScopeStats mocks base method. -func (m *MockQuerier) GetScopeStats(ctx context.Context, iatas []string) ([]db.GetScopeStatsRow, error) { +func (m *MockQuerier) GetScopeStats(ctx context.Context, arg db.GetScopeStatsParams) ([]db.GetScopeStatsRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetScopeStats", ctx, iatas) + ret := m.ctrl.Call(m, "GetScopeStats", ctx, arg) ret0, _ := ret[0].([]db.GetScopeStatsRow) ret1, _ := ret[1].(error) return ret0, ret1 } // GetScopeStats indicates an expected call of GetScopeStats. -func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, iatas any) *gomock.Call { +func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, iatas) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, arg) } // GetSignalStats mocks base method. @@ -917,34 +1017,34 @@ func (mr *MockQuerierMockRecorder) GetStatsNodeTypes(ctx, dollar_1 any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsNodeTypes", reflect.TypeOf((*MockQuerier)(nil).GetStatsNodeTypes), ctx, dollar_1) } -// GetStatsOverview mocks base method. -func (m *MockQuerier) GetStatsOverview(ctx context.Context, dollar_1 []string) (db.GetStatsOverviewRow, error) { +// GetStatsPayloadBreakdown mocks base method. +func (m *MockQuerier) GetStatsPayloadBreakdown(ctx context.Context, arg db.GetStatsPayloadBreakdownParams) ([]db.GetStatsPayloadBreakdownRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetStatsOverview", ctx, dollar_1) - ret0, _ := ret[0].(db.GetStatsOverviewRow) + ret := m.ctrl.Call(m, "GetStatsPayloadBreakdown", ctx, arg) + ret0, _ := ret[0].([]db.GetStatsPayloadBreakdownRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetStatsOverview indicates an expected call of GetStatsOverview. -func (mr *MockQuerierMockRecorder) GetStatsOverview(ctx, dollar_1 any) *gomock.Call { +// GetStatsPayloadBreakdown indicates an expected call of GetStatsPayloadBreakdown. +func (mr *MockQuerierMockRecorder) GetStatsPayloadBreakdown(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsOverview", reflect.TypeOf((*MockQuerier)(nil).GetStatsOverview), ctx, dollar_1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).GetStatsPayloadBreakdown), ctx, arg) } -// GetStatsPayloadBreakdown mocks base method. -func (m *MockQuerier) GetStatsPayloadBreakdown(ctx context.Context, arg db.GetStatsPayloadBreakdownParams) ([]db.GetStatsPayloadBreakdownRow, error) { +// GetStatsSeries mocks base method. +func (m *MockQuerier) GetStatsSeries(ctx context.Context, arg db.GetStatsSeriesParams) ([]db.GetStatsSeriesRow, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetStatsPayloadBreakdown", ctx, arg) - ret0, _ := ret[0].([]db.GetStatsPayloadBreakdownRow) + ret := m.ctrl.Call(m, "GetStatsSeries", ctx, arg) + ret0, _ := ret[0].([]db.GetStatsSeriesRow) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetStatsPayloadBreakdown indicates an expected call of GetStatsPayloadBreakdown. -func (mr *MockQuerierMockRecorder) GetStatsPayloadBreakdown(ctx, arg any) *gomock.Call { +// GetStatsSeries indicates an expected call of GetStatsSeries. +func (mr *MockQuerierMockRecorder) GetStatsSeries(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).GetStatsPayloadBreakdown), ctx, arg) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatsSeries", reflect.TypeOf((*MockQuerier)(nil).GetStatsSeries), ctx, arg) } // GetStatsTopAdvertisers mocks base method. @@ -993,10 +1093,10 @@ func (mr *MockQuerierMockRecorder) GetStatsTopTalkers(ctx, arg any) *gomock.Call } // GetTopNodes mocks base method. -func (m *MockQuerier) GetTopNodes(ctx context.Context, arg db.GetTopNodesParams) ([]db.MvTopNodesByIatum, error) { +func (m *MockQuerier) GetTopNodes(ctx context.Context, arg db.GetTopNodesParams) ([]db.GetTopNodesRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetTopNodes", ctx, arg) - ret0, _ := ret[0].([]db.MvTopNodesByIatum) + ret0, _ := ret[0].([]db.GetTopNodesRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -1053,10 +1153,10 @@ func (mr *MockQuerierMockRecorder) InsertChannelMessage(ctx, arg any) *gomock.Ca } // InsertObservation mocks base method. -func (m *MockQuerier) InsertObservation(ctx context.Context, arg db.InsertObservationParams) (db.PacketObservation, error) { +func (m *MockQuerier) InsertObservation(ctx context.Context, arg db.InsertObservationParams) (db.InsertObservationRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InsertObservation", ctx, arg) - ret0, _ := ret[0].(db.PacketObservation) + ret0, _ := ret[0].(db.InsertObservationRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -1186,6 +1286,21 @@ func (mr *MockQuerierMockRecorder) ListChannelsAfter(ctx, arg any) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListChannelsAfter", reflect.TypeOf((*MockQuerier)(nil).ListChannelsAfter), ctx, arg) } +// ListDirtyRollupHours mocks base method. +func (m *MockQuerier) ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDirtyRollupHours", ctx, limit) + ret0, _ := ret[0].([]pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDirtyRollupHours indicates an expected call of ListDirtyRollupHours. +func (mr *MockQuerierMockRecorder) ListDirtyRollupHours(ctx, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDirtyRollupHours", reflect.TypeOf((*MockQuerier)(nil).ListDirtyRollupHours), ctx, limit) +} + // ListIATAs mocks base method. func (m *MockQuerier) ListIATAs(ctx context.Context) ([]db.IataCode, error) { m.ctrl.T.Helper() @@ -1231,6 +1346,21 @@ func (mr *MockQuerierMockRecorder) ListMessagesAfterID(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMessagesAfterID", reflect.TypeOf((*MockQuerier)(nil).ListMessagesAfterID), ctx, arg) } +// ListMissingRollupHours mocks base method. +func (m *MockQuerier) ListMissingRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListMissingRollupHours", ctx, limit) + ret0, _ := ret[0].([]pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListMissingRollupHours indicates an expected call of ListMissingRollupHours. +func (mr *MockQuerierMockRecorder) ListMissingRollupHours(ctx, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListMissingRollupHours", reflect.TypeOf((*MockQuerier)(nil).ListMissingRollupHours), ctx, limit) +} + // ListNodeObservations mocks base method. func (m *MockQuerier) ListNodeObservations(ctx context.Context, arg db.ListNodeObservationsParams) ([]db.ListNodeObservationsRow, error) { m.ctrl.T.Helper() @@ -1486,6 +1616,50 @@ func (mr *MockQuerierMockRecorder) ListZoneLists(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListZoneLists", reflect.TypeOf((*MockQuerier)(nil).ListZoneLists), ctx) } +// LockRollupHour mocks base method. +func (m *MockQuerier) LockRollupHour(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LockRollupHour", ctx, hour) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LockRollupHour indicates an expected call of LockRollupHour. +func (mr *MockQuerierMockRecorder) LockRollupHour(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockRollupHour", reflect.TypeOf((*MockQuerier)(nil).LockRollupHour), ctx, hour) +} + +// MarkPartialRollupHours mocks base method. +func (m *MockQuerier) MarkPartialRollupHours(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkPartialRollupHours", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkPartialRollupHours indicates an expected call of MarkPartialRollupHours. +func (mr *MockQuerierMockRecorder) MarkPartialRollupHours(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkPartialRollupHours", reflect.TypeOf((*MockQuerier)(nil).MarkPartialRollupHours), ctx) +} + +// OldestMissingRollupHour mocks base method. +func (m *MockQuerier) OldestMissingRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "OldestMissingRollupHour", ctx) + ret0, _ := ret[0].(pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// OldestMissingRollupHour indicates an expected call of OldestMissingRollupHour. +func (mr *MockQuerierMockRecorder) OldestMissingRollupHour(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OldestMissingRollupHour", reflect.TypeOf((*MockQuerier)(nil).OldestMissingRollupHour), ctx) +} + // PruneImportedRegions mocks base method. func (m *MockQuerier) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) { m.ctrl.T.Helper() @@ -1545,60 +1719,18 @@ func (mr *MockQuerierMockRecorder) ReconfirmRoutes(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReconfirmRoutes", reflect.TypeOf((*MockQuerier)(nil).ReconfirmRoutes), ctx, arg) } -// RefreshHourlyStats mocks base method. -func (m *MockQuerier) RefreshHourlyStats(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshHourlyStats", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshHourlyStats indicates an expected call of RefreshHourlyStats. -func (mr *MockQuerierMockRecorder) RefreshHourlyStats(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshHourlyStats", reflect.TypeOf((*MockQuerier)(nil).RefreshHourlyStats), ctx) -} - -// RefreshObserverActivity mocks base method. -func (m *MockQuerier) RefreshObserverActivity(ctx context.Context) error { +// RecordTrace mocks base method. +func (m *MockQuerier) RecordTrace(ctx context.Context, arg db.RecordTraceParams) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshObserverActivity", ctx) + ret := m.ctrl.Call(m, "RecordTrace", ctx, arg) ret0, _ := ret[0].(error) return ret0 } -// RefreshObserverActivity indicates an expected call of RefreshObserverActivity. -func (mr *MockQuerierMockRecorder) RefreshObserverActivity(ctx any) *gomock.Call { +// RecordTrace indicates an expected call of RecordTrace. +func (mr *MockQuerierMockRecorder) RecordTrace(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshObserverActivity", reflect.TypeOf((*MockQuerier)(nil).RefreshObserverActivity), ctx) -} - -// RefreshPathStats mocks base method. -func (m *MockQuerier) RefreshPathStats(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshPathStats", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshPathStats indicates an expected call of RefreshPathStats. -func (mr *MockQuerierMockRecorder) RefreshPathStats(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshPathStats", reflect.TypeOf((*MockQuerier)(nil).RefreshPathStats), ctx) -} - -// RefreshPayloadBreakdown mocks base method. -func (m *MockQuerier) RefreshPayloadBreakdown(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshPayloadBreakdown", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshPayloadBreakdown indicates an expected call of RefreshPayloadBreakdown. -func (mr *MockQuerierMockRecorder) RefreshPayloadBreakdown(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).RefreshPayloadBreakdown), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordTrace", reflect.TypeOf((*MockQuerier)(nil).RecordTrace), ctx, arg) } // RefreshRadioPresets mocks base method. @@ -1615,74 +1747,18 @@ func (mr *MockQuerierMockRecorder) RefreshRadioPresets(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshRadioPresets", reflect.TypeOf((*MockQuerier)(nil).RefreshRadioPresets), ctx) } -// RefreshSignalStats mocks base method. -func (m *MockQuerier) RefreshSignalStats(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshSignalStats", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshSignalStats indicates an expected call of RefreshSignalStats. -func (mr *MockQuerierMockRecorder) RefreshSignalStats(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshSignalStats", reflect.TypeOf((*MockQuerier)(nil).RefreshSignalStats), ctx) -} - -// RefreshTopAdvertisers mocks base method. -func (m *MockQuerier) RefreshTopAdvertisers(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopAdvertisers", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshTopAdvertisers indicates an expected call of RefreshTopAdvertisers. -func (mr *MockQuerierMockRecorder) RefreshTopAdvertisers(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopAdvertisers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopAdvertisers), ctx) -} - -// RefreshTopNodes mocks base method. -func (m *MockQuerier) RefreshTopNodes(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopNodes", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshTopNodes indicates an expected call of RefreshTopNodes. -func (mr *MockQuerierMockRecorder) RefreshTopNodes(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopNodes", reflect.TypeOf((*MockQuerier)(nil).RefreshTopNodes), ctx) -} - -// RefreshTopObservers mocks base method. -func (m *MockQuerier) RefreshTopObservers(ctx context.Context) error { +// RegisterRollupHours mocks base method. +func (m *MockQuerier) RegisterRollupHours(ctx context.Context, since pgtype.Timestamptz) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopObservers", ctx) + ret := m.ctrl.Call(m, "RegisterRollupHours", ctx, since) ret0, _ := ret[0].(error) return ret0 } -// RefreshTopObservers indicates an expected call of RefreshTopObservers. -func (mr *MockQuerierMockRecorder) RefreshTopObservers(ctx any) *gomock.Call { +// RegisterRollupHours indicates an expected call of RegisterRollupHours. +func (mr *MockQuerierMockRecorder) RegisterRollupHours(ctx, since any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopObservers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopObservers), ctx) -} - -// RefreshTopTalkers mocks base method. -func (m *MockQuerier) RefreshTopTalkers(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RefreshTopTalkers", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// RefreshTopTalkers indicates an expected call of RefreshTopTalkers. -func (mr *MockQuerierMockRecorder) RefreshTopTalkers(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshTopTalkers", reflect.TypeOf((*MockQuerier)(nil).RefreshTopTalkers), ctx) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterRollupHours", reflect.TypeOf((*MockQuerier)(nil).RegisterRollupHours), ctx, since) } // ResolveEndpointHashPairs mocks base method. @@ -1775,6 +1851,189 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } +// RollAdvertHearings mocks base method. +func (m *MockQuerier) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollAdvertHearings", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollAdvertHearings indicates an expected call of RollAdvertHearings. +func (mr *MockQuerierMockRecorder) RollAdvertHearings(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollAdvertHearings", reflect.TypeOf((*MockQuerier)(nil).RollAdvertHearings), ctx, hour) +} + +// RollAdvertSets mocks base method. +func (m *MockQuerier) RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollAdvertSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollAdvertSets indicates an expected call of RollAdvertSets. +func (mr *MockQuerierMockRecorder) RollAdvertSets(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollAdvertSets", reflect.TypeOf((*MockQuerier)(nil).RollAdvertSets), ctx, hour) +} + +// RollIATAObservations mocks base method. +func (m *MockQuerier) RollIATAObservations(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollIATAObservations", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollIATAObservations indicates an expected call of RollIATAObservations. +func (mr *MockQuerierMockRecorder) RollIATAObservations(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollIATAObservations", reflect.TypeOf((*MockQuerier)(nil).RollIATAObservations), ctx, hour) +} + +// RollObserverActivity mocks base method. +func (m *MockQuerier) RollObserverActivity(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollObserverActivity", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollObserverActivity indicates an expected call of RollObserverActivity. +func (mr *MockQuerierMockRecorder) RollObserverActivity(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollObserverActivity", reflect.TypeOf((*MockQuerier)(nil).RollObserverActivity), ctx, hour) +} + +// RollObserverIdentity mocks base method. +func (m *MockQuerier) RollObserverIdentity(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollObserverIdentity", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollObserverIdentity indicates an expected call of RollObserverIdentity. +func (mr *MockQuerierMockRecorder) RollObserverIdentity(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollObserverIdentity", reflect.TypeOf((*MockQuerier)(nil).RollObserverIdentity), ctx, hour) +} + +// RollPacketSets mocks base method. +func (m *MockQuerier) RollPacketSets(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollPacketSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollPacketSets indicates an expected call of RollPacketSets. +func (mr *MockQuerierMockRecorder) RollPacketSets(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPacketSets", reflect.TypeOf((*MockQuerier)(nil).RollPacketSets), ctx, hour) +} + +// RollPaths mocks base method. +func (m *MockQuerier) RollPaths(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollPaths", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollPaths indicates an expected call of RollPaths. +func (mr *MockQuerierMockRecorder) RollPaths(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPaths", reflect.TypeOf((*MockQuerier)(nil).RollPaths), ctx, hour) +} + +// RollPayloadBreakdown mocks base method. +func (m *MockQuerier) RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollPayloadBreakdown", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollPayloadBreakdown indicates an expected call of RollPayloadBreakdown. +func (mr *MockQuerierMockRecorder) RollPayloadBreakdown(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).RollPayloadBreakdown), ctx, hour) +} + +// RollScopeSets mocks base method. +func (m *MockQuerier) RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollScopeSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollScopeSets indicates an expected call of RollScopeSets. +func (mr *MockQuerierMockRecorder) RollScopeSets(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollScopeSets", reflect.TypeOf((*MockQuerier)(nil).RollScopeSets), ctx, hour) +} + +// RollSignal mocks base method. +func (m *MockQuerier) RollSignal(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollSignal", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollSignal indicates an expected call of RollSignal. +func (mr *MockQuerierMockRecorder) RollSignal(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollSignal", reflect.TypeOf((*MockQuerier)(nil).RollSignal), ctx, hour) +} + +// RollTalkerSets mocks base method. +func (m *MockQuerier) RollTalkerSets(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollTalkerSets", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollTalkerSets indicates an expected call of RollTalkerSets. +func (mr *MockQuerierMockRecorder) RollTalkerSets(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollTalkerSets", reflect.TypeOf((*MockQuerier)(nil).RollTalkerSets), ctx, hour) +} + +// RollupContentHash mocks base method. +func (m *MockQuerier) RollupContentHash(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollupContentHash", ctx, hour) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RollupContentHash indicates an expected call of RollupContentHash. +func (mr *MockQuerierMockRecorder) RollupContentHash(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollupContentHash", reflect.TypeOf((*MockQuerier)(nil).RollupContentHash), ctx, hour) +} + +// RollupUnlock mocks base method. +func (m *MockQuerier) RollupUnlock(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollupUnlock", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollupUnlock indicates an expected call of RollupUnlock. +func (mr *MockQuerierMockRecorder) RollupUnlock(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollupUnlock", reflect.TypeOf((*MockQuerier)(nil).RollupUnlock), ctx) +} + // SaveChannelCatalogue mocks base method. func (m *MockQuerier) SaveChannelCatalogue(ctx context.Context, arg db.SaveChannelCatalogueParams) error { m.ctrl.T.Helper() @@ -1902,6 +2161,20 @@ func (mr *MockQuerierMockRecorder) SetPacketDecrypted(ctx, packetHash any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPacketDecrypted", reflect.TypeOf((*MockQuerier)(nil).SetPacketDecrypted), ctx, packetHash) } +// SetRollupHourPartial mocks base method. +func (m *MockQuerier) SetRollupHourPartial(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetRollupHourPartial", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetRollupHourPartial indicates an expected call of SetRollupHourPartial. +func (mr *MockQuerierMockRecorder) SetRollupHourPartial(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetRollupHourPartial", reflect.TypeOf((*MockQuerier)(nil).SetRollupHourPartial), ctx, hour) +} + // TouchObserverBrokers mocks base method. func (m *MockQuerier) TouchObserverBrokers(ctx context.Context, arg db.TouchObserverBrokersParams) error { m.ctrl.T.Helper() @@ -1944,6 +2217,21 @@ func (mr *MockQuerierMockRecorder) TouchPackets(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TouchPackets", reflect.TypeOf((*MockQuerier)(nil).TouchPackets), ctx, arg) } +// TryRollupLock mocks base method. +func (m *MockQuerier) TryRollupLock(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TryRollupLock", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TryRollupLock indicates an expected call of TryRollupLock. +func (mr *MockQuerierMockRecorder) TryRollupLock(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TryRollupLock", reflect.TypeOf((*MockQuerier)(nil).TryRollupLock), ctx) +} + // UpdateObserverRegionScope mocks base method. func (m *MockQuerier) UpdateObserverRegionScope(ctx context.Context, arg db.UpdateObserverRegionScopeParams) error { m.ctrl.T.Helper() @@ -2146,18 +2434,18 @@ func (mr *MockQuerierMockRecorder) UpsertNodeShortID(ctx, arg any) *gomock.Call } // UpsertObserver mocks base method. -func (m *MockQuerier) UpsertObserver(ctx context.Context, publicKey []byte) (db.Observer, error) { +func (m *MockQuerier) UpsertObserver(ctx context.Context, arg db.UpsertObserverParams) (db.Observer, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpsertObserver", ctx, publicKey) + ret := m.ctrl.Call(m, "UpsertObserver", ctx, arg) ret0, _ := ret[0].(db.Observer) ret1, _ := ret[1].(error) return ret0, ret1 } // UpsertObserver indicates an expected call of UpsertObserver. -func (mr *MockQuerierMockRecorder) UpsertObserver(ctx, publicKey any) *gomock.Call { +func (mr *MockQuerierMockRecorder) UpsertObserver(ctx, arg any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertObserver", reflect.TypeOf((*MockQuerier)(nil).UpsertObserver), ctx, publicKey) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertObserver", reflect.TypeOf((*MockQuerier)(nil).UpsertObserver), ctx, arg) } // UpsertObserverBroker mocks base method. @@ -2218,20 +2506,6 @@ func (mr *MockQuerierMockRecorder) UpsertRegion(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertRegion", reflect.TypeOf((*MockQuerier)(nil).UpsertRegion), ctx, arg) } -// UpsertTraceIATA mocks base method. -func (m *MockQuerier) UpsertTraceIATA(ctx context.Context, arg db.UpsertTraceIATAParams) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpsertTraceIATA", ctx, arg) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpsertTraceIATA indicates an expected call of UpsertTraceIATA. -func (mr *MockQuerierMockRecorder) UpsertTraceIATA(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertTraceIATA", reflect.TypeOf((*MockQuerier)(nil).UpsertTraceIATA), ctx, arg) -} - // UpsertTransportScope mocks base method. func (m *MockQuerier) UpsertTransportScope(ctx context.Context, arg db.UpsertTransportScopeParams) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index e08581cd..709db8f6 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -16,96 +16,40 @@ type Account struct { DeactivatedAt pgtype.Timestamptz `json:"deactivated_at"` } -type AnalyticsHourlyIataStat struct { - Iata string `json:"iata"` - Hour pgtype.Timestamptz `json:"hour"` - ObservationCount int64 `json:"observation_count"` - UniquePackets int64 `json:"unique_packets"` -} - -type AnalyticsLiveHourlyIataStat struct { - Iata string `json:"iata"` - Hour pgtype.Timestamptz `json:"hour"` - ObservationCount int64 `json:"observation_count"` - UniquePackets int64 `json:"unique_packets"` +type AnalyticsDirtyHour struct { + Hour pgtype.Timestamptz `json:"hour"` + EnqueuedAt pgtype.Timestamptz `json:"enqueued_at"` } -type AnalyticsLiveObserverActivityHourly struct { - ObserverID uuid.UUID `json:"observer_id"` - PayloadType int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` +type AnalyticsHourlyAdvertHearing struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + OriginPubkey []byte `json:"origin_pubkey"` Observations int64 `json:"observations"` - AirtimeMs float32 `json:"airtime_ms"` - AirtimeN int64 `json:"airtime_n"` - SnrSum float32 `json:"snr_sum"` - SnrN int64 `json:"snr_n"` - SnrMin float32 `json:"snr_min"` - RssiSum int64 `json:"rssi_sum"` - RssiN int64 `json:"rssi_n"` + LastHeard pgtype.Timestamptz `json:"last_heard"` + Name *string `json:"name"` + NodeType *int16 `json:"node_type"` } -type AnalyticsLivePathStatsHourly struct { - Iata string `json:"iata"` - Hour interface{} `json:"hour"` - Category int32 `json:"category"` - HashBytes int32 `json:"hash_bytes"` - Entries int32 `json:"entries"` - Receptions int64 `json:"receptions"` +type AnalyticsHourlyAdvertSet struct { + Hour pgtype.Timestamptz `json:"hour"` + OriginPubkey []byte `json:"origin_pubkey"` + Iatas []string `json:"iatas"` + AdvertPackets int64 `json:"advert_packets"` + FloodPackets int64 `json:"flood_packets"` + DirectPackets int64 `json:"direct_packets"` } -type AnalyticsLivePayloadBreakdownByIatum struct { - Iata string `json:"iata"` - PayloadType *int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - Count int64 `json:"count"` -} - -type AnalyticsLiveSignalStatsHourly struct { - Iata string `json:"iata"` - Hour interface{} `json:"hour"` - Kind int32 `json:"kind"` - SnrBin int32 `json:"snr_bin"` - RssiBin int32 `json:"rssi_bin"` - Receptions int64 `json:"receptions"` - SnrSamples int64 `json:"snr_samples"` - SnrSum float64 `json:"snr_sum"` - RssiSamples int64 `json:"rssi_samples"` - RssiSum float64 `json:"rssi_sum"` -} - -type AnalyticsLiveTopAdvertisersByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - AdvertCount int64 `json:"advert_count"` - FloodAdvertCount int64 `json:"flood_advert_count"` - DirectAdvertCount int64 `json:"direct_advert_count"` - LastHeard interface{} `json:"last_heard"` -} - -type AnalyticsLiveTopObserversByIatum struct { +type AnalyticsHourlyIataObservation struct { + Hour pgtype.Timestamptz `json:"hour"` Iata string `json:"iata"` - ObserverID uuid.UUID `json:"observer_id"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` - Bucket pgtype.Timestamptz `json:"bucket"` ObservationCount int64 `json:"observation_count"` } -type AnalyticsLiveTopTalkersByIatum struct { - Iata string `json:"iata"` - SenderName *string `json:"sender_name"` - Bucket pgtype.Timestamptz `json:"bucket"` - MessageCount int64 `json:"message_count"` - LastSent interface{} `json:"last_sent"` -} - -type AnalyticsObserverActivityHourly struct { +type AnalyticsHourlyObserverActivity struct { + Hour pgtype.Timestamptz `json:"hour"` ObserverID uuid.UUID `json:"observer_id"` PayloadType int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs *float32 `json:"airtime_ms"` AirtimeN int64 `json:"airtime_n"` @@ -116,62 +60,81 @@ type AnalyticsObserverActivityHourly struct { RssiN int64 `json:"rssi_n"` } -type AnalyticsPathStatsHourly struct { - Iata string `json:"iata"` +type AnalyticsHourlyObserverIdentity struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + ObservationCount int64 `json:"observation_count"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` +} + +type AnalyticsHourlyPacketSet struct { + Hour pgtype.Timestamptz `json:"hour"` + Iatas []string `json:"iatas"` + Packets int64 `json:"packets"` +} + +type AnalyticsHourlyPath struct { Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` Category int32 `json:"category"` HashBytes int32 `json:"hash_bytes"` Entries int32 `json:"entries"` Receptions int64 `json:"receptions"` } -type AnalyticsPayloadBreakdownByIatum struct { +type AnalyticsHourlyPayloadBreakdown struct { + Hour pgtype.Timestamptz `json:"hour"` Iata string `json:"iata"` PayloadType int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` Count int64 `json:"count"` } -type AnalyticsSignalStatsHourly struct { - Iata string `json:"iata"` +type AnalyticsHourlyScopeSet struct { + Hour pgtype.Timestamptz `json:"hour"` + Iatas []string `json:"iatas"` + ScopeID int32 `json:"scope_id"` + Packets int64 `json:"packets"` +} + +type AnalyticsHourlySignal struct { Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` Kind int32 `json:"kind"` SnrBin int32 `json:"snr_bin"` RssiBin int32 `json:"rssi_bin"` Receptions int64 `json:"receptions"` SnrSamples int64 `json:"snr_samples"` - SnrSum *float64 `json:"snr_sum"` + SnrSum float64 `json:"snr_sum"` RssiSamples int64 `json:"rssi_samples"` - RssiSum *float64 `json:"rssi_sum"` + RssiSum float64 `json:"rssi_sum"` } -type AnalyticsTopAdvertisersByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Bucket pgtype.Timestamptz `json:"bucket"` - AdvertCount int64 `json:"advert_count"` - FloodAdvertCount int64 `json:"flood_advert_count"` - DirectAdvertCount int64 `json:"direct_advert_count"` - LastHeard pgtype.Timestamptz `json:"last_heard"` - Name *string `json:"name"` - NodeType *int16 `json:"node_type"` +type AnalyticsHourlyTalkerSet struct { + Hour pgtype.Timestamptz `json:"hour"` + SenderName string `json:"sender_name"` + Iatas []string `json:"iatas"` + Messages int64 `json:"messages"` + LastSent pgtype.Timestamptz `json:"last_sent"` } -type AnalyticsTopObserversByIatum struct { - Iata string `json:"iata"` - ObserverID uuid.UUID `json:"observer_id"` - Bucket pgtype.Timestamptz `json:"bucket"` - ObservationCount int64 `json:"observation_count"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` +type AnalyticsRawState struct { + Singleton bool `json:"singleton"` + RawDeletedBefore pgtype.Timestamptz `json:"raw_deleted_before"` + Coverage int64 `json:"coverage"` } -type AnalyticsTopTalkersByIatum struct { - Iata string `json:"iata"` - SenderName string `json:"sender_name"` - Bucket pgtype.Timestamptz `json:"bucket"` - MessageCount int64 `json:"message_count"` - LastSent pgtype.Timestamptz `json:"last_sent"` +type AnalyticsRollupHour struct { + Hour pgtype.Timestamptz `json:"hour"` + Status string `json:"status"` + RolledAt pgtype.Timestamptz `json:"rolled_at"` + ContentHash *string `json:"content_hash"` +} + +type AnalyticsState struct { + Singleton bool `json:"singleton"` + Revision int64 `json:"revision"` } type Channel struct { @@ -185,7 +148,7 @@ type Channel struct { KeyKnown *bool `json:"key_known"` FirstSeen pgtype.Timestamptz `json:"first_seen"` LastSeen pgtype.Timestamptz `json:"last_seen"` - MessageCount *int64 `json:"message_count"` + MessageCount int64 `json:"message_count"` } type ChannelConfigScope struct { @@ -287,100 +250,11 @@ type MeshmapperZoneList struct { LastError string `json:"last_error"` } -type MvHourlyIataStat struct { - Iata string `json:"iata"` - Hour pgtype.Timestamptz `json:"hour"` - ObservationCount int64 `json:"observation_count"` - UniquePackets int64 `json:"unique_packets"` - ActiveObservers int64 `json:"active_observers"` -} - -type MvObserverActivityHourly struct { - ObserverID uuid.UUID `json:"observer_id"` - PayloadType int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - Observations int64 `json:"observations"` - AirtimeMs float32 `json:"airtime_ms"` - AirtimeN int64 `json:"airtime_n"` - SnrSum float32 `json:"snr_sum"` - SnrN int64 `json:"snr_n"` - SnrMin float32 `json:"snr_min"` - RssiSum int64 `json:"rssi_sum"` - RssiN int64 `json:"rssi_n"` -} - -type MvPathStatsHourly struct { - Iata string `json:"iata"` - Hour interface{} `json:"hour"` - Category int32 `json:"category"` - HashBytes int32 `json:"hash_bytes"` - Entries int32 `json:"entries"` - Receptions int64 `json:"receptions"` -} - -type MvPayloadBreakdownByIatum struct { - Iata string `json:"iata"` - PayloadType *int16 `json:"payload_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - Count int64 `json:"count"` -} - type MvRadioPreset struct { - Preset string `json:"preset"` - Iata string `json:"iata"` - SourceType string `json:"source_type"` - Count int64 `json:"count"` -} - -type MvSignalStatsHourly struct { - Iata string `json:"iata"` - Hour interface{} `json:"hour"` - Kind int32 `json:"kind"` - SnrBin int32 `json:"snr_bin"` - RssiBin int32 `json:"rssi_bin"` - Receptions int64 `json:"receptions"` - SnrSamples int64 `json:"snr_samples"` - SnrSum float64 `json:"snr_sum"` - RssiSamples int64 `json:"rssi_samples"` - RssiSum float64 `json:"rssi_sum"` -} - -type MvTopAdvertisersByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - AdvertCount int64 `json:"advert_count"` - FloodAdvertCount int64 `json:"flood_advert_count"` - DirectAdvertCount int64 `json:"direct_advert_count"` - LastHeard interface{} `json:"last_heard"` -} - -type MvTopNodesByIatum struct { - Iata string `json:"iata"` - NodeID uuid.UUID `json:"node_id"` - Name *string `json:"name"` - NodeType int16 `json:"node_type"` - ObservationCount *int64 `json:"observation_count"` - LastHeard pgtype.Timestamptz `json:"last_heard"` -} - -type MvTopObserversByIatum struct { - Iata string `json:"iata"` - ObserverID uuid.UUID `json:"observer_id"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` - Bucket pgtype.Timestamptz `json:"bucket"` - ObservationCount int64 `json:"observation_count"` -} - -type MvTopTalkersByIatum struct { - Iata string `json:"iata"` - SenderName *string `json:"sender_name"` - Bucket pgtype.Timestamptz `json:"bucket"` - MessageCount int64 `json:"message_count"` - LastSent interface{} `json:"last_sent"` + Preset string `json:"preset"` + Iata *string `json:"iata"` + SourceType string `json:"source_type"` + Count int64 `json:"count"` } type Node struct { @@ -455,6 +329,8 @@ type Observer struct { ObservationCount *int64 `json:"observation_count"` Metadata []byte `json:"metadata"` RegionScope *string `json:"region_scope"` + LastIata *string `json:"last_iata"` + LastIataAt pgtype.Timestamptz `json:"last_iata_at"` } type ObserverBroker struct { @@ -525,6 +401,7 @@ type Packet struct { TraceTag []byte `json:"trace_tag"` FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` } type PacketObservation struct { @@ -569,12 +446,42 @@ type RegionIata struct { AddedAt pgtype.Timestamptz `json:"added_at"` } +type RollupOb struct { + PacketHash []byte `json:"packet_hash"` + ObserverID uuid.UUID `json:"observer_id"` + Iata string `json:"iata"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + PathLengthByte int16 `json:"path_length_byte"` + HashSize int16 `json:"hash_size"` + HopCount int16 `json:"hop_count"` + PathLen int32 `json:"path_len"` + Rssi *int16 `json:"rssi"` + Snr *float32 `json:"snr"` + AirtimeMs *float32 `json:"airtime_ms"` + PayloadType *int16 `json:"payload_type"` + RouteType int16 `json:"route_type"` + OriginPubkey []byte `json:"origin_pubkey"` + ScopeID *int32 `json:"scope_id"` +} + type TraceIata struct { TraceTag []byte `json:"trace_tag"` Iata string `json:"iata"` LastHeard pgtype.Timestamptz `json:"last_heard"` } +type TraceTag struct { + TraceTag []byte `json:"trace_tag"` + FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` + LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + PacketCount int64 `json:"packet_count"` + TraceType *string `json:"trace_type"` + ScopeID *int32 `json:"scope_id"` + BestPayload []byte `json:"best_payload"` + BestPathLen int32 `json:"best_path_len"` + Heard bool `json:"heard"` +} + type TransportScope struct { ID int32 `json:"id"` Name string `json:"name"` diff --git a/db/sqlc/paths.sql.go b/db/sqlc/paths.sql.go index 05bf5b1c..f5c24f13 100644 --- a/db/sqlc/paths.sql.go +++ b/db/sqlc/paths.sql.go @@ -13,11 +13,11 @@ import ( const getPathStats = `-- name: GetPathStats :many WITH readings AS ( - SELECT iata, hour, category, hash_bytes, entries, receptions FROM mv_path_stats_hourly + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND COALESCE(cardinality($3::bpchar[]), 0) = 0 UNION ALL - SELECT iata, hour, category, hash_bytes, entries, receptions FROM mv_path_stats_hourly + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_hourly_paths WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) ) @@ -45,7 +45,7 @@ type GetPathStatsRow struct { Receptions int64 `json:"receptions"` } -// All HTTP aggregates read the compact classification snapshot. +// All HTTP aggregates read the hourly rollup, never observations. func (q *Queries) GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]GetPathStatsRow, error) { rows, err := q.db.Query(ctx, getPathStats, arg.Since, arg.Until, arg.Iatas) if err != nil { @@ -72,12 +72,3 @@ func (q *Queries) GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]G } return items, nil } - -const refreshPathStats = `-- name: RefreshPathStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_path_stats_hourly -` - -func (q *Queries) RefreshPathStats(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshPathStats) - return err -} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index c2adf524..1eaf656e 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -40,8 +40,11 @@ type Querier interface { // Opt-in age-out: preserve retained history and manually recorded ownership. // Bound deletions per cleanup tick and skip observers being updated by ingest. DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) - // Locks, archives and cascades one bounded packet cohort in a single transaction. + // Deletes one bounded cohort (observations cascade). SKIP LOCKED leaves packets an in-flight + // observation insert holds. raw_deleted_before tells the rollup which hours lost raw rows. DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) + // Bumps coverage when hours go, so cached responses that included them are not reused. + DeleteOldRollups(ctx context.Context, cutoff pgtype.Timestamptz) error // One batch of routes past retention, or past grace with too few observations. // GREATEST keeps the scan on idx_known_routes_last_seen. DeleteOldRoutes(ctx context.Context, arg DeleteOldRoutesParams) (int64, error) @@ -49,12 +52,25 @@ type Querier interface { DeleteOldTelemetry(ctx context.Context, reportedAt pgtype.Timestamptz) error // Keeps the trace IATA filter in step with packet retention. DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Timestamptz) error + DeleteOldTraceTags(ctx context.Context, lastHeardAt pgtype.Timestamptz) error DeleteRegionIATAsNotIn(ctx context.Context, arg DeleteRegionIATAsNotInParams) error + DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error + // Dirty hours that can no longer be re-rolled. + DeleteStaleDirtyHours(ctx context.Context) error + FillRollupObs(ctx context.Context, dollar_1 pgtype.Timestamptz) error + // Marks the hour complete and bumps the revision only when its content changed. + FinishRollupHour(ctx context.Context, arg FinishRollupHourParams) (bool, error) GetAccount(ctx context.Context, id uuid.UUID) (Account, error) + // Both counters only grow, so their sum changes whenever rolled content or coverage does. + GetAnalyticsRevision(ctx context.Context) (int64, error) GetChannelByID(ctx context.Context, id int32) (Channel, error) // Returns neighbors of a node that are in a different IATA. GetCrossIATANeighbors(ctx context.Context, arg GetCrossIATANeighborsParams) ([]GetCrossIATANeighborsRow, error) - GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]MvHourlyIataStat, error) + GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) + // ============================================================ + // STATS + // ============================================================ + GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]GetHourlyStatsRow, error) GetIATA(ctx context.Context, iata string) (IataCode, error) // An imported MeshMapper boundary overrides the configured one. border is NULL // when neither exists; a missing row (unknown IATA) is sql.ErrNoRows, same @@ -83,30 +99,30 @@ type Querier interface { // Count packet identities, not reception rows: a broker repeat or another // reception by the same observer must not inflate the comparison. GetObserverComparison(ctx context.Context, arg GetObserverComparisonParams) (GetObserverComparisonRow, error) - GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) + GetObserverLastIATA(ctx context.Context, id uuid.UUID) (string, error) GetObserverRadio(ctx context.Context, id uuid.UUID) (GetObserverRadioRow, error) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) GetObserverTelemetry(ctx context.Context, arg GetObserverTelemetryParams) ([]GetObserverTelemetryRow, error) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPacketByHashRow, error) - GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) // Return distinct observation IATAs in first-heard order for path resolution, // without fetching full observations separately for every trace packet. GetPacketsByTraceTag(ctx context.Context, decode string) ([]GetPacketsByTraceTagRow, error) - // All HTTP aggregates read the compact classification snapshot. + // All HTTP aggregates read the hourly rollup, never observations. GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]GetPathStatsRow, error) GetRadioPresets(ctx context.Context, arg GetRadioPresetsParams) ([]MvRadioPreset, error) GetRegion(ctx context.Context, id int32) (GetRegionRow, error) GetRegionBySlug(ctx context.Context, slug string) (GetRegionBySlugRow, error) GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) + // Packets and IATAs cover every retained rollup hour; IATAs are those that heard the + // scope's own packets. Observer and node counts are current memberships. GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) GetScopeNames(ctx context.Context) ([]string, error) - // Aggregate matching observations once, separately from node memberships to avoid - // a cross-join. Empty IATAs keep the original global counts, including associations - // whose observations have expired; the filtered aggregates are empty in that case. - GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) - // Read compact hourly snapshots, never observations on an HTTP request. + // Packets since the given hour come from the IATA-set rollup (counted once per hour heard). + // Observer and node counts are current memberships; observers filter by their latest IATA. + GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([]GetScopeStatsRow, error) + // Read the hourly rollup, never observations on an HTTP request. // Weight averages by sample counts instead of averaging regional/hourly means. GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) // Repeaters/room servers (node_type 2/3) whose current advert-derived clock drift exceeds @@ -116,21 +132,24 @@ type Querier interface { GetStatsClockDrift(ctx context.Context, arg GetStatsClockDriftParams) ([]GetStatsClockDriftRow, error) // Returns node counts grouped by type, optionally filtered by IATA. GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]GetStatsNodeTypesRow, error) - // ============================================================ - // STATS - // ============================================================ - GetStatsOverview(ctx context.Context, dollar_1 []string) (GetStatsOverviewRow, error) - // Payload-type counts for the IATA within the window, summed from the - // precomputed hourly buckets. + // Payload-type observation counts since the given hour, from the hourly rollup. GetStatsPayloadBreakdown(ctx context.Context, arg GetStatsPayloadBreakdownParams) ([]GetStatsPayloadBreakdownRow, error) - // Top N advertisers in the window, summed from the hourly buckets. + // One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours. + // Counts sum; observers, IATAs and scopes are distinct across the window; packets count + // once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate. + GetStatsSeries(ctx context.Context, arg GetStatsSeriesParams) ([]GetStatsSeriesRow, error) + // Top N advertisers since the given hour. Each ADVERT packet counts once per hour heard, + // however many of the requested IATAs heard it (IATA-set rollup). Live names win. GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdvertisersParams) ([]GetStatsTopAdvertisersRow, error) - // Top N observers for the IATA within the window, summed from the precomputed - // hourly buckets. Counts sum across matched IATAs; iata is a representative one. + // Top N observers since the given hour. Counts sum across matched IATAs; iata is a + // representative one. Live names win over the rolled snapshot. GetStatsTopObservers(ctx context.Context, arg GetStatsTopObserversParams) ([]GetStatsTopObserversRow, error) - // Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. + // Top N senders since the given hour. Each message counts once per hour heard, however many + // of the requested IATAs heard it (IATA-set rollup). GetStatsTopTalkers(ctx context.Context, arg GetStatsTopTalkersParams) ([]GetStatsTopTalkersRow, error) - GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]MvTopNodesByIatum, error) + // Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; + // node_id is NULL once the node row is gone. iata is a representative one. + GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]GetTopNodesRow, error) GetTransportScopeByName(ctx context.Context, name string) (int32, error) GetTransportScopes(ctx context.Context) ([]GetTransportScopesRow, error) // ============================================================ @@ -138,11 +157,14 @@ type Querier interface { // ============================================================ // Read the immutable first-packet scope in the same statement as insertion. // A later reception's transport code must not give live and historical messages different tags. + // message_count is a lifetime count, bumped only for a new message. A historical (backfilled) + // message queues its observation hours for a re-roll in the same statement, so the two can't diverge. InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) // ============================================================ // PACKET OBSERVATIONS // ============================================================ - InsertObservation(ctx context.Context, arg InsertObservationParams) (PacketObservation, error) + // Bumps packets.observation_count only for a new row; a duplicate returns the current count. + InsertObservation(ctx context.Context, arg InsertObservationParams) (InsertObservationRow, error) // Inserts a telemetry snapshot for an observer. The reported_at timestamp should // be truncated to the configured resolution before calling to ensure deduplication. InsertObserverTelemetry(ctx context.Context, arg InsertObserverTelemetryParams) error @@ -170,6 +192,9 @@ type Querier interface { // Keep the non-null tuple boundary separate from the legacy optional cursor so // generic prepared plans can seek directly into the composite ordered index. ListChannelsAfter(ctx context.Context, arg ListChannelsAfterParams) ([]Channel, error) + // Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours + // wait: their first roll consumes the entry only if the entry predates its snapshot. + ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) ListIATAs(ctx context.Context) ([]IataCode, error) // Only one branch runs. Keep the IATA range ordered by the composite index: // generic plans can otherwise prefer scanning the global timestamp index. @@ -178,6 +203,7 @@ type Querier interface { // Returns messages after the given message ID, ordered oldest first. // Used for WS reconnect backfill. ListMessagesAfterID(ctx context.Context, arg ListMessagesAfterIDParams) ([]ListMessagesAfterIDRow, error) + ListMissingRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) ListNodeObservations(ctx context.Context, arg ListNodeObservationsParams) ([]ListNodeObservationsRow, error) // Limit the filtered node page before enriching IATA membership and neighbours. ListNodes(ctx context.Context, arg ListNodesParams) ([]ListNodesRow, error) @@ -224,9 +250,7 @@ type Querier interface { // ============================================================ // TRACES // ============================================================ - // Returns distinct trace tags with summary info, ordered by most recent first. - // IATA membership comes from trace_iatas (joining observations here spilled the - // hash join). Per-tag details filled in only for the returned page. + // Tags, most recent first. Filters match the tag summary (see RecordTrace). ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([]ListTraceTagsRow, error) // Returns GRP_TXT packets (payload_type=5) never successfully decrypted. Used at boot to // retry decryption against the current keystore for packets whose channel key was only added @@ -237,6 +261,12 @@ type Querier interface { ListUndecryptedGroupTextPacketsByHash(ctx context.Context, hashes [][]byte) ([]ListUndecryptedGroupTextPacketsByHashRow, error) ListZoneBoundaries(ctx context.Context) ([]MeshmapperZoneBoundary, error) ListZoneLists(ctx context.Context) ([]MeshmapperZoneList, error) + // Returns the hour's status, or 'partial' if raw deletion has reached it since registration. + LockRollupHour(ctx context.Context, hour pgtype.Timestamptz) (string, error) + // Hours whose raw rows cleanup already started deleting can never be rolled completely. + // Readers report the new status, so the revision moves too. + MarkPartialRollupHours(ctx context.Context) error + OldestMissingRollupHour(ctx context.Context) (pgtype.Timestamptz, error) PruneImportedRegions(ctx context.Context, keep []string) ([]string, error) // Drops imports for IATAs no longer configured, so their manual border returns. PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) @@ -248,16 +278,15 @@ type Querier interface { // 1/2/3/4-byte hop prefixes check prefix_1/2/3/4; ambiguity set supplied by AmbiguousPrefixes), // and stamps the survivors. ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams) (int64, error) - RefreshHourlyStats(ctx context.Context) error - RefreshObserverActivity(ctx context.Context) error - RefreshPathStats(ctx context.Context) error - RefreshPayloadBreakdown(ctx context.Context) error + // One hearing of a TRACE packet: trace_iatas refreshes at most hourly and the tag's heard + // window widens (the first hearing replaces the packet's provisional times). Packet counts + // and payloads come from UpsertPacket. + RecordTrace(ctx context.Context, arg RecordTraceParams) error RefreshRadioPresets(ctx context.Context) error - RefreshSignalStats(ctx context.Context) error - RefreshTopAdvertisers(ctx context.Context) error - RefreshTopNodes(ctx context.Context) error - RefreshTopObservers(ctx context.Context) error - RefreshTopTalkers(ctx context.Context) error + // Adds a 'missing' row for every eligible hour since the earliest retained observation + // (and not before @since), zero-observation hours included. + // No observations yet means nothing to register (GREATEST would ignore the NULL MIN). + RegisterRollupHours(ctx context.Context, since pgtype.Timestamptz) error // Batch form of ResolveEndpointHashes for a page of packets. Matches the cross // product of IATAs and hashes; callers pick out the pairs they asked for. ResolveEndpointHashPairs(ctx context.Context, arg ResolveEndpointHashPairsParams) ([]ResolveEndpointHashPairsRow, error) @@ -276,6 +305,24 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error + // Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. + RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error + RollIATAObservations(ctx context.Context, hour pgtype.Timestamptz) error + RollObserverActivity(ctx context.Context, hour pgtype.Timestamptz) error + RollObserverIdentity(ctx context.Context, hour pgtype.Timestamptz) error + RollPacketSets(ctx context.Context, hour pgtype.Timestamptz) error + // category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). + RollPaths(ctx context.Context, hour pgtype.Timestamptz) error + RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error + RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error + // kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. + // rssi=0 AND snr=0 means "not reported". + RollSignal(ctx context.Context, hour pgtype.Timestamptz) error + RollTalkerSets(ctx context.Context, hour pgtype.Timestamptz) error + // Order-independent fingerprint of every family row for the hour. + RollupContentHash(ctx context.Context, hour pgtype.Timestamptz) (string, error) + RollupUnlock(ctx context.Context) error // NULL payload/etag/checked_at retain the last good list after an error or 304. SaveChannelCatalogue(ctx context.Context, arg SaveChannelCatalogueParams) error // One statement commits the validated snapshot and its lookup identities together. @@ -293,11 +340,17 @@ type Querier interface { SetNodeMultibytePaths(ctx context.Context, id uuid.UUID) error SetNodeMultibyteTraces(ctx context.Context, id uuid.UUID) error SetPacketDecrypted(ctx context.Context, packetHash []byte) error + SetRollupHourPartial(ctx context.Context, hour pgtype.Timestamptz) error TouchObserverBrokers(ctx context.Context, arg TouchObserverBrokersParams) error // Batched flush of coalesced presence bumps. GREATEST keeps a late flush // from regressing a newer write-through (e.g. a status update). TouchObservers(ctx context.Context, arg TouchObserversParams) error TouchPackets(ctx context.Context, arg TouchPacketsParams) error + // ============================================================ + // HOURLY ANALYTICS ROLLUPS + // An hour [H, H+1h) is eligible once now() >= H + 95 min (ingest clamps heard_at to ±30 min). + // ============================================================ + TryRollupLock(ctx context.Context) (bool, error) // Records the observer's own OTA-reported region scope, from the "self" // field of a /neighbors report. Always known (not queried OTA), so this // unconditionally overwrites, unlike the neighbor-side region_scope. @@ -356,7 +409,8 @@ type Querier interface { // ============================================================ // OBSERVERS // ============================================================ - UpsertObserver(ctx context.Context, publicKey []byte) (Observer, error) + // Empty iata (status/neighbors) leaves last_iata alone; otherwise the newest iata_at wins. + UpsertObserver(ctx context.Context, arg UpsertObserverParams) (Observer, error) // ============================================================ // OBSERVER BROKERS // ============================================================ @@ -365,10 +419,10 @@ type Querier interface { // ============================================================ // PACKETS // ============================================================ + // A new TRACE packet adds itself to its tag summary in the same statement, so the summary + // can't miss a stored packet. 'TRACE' > 'PING'; the longest path keeps the best payload. UpsertPacket(ctx context.Context, arg UpsertPacketParams) (UpsertPacketRow, error) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int32, error) - // Refreshes at most hourly so repeat hears don't churn the row. - UpsertTraceIATA(ctx context.Context, arg UpsertTraceIATAParams) error // ============================================================ // TRANSPORT CODES // ============================================================ diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 30db7f5c..0a3b572f 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -254,7 +254,21 @@ func (q *Queries) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timest } const deleteOldPackets = `-- name: DeleteOldPackets :one -SELECT archive_delete_packets($1::timestamptz, $2::integer)::bigint +WITH victims AS MATERIALIZED ( + SELECT vp.packet_hash, vp.last_heard_at FROM packets vp + WHERE vp.last_heard_at < $1::timestamptz + ORDER BY vp.last_heard_at, vp.packet_hash + LIMIT $2::integer + FOR UPDATE OF vp SKIP LOCKED +), deleted AS ( + DELETE FROM packets dp USING victims v WHERE dp.packet_hash = v.packet_hash + RETURNING dp.last_heard_at +), mark AS ( + UPDATE analytics_raw_state + SET raw_deleted_before = GREATEST(raw_deleted_before, (SELECT max(last_heard_at) FROM deleted)) + WHERE EXISTS (SELECT 1 FROM deleted) +) +SELECT count(*) FROM deleted ` type DeleteOldPacketsParams struct { @@ -262,12 +276,13 @@ type DeleteOldPacketsParams struct { BatchSize int32 `json:"batch_size"` } -// Locks, archives and cascades one bounded packet cohort in a single transaction. +// Deletes one bounded cohort (observations cascade). SKIP LOCKED leaves packets an in-flight +// observation insert holds. raw_deleted_before tells the rollup which hours lost raw rows. func (q *Queries) DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) { row := q.db.QueryRow(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) - var column_1 int64 - err := row.Scan(&column_1) - return column_1, err + var count int64 + err := row.Scan(&count) + return count, err } const deleteOldRoutes = `-- name: DeleteOldRoutes :execrows @@ -326,6 +341,15 @@ func (q *Queries) DeleteOldTraceIATAs(ctx context.Context, lastHeard pgtype.Time return err } +const deleteOldTraceTags = `-- name: DeleteOldTraceTags :exec +DELETE FROM trace_tags WHERE last_heard_at < $1 +` + +func (q *Queries) DeleteOldTraceTags(ctx context.Context, lastHeardAt pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, deleteOldTraceTags, lastHeardAt) + return err +} + const deleteRegionIATAsNotIn = `-- name: DeleteRegionIATAsNotIn :exec DELETE FROM region_iatas WHERE region_id = $1 AND NOT (iata = ANY($2::bpchar[])) ` @@ -439,34 +463,38 @@ func (q *Queries) GetCrossIATANeighbors(ctx context.Context, arg GetCrossIATANei } const getHourlyStats = `-- name: GetHourlyStats :many -SELECT iata, hour, observation_count, unique_packets, active_observers -FROM mv_hourly_iata_stats + +SELECT iata, hour, observation_count +FROM analytics_hourly_iata_observations WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND hour >= NOW() - $2::interval + AND hour >= $2::timestamptz ORDER BY iata, hour ` type GetHourlyStatsParams struct { - Column1 []string `json:"column_1"` - Column2 pgtype.Interval `json:"column_2"` + Iatas []string `json:"iatas"` + Since pgtype.Timestamptz `json:"since"` } -func (q *Queries) GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]MvHourlyIataStat, error) { - rows, err := q.db.Query(ctx, getHourlyStats, arg.Column1, arg.Column2) +type GetHourlyStatsRow struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount int64 `json:"observation_count"` +} + +// ============================================================ +// STATS +// ============================================================ +func (q *Queries) GetHourlyStats(ctx context.Context, arg GetHourlyStatsParams) ([]GetHourlyStatsRow, error) { + rows, err := q.db.Query(ctx, getHourlyStats, arg.Iatas, arg.Since) if err != nil { return nil, err } defer rows.Close() - items := []MvHourlyIataStat{} + items := []GetHourlyStatsRow{} for rows.Next() { - var i MvHourlyIataStat - if err := rows.Scan( - &i.Iata, - &i.Hour, - &i.ObservationCount, - &i.UniquePackets, - &i.ActiveObservers, - ); err != nil { + var i GetHourlyStatsRow + if err := rows.Scan(&i.Iata, &i.Hour, &i.ObservationCount); err != nil { return nil, err } items = append(items, i) @@ -786,7 +814,7 @@ func (q *Queries) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]Ge const getObserverActivityHourly = `-- name: GetObserverActivityHourly :many SELECT - date_bin($3::interval, bucket, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + date_bin($3::interval, hour, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, SUM(observations)::bigint AS observations, COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, @@ -795,8 +823,8 @@ SELECT COALESCE(MIN(snr_min), 0)::real AS snr_min, COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $4::timestamptz +FROM analytics_hourly_observer_activity +WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < $4::timestamptz GROUP BY 1 ORDER BY 1 ` @@ -858,8 +886,8 @@ func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserver const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count -FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $3::timestamptz +FROM analytics_hourly_observer_activity +WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -1081,7 +1109,7 @@ func (q *Queries) GetObserverBrokers(ctx context.Context, observerID uuid.UUID) } const getObserverByID = `-- name: GetObserverByID :one -SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope FROM observers WHERE id = $1 +SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope, last_iata, last_iata_at FROM observers WHERE id = $1 ` func (q *Queries) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, error) { @@ -1109,12 +1137,14 @@ func (q *Queries) GetObserverByID(ctx context.Context, id uuid.UUID) (Observer, &i.ObservationCount, &i.Metadata, &i.RegionScope, + &i.LastIata, + &i.LastIataAt, ) return i, err } const getObserverByPubkey = `-- name: GetObserverByPubkey :one -SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope FROM observers WHERE public_key = $1 +SELECT id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope, last_iata, last_iata_at FROM observers WHERE public_key = $1 ` func (q *Queries) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Observer, error) { @@ -1142,22 +1172,21 @@ func (q *Queries) GetObserverByPubkey(ctx context.Context, publicKey []byte) (Ob &i.ObservationCount, &i.Metadata, &i.RegionScope, + &i.LastIata, + &i.LastIataAt, ) return i, err } const getObserverLastIATA = `-- name: GetObserverLastIATA :one -SELECT iata FROM packet_observations -WHERE observer_id = $1 -ORDER BY heard_at DESC -LIMIT 1 +SELECT COALESCE(last_iata, '')::text FROM observers WHERE id = $1 ` -func (q *Queries) GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) { - row := q.db.QueryRow(ctx, getObserverLastIATA, observerID) - var iata string - err := row.Scan(&iata) - return iata, err +func (q *Queries) GetObserverLastIATA(ctx context.Context, id uuid.UUID) (string, error) { + row := q.db.QueryRow(ctx, getObserverLastIATA, id) + var column_1 string + err := row.Scan(&column_1) + return column_1, err } const getObserverRadio = `-- name: GetObserverRadio :one @@ -1280,7 +1309,7 @@ func (q *Queries) GetObserverTelemetry(ctx context.Context, arg GetObserverTelem } const getPacketByHash = `-- name: GetPacketByHash :one -SELECT p.packet_hash, p.payload_type, p.payload_version, p.route_type, p.transport_codes_present, p.region_code, p.sub_region_code, p.scope_id, p.origin_pubkey, p.raw_payload, p.raw_header, p.parsed_payload, p.decrypted, p.channel_hash, p.trace_tag, p.first_heard_at, p.last_heard_at, ts.name AS scope_name, +SELECT p.packet_hash, p.payload_type, p.payload_version, p.route_type, p.transport_codes_present, p.region_code, p.sub_region_code, p.scope_id, p.origin_pubkey, p.raw_payload, p.raw_header, p.parsed_payload, p.decrypted, p.channel_hash, p.trace_tag, p.first_heard_at, p.last_heard_at, p.observation_count, ts.name AS scope_name, cm.sender_name AS cm_sender_name, cm.content AS cm_content, cm.sent_at AS cm_sent_at @@ -1308,6 +1337,7 @@ type GetPacketByHashRow struct { TraceTag []byte `json:"trace_tag"` FirstHeardAt pgtype.Timestamptz `json:"first_heard_at"` LastHeardAt pgtype.Timestamptz `json:"last_heard_at"` + ObservationCount int64 `json:"observation_count"` ScopeName *string `json:"scope_name"` CmSenderName *string `json:"cm_sender_name"` CmContent *string `json:"cm_content"` @@ -1335,6 +1365,7 @@ func (q *Queries) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPa &i.TraceTag, &i.FirstHeardAt, &i.LastHeardAt, + &i.ObservationCount, &i.ScopeName, &i.CmSenderName, &i.CmContent, @@ -1343,17 +1374,6 @@ func (q *Queries) GetPacketByHash(ctx context.Context, packetHash []byte) (GetPa return i, err } -const getPacketObservationCount = `-- name: GetPacketObservationCount :one -SELECT COUNT(*) FROM packet_observations WHERE packet_hash = $1 -` - -func (q *Queries) GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) { - row := q.db.QueryRow(ctx, getPacketObservationCount, packetHash) - var count int64 - err := row.Scan(&count) - return count, err -} - const getPacketsByTraceTag = `-- name: GetPacketsByTraceTag :many SELECT encode(p.packet_hash, 'hex') AS packet_hash_hex, p.route_type, @@ -1546,19 +1566,18 @@ func (q *Queries) GetRegionIATAs(ctx context.Context, regionID int32) ([]string, const getScopeByName = `-- name: GetScopeByName :one SELECT ts.name, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count, - COUNT(DISTINCT n.id) AS node_count, - COUNT(DISTINCT po.iata) AS iata_count, - array_remove(array_agg(DISTINCT po.iata ORDER BY po.iata), NULL)::text[] AS iatas + COALESCE((SELECT SUM(s.packets) FROM analytics_hourly_scope_sets s WHERE s.scope_id = ts.id), 0)::bigint AS packet_count, + (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id)::bigint AS observer_count, + (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id)::bigint AS node_count, + cardinality(i.iatas)::bigint AS iata_count, + i.iatas FROM transport_scopes ts -LEFT JOIN packets p ON p.scope_id = ts.id -LEFT JOIN observer_scopes os ON os.scope_id = ts.id -LEFT JOIN observers o ON o.id = os.observer_id -LEFT JOIN packet_observations po ON po.observer_id = o.id -LEFT JOIN nodes n ON n.default_scope_id = ts.id +CROSS JOIN LATERAL ( + SELECT COALESCE(array_agg(DISTINCT x.iata::text ORDER BY x.iata::text), '{}')::text[] AS iatas + FROM analytics_hourly_scope_sets s CROSS JOIN LATERAL unnest(s.iatas) AS x(iata) + WHERE s.scope_id = ts.id +) i WHERE ts.name = $1 -GROUP BY ts.name ` type GetScopeByNameRow struct { @@ -1570,6 +1589,8 @@ type GetScopeByNameRow struct { Iatas []string `json:"iatas"` } +// Packets and IATAs cover every retained rollup hour; IATAs are those that heard the +// scope's own packets. Observer and node counts are current memberships. func (q *Queries) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) { row := q.db.QueryRow(ctx, getScopeByName, name) var i GetScopeByNameRow @@ -1634,38 +1655,42 @@ func (q *Queries) GetScopeNames(ctx context.Context) ([]string, error) { } const getScopeStats = `-- name: GetScopeStats :many -WITH observation_counts AS ( - SELECT p.scope_id, - COUNT(DISTINCT p.packet_hash) AS packet_count, - COUNT(DISTINCT os.observer_id) AS observer_count - FROM packet_observations po - JOIN packets p ON p.packet_hash = po.packet_hash - LEFT JOIN observer_scopes os ON os.scope_id = p.scope_id AND os.observer_id = po.observer_id - WHERE po.iata = ANY($1::bpchar[]) AND p.scope_id IS NOT NULL - GROUP BY p.scope_id +WITH packet_counts AS ( + SELECT s.scope_id, SUM(s.packets)::bigint AS packet_count + FROM analytics_hourly_scope_sets s + WHERE s.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR s.iatas && $2::bpchar[]) + GROUP BY s.scope_id +), observer_counts AS ( + SELECT os.scope_id, COUNT(*)::bigint AS observer_count + FROM observer_scopes os JOIN observers o ON o.id = os.observer_id + WHERE COALESCE(cardinality($2::bpchar[]), 0) = 0 OR o.last_iata = ANY($2::bpchar[]) + GROUP BY os.scope_id ), node_counts AS ( - SELECT n.default_scope_id AS scope_id, COUNT(*) AS node_count + SELECT n.default_scope_id AS scope_id, COUNT(*)::bigint AS node_count FROM nodes n - WHERE n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($1::bpchar[])) + WHERE n.default_scope_id IS NOT NULL + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 + OR n.id IN (SELECT node_id FROM node_iatas WHERE iata = ANY($2::bpchar[]))) GROUP BY n.default_scope_id ) SELECT ts.name, - CASE WHEN COALESCE(cardinality($1::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM packets p WHERE p.scope_id = ts.id) - ELSE COALESCE(oc.packet_count, 0) END::bigint AS packet_count, - CASE WHEN COALESCE(cardinality($1::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM observer_scopes os WHERE os.scope_id = ts.id) - ELSE COALESCE(oc.observer_count, 0) END::bigint AS observer_count, - CASE WHEN COALESCE(cardinality($1::bpchar[]), 0) = 0 - THEN (SELECT COUNT(*) FROM nodes n WHERE n.default_scope_id = ts.id) - ELSE COALESCE(nc.node_count, 0) END::bigint AS node_count + COALESCE(pc.packet_count, 0)::bigint AS packet_count, + COALESCE(oc.observer_count, 0)::bigint AS observer_count, + COALESCE(nc.node_count, 0)::bigint AS node_count FROM transport_scopes ts -LEFT JOIN observation_counts oc ON oc.scope_id = ts.id +LEFT JOIN packet_counts pc ON pc.scope_id = ts.id +LEFT JOIN observer_counts oc ON oc.scope_id = ts.id LEFT JOIN node_counts nc ON nc.scope_id = ts.id ORDER BY ts.name ` +type GetScopeStatsParams struct { + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` +} + type GetScopeStatsRow struct { Name string `json:"name"` PacketCount int64 `json:"packet_count"` @@ -1673,11 +1698,10 @@ type GetScopeStatsRow struct { NodeCount int64 `json:"node_count"` } -// Aggregate matching observations once, separately from node memberships to avoid -// a cross-join. Empty IATAs keep the original global counts, including associations -// whose observations have expired; the filtered aggregates are empty in that case. -func (q *Queries) GetScopeStats(ctx context.Context, iatas []string) ([]GetScopeStatsRow, error) { - rows, err := q.db.Query(ctx, getScopeStats, iatas) +// Packets since the given hour come from the IATA-set rollup (counted once per hour heard). +// Observer and node counts are current memberships; observers filter by their latest IATA. +func (q *Queries) GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([]GetScopeStatsRow, error) { + rows, err := q.db.Query(ctx, getScopeStats, arg.Since, arg.Iatas) if err != nil { return nil, err } @@ -1807,65 +1831,30 @@ func (q *Queries) GetStatsNodeTypes(ctx context.Context, dollar_1 []string) ([]G return items, nil } -const getStatsOverview = `-- name: GetStatsOverview :one - -SELECT - COUNT(DISTINCT po.packet_hash) AS total_packets, - COUNT(*) AS total_observations, - COUNT(DISTINCT po.observer_id) AS active_observers, - COUNT(DISTINCT po.iata) AS active_iatas -FROM packet_observations po -WHERE po.heard_at > NOW() - INTERVAL '24 hours' - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR po.iata = ANY($1::bpchar[])) -` - -type GetStatsOverviewRow struct { - TotalPackets int64 `json:"total_packets"` - TotalObservations int64 `json:"total_observations"` - ActiveObservers int64 `json:"active_observers"` - ActiveIatas int64 `json:"active_iatas"` -} - -// ============================================================ -// STATS -// ============================================================ -func (q *Queries) GetStatsOverview(ctx context.Context, dollar_1 []string) (GetStatsOverviewRow, error) { - row := q.db.QueryRow(ctx, getStatsOverview, dollar_1) - var i GetStatsOverviewRow - err := row.Scan( - &i.TotalPackets, - &i.TotalObservations, - &i.ActiveObservers, - &i.ActiveIatas, - ) - return i, err -} - const getStatsPayloadBreakdown = `-- name: GetStatsPayloadBreakdown :many SELECT payload_type, SUM(count)::bigint AS count -FROM mv_payload_breakdown_by_iata +FROM analytics_hourly_payload_breakdown WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) - AND bucket >= NOW() - $2::interval + AND hour >= $2::timestamptz GROUP BY payload_type ORDER BY count DESC ` type GetStatsPayloadBreakdownParams struct { - Column1 []string `json:"column_1"` - Column2 pgtype.Interval `json:"column_2"` + Iatas []string `json:"iatas"` + Since pgtype.Timestamptz `json:"since"` } type GetStatsPayloadBreakdownRow struct { - PayloadType *int16 `json:"payload_type"` - Count int64 `json:"count"` + PayloadType int16 `json:"payload_type"` + Count int64 `json:"count"` } -// Payload-type counts for the IATA within the window, summed from the -// precomputed hourly buckets. +// Payload-type observation counts since the given hour, from the hourly rollup. func (q *Queries) GetStatsPayloadBreakdown(ctx context.Context, arg GetStatsPayloadBreakdownParams) ([]GetStatsPayloadBreakdownRow, error) { - rows, err := q.db.Query(ctx, getStatsPayloadBreakdown, arg.Column1, arg.Column2) + rows, err := q.db.Query(ctx, getStatsPayloadBreakdown, arg.Iatas, arg.Since) if err != nil { return nil, err } @@ -1885,32 +1874,43 @@ func (q *Queries) GetStatsPayloadBreakdown(ctx context.Context, arg GetStatsPayl } const getStatsTopAdvertisers = `-- name: GetStatsTopAdvertisers :many -SELECT - node_id AS id, - name, - node_type, - COALESCE(SUM(advert_count), 0)::bigint AS advert_count, - COALESCE(SUM(flood_advert_count), 0)::bigint AS flood_advert_count, - COALESCE(SUM(direct_advert_count), 0)::bigint AS direct_advert_count, - MAX(last_heard)::timestamptz AS last_heard, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_advertisers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY node_id, name, node_type -ORDER BY advert_count DESC -LIMIT $3 +WITH ranked AS ( + SELECT origin_pubkey, SUM(advert_packets)::bigint AS advert_count, + SUM(flood_packets)::bigint AS flood_advert_count, SUM(direct_packets)::bigint AS direct_advert_count + FROM analytics_hourly_advert_sets + WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iatas && $2::bpchar[]) + GROUP BY origin_pubkey + ORDER BY advert_count DESC, origin_pubkey + LIMIT $3 +), heard AS ( + SELECT h.origin_pubkey, MAX(h.last_heard) AS last_heard, MAX(h.iata) AS iata, + MAX(h.name) AS name, MAX(h.node_type) AS node_type + FROM analytics_hourly_advert_hearings h + WHERE h.hour >= $1::timestamptz AND h.origin_pubkey IN (SELECT origin_pubkey FROM ranked) + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR h.iata = ANY($2::bpchar[])) + GROUP BY h.origin_pubkey +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, h.name, '')::text AS name, COALESCE(n.node_type, h.node_type, 0)::smallint AS node_type, + r.advert_count, r.flood_advert_count, r.direct_advert_count, + h.last_heard::timestamptz AS last_heard, COALESCE(h.iata, '')::bpchar AS iata +FROM ranked r +LEFT JOIN heard h ON h.origin_pubkey = r.origin_pubkey +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.advert_count DESC, r.origin_pubkey ` type GetStatsTopAdvertisersParams struct { - Column1 pgtype.Interval `json:"column_1"` - Column2 []string `json:"column_2"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } type GetStatsTopAdvertisersRow struct { - ID uuid.UUID `json:"id"` - Name *string `json:"name"` + NodeID pgtype.UUID `json:"node_id"` + PublicKey string `json:"public_key"` + Name string `json:"name"` NodeType int16 `json:"node_type"` AdvertCount int64 `json:"advert_count"` FloodAdvertCount int64 `json:"flood_advert_count"` @@ -1919,9 +1919,10 @@ type GetStatsTopAdvertisersRow struct { Iata string `json:"iata"` } -// Top N advertisers in the window, summed from the hourly buckets. +// Top N advertisers since the given hour. Each ADVERT packet counts once per hour heard, +// however many of the requested IATAs heard it (IATA-set rollup). Live names win. func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdvertisersParams) ([]GetStatsTopAdvertisersRow, error) { - rows, err := q.db.Query(ctx, getStatsTopAdvertisers, arg.Column1, arg.Column2, arg.Limit) + rows, err := q.db.Query(ctx, getStatsTopAdvertisers, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } @@ -1930,7 +1931,8 @@ func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdv for rows.Next() { var i GetStatsTopAdvertisersRow if err := rows.Scan( - &i.ID, + &i.NodeID, + &i.PublicKey, &i.Name, &i.NodeType, &i.AdvertCount, @@ -1950,38 +1952,42 @@ func (q *Queries) GetStatsTopAdvertisers(ctx context.Context, arg GetStatsTopAdv } const getStatsTopObservers = `-- name: GetStatsTopObservers :many -SELECT - observer_id AS id, - display_name, - observer_type, - COALESCE(SUM(observation_count), 0)::bigint AS observation_count, - COALESCE(MAX(iata), '')::bpchar AS iata -FROM mv_top_observers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) -GROUP BY observer_id, display_name, observer_type -ORDER BY observation_count DESC -LIMIT $3 +WITH ranked AS ( + SELECT observer_id, SUM(observation_count)::bigint AS observation_count, MAX(iata) AS iata, + MAX(display_name) AS display_name, MAX(observer_type) AS observer_type + FROM analytics_hourly_observer_identity + WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) + GROUP BY observer_id + ORDER BY observation_count DESC, observer_id + LIMIT $3 +) +SELECT r.observer_id AS id, COALESCE(o.display_name, r.display_name, '')::text AS display_name, + COALESCE(o.observer_type, r.observer_type, '')::text AS observer_type, + r.observation_count, r.iata::bpchar AS iata +FROM ranked r +LEFT JOIN observers o ON o.id = r.observer_id +ORDER BY r.observation_count DESC, r.observer_id ` type GetStatsTopObserversParams struct { - Column1 pgtype.Interval `json:"column_1"` - Column2 []string `json:"column_2"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } type GetStatsTopObserversRow struct { ID uuid.UUID `json:"id"` - DisplayName *string `json:"display_name"` - ObserverType *string `json:"observer_type"` + DisplayName string `json:"display_name"` + ObserverType string `json:"observer_type"` ObservationCount int64 `json:"observation_count"` Iata string `json:"iata"` } -// Top N observers for the IATA within the window, summed from the precomputed -// hourly buckets. Counts sum across matched IATAs; iata is a representative one. +// Top N observers since the given hour. Counts sum across matched IATAs; iata is a +// representative one. Live names win over the rolled snapshot. func (q *Queries) GetStatsTopObservers(ctx context.Context, arg GetStatsTopObserversParams) ([]GetStatsTopObserversRow, error) { - rows, err := q.db.Query(ctx, getStatsTopObservers, arg.Column1, arg.Column2, arg.Limit) + rows, err := q.db.Query(ctx, getStatsTopObservers, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } @@ -2009,31 +2015,32 @@ func (q *Queries) GetStatsTopObservers(ctx context.Context, arg GetStatsTopObser const getStatsTopTalkers = `-- name: GetStatsTopTalkers :many SELECT sender_name, - COALESCE(SUM(message_count), 0)::bigint AS message_count, + SUM(messages)::bigint AS message_count, MAX(last_sent)::timestamptz AS last_sent -FROM mv_top_talkers_by_iata -WHERE bucket >= NOW() - $1::interval - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) +FROM analytics_hourly_talker_sets +WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iatas && $2::bpchar[]) GROUP BY sender_name -ORDER BY message_count DESC +ORDER BY message_count DESC, sender_name LIMIT $3 ` type GetStatsTopTalkersParams struct { - Column1 pgtype.Interval `json:"column_1"` - Column2 []string `json:"column_2"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` } type GetStatsTopTalkersRow struct { - SenderName *string `json:"sender_name"` + SenderName string `json:"sender_name"` MessageCount int64 `json:"message_count"` LastSent pgtype.Timestamptz `json:"last_sent"` } -// Top N talkers (by decrypted sender_name) in the window, summed from the hourly buckets. +// Top N senders since the given hour. Each message counts once per hour heard, however many +// of the requested IATAs heard it (IATA-set rollup). func (q *Queries) GetStatsTopTalkers(ctx context.Context, arg GetStatsTopTalkersParams) ([]GetStatsTopTalkersRow, error) { - rows, err := q.db.Query(ctx, getStatsTopTalkers, arg.Column1, arg.Column2, arg.Limit) + rows, err := q.db.Query(ctx, getStatsTopTalkers, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } @@ -2053,31 +2060,57 @@ func (q *Queries) GetStatsTopTalkers(ctx context.Context, arg GetStatsTopTalkers } const getTopNodes = `-- name: GetTopNodes :many -SELECT iata, node_id, name, node_type, observation_count, last_heard FROM mv_top_nodes_by_iata -WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR iata = ANY($1::bpchar[])) -ORDER BY observation_count DESC -LIMIT $2 +WITH ranked AS ( + SELECT origin_pubkey, SUM(observations)::bigint AS observation_count, MAX(last_heard) AS last_heard, + MAX(iata) AS iata, MAX(name) AS name, MAX(node_type) AS node_type + FROM analytics_hourly_advert_hearings + WHERE hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) + GROUP BY origin_pubkey + ORDER BY observation_count DESC, origin_pubkey + LIMIT $3 +) +SELECT n.id AS node_id, encode(r.origin_pubkey, 'hex') AS public_key, + COALESCE(n.name, r.name, '')::text AS name, COALESCE(n.node_type, r.node_type, 0)::smallint AS node_type, + r.iata::bpchar AS iata, r.observation_count, r.last_heard::timestamptz AS last_heard +FROM ranked r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +ORDER BY r.observation_count DESC, r.origin_pubkey ` type GetTopNodesParams struct { - Column1 []string `json:"column_1"` - Limit int32 `json:"limit"` + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` + RowLimit int32 `json:"row_limit"` +} + +type GetTopNodesRow struct { + NodeID pgtype.UUID `json:"node_id"` + PublicKey string `json:"public_key"` + Name string `json:"name"` + NodeType int16 `json:"node_type"` + Iata string `json:"iata"` + ObservationCount int64 `json:"observation_count"` + LastHeard pgtype.Timestamptz `json:"last_heard"` } -func (q *Queries) GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]MvTopNodesByIatum, error) { - rows, err := q.db.Query(ctx, getTopNodes, arg.Column1, arg.Limit) +// Nodes by ADVERT hearings since the given hour. Live names win over the rolled snapshot; +// node_id is NULL once the node row is gone. iata is a representative one. +func (q *Queries) GetTopNodes(ctx context.Context, arg GetTopNodesParams) ([]GetTopNodesRow, error) { + rows, err := q.db.Query(ctx, getTopNodes, arg.Since, arg.Iatas, arg.RowLimit) if err != nil { return nil, err } defer rows.Close() - items := []MvTopNodesByIatum{} + items := []GetTopNodesRow{} for rows.Next() { - var i MvTopNodesByIatum + var i GetTopNodesRow if err := rows.Scan( - &i.Iata, &i.NodeID, + &i.PublicKey, &i.Name, &i.NodeType, + &i.Iata, &i.ObservationCount, &i.LastHeard, ); err != nil { @@ -2138,7 +2171,16 @@ WITH inserted AS ( INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (packet_hash) DO NOTHING - RETURNING id, packet_hash + RETURNING id, packet_hash, channel_id +), bump AS ( + UPDATE channels c SET message_count = c.message_count + 1 + FROM inserted WHERE c.id = inserted.channel_id +), dirty AS ( + INSERT INTO analytics_dirty_hours (hour) + SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC') + FROM packet_observations po JOIN inserted i ON i.packet_hash = po.packet_hash + WHERE $6::boolean + ON CONFLICT (hour) DO UPDATE SET enqueued_at = now() ) SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present FROM inserted @@ -2152,6 +2194,7 @@ type InsertChannelMessageParams struct { SenderName *string `json:"sender_name"` Content *string `json:"content"` SentAt pgtype.Timestamptz `json:"sent_at"` + Column6 bool `json:"column_6"` } type InsertChannelMessageRow struct { @@ -2165,6 +2208,8 @@ type InsertChannelMessageRow struct { // ============================================================ // Read the immutable first-packet scope in the same statement as insertion. // A later reception's transport code must not give live and historical messages different tags. +// message_count is a lifetime count, bumped only for a new message. A historical (backfilled) +// message queues its observation hours for a re-roll in the same statement, so the two can't diverge. func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) { row := q.db.QueryRow(ctx, insertChannelMessage, arg.ChannelID, @@ -2172,6 +2217,7 @@ func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMes arg.SenderName, arg.Content, arg.SentAt, + arg.Column6, ) var i InsertChannelMessageRow err := row.Scan(&i.ID, &i.ScopeName, &i.TransportCodesPresent) @@ -2180,6 +2226,7 @@ func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMes const insertObservation = `-- name: InsertObservation :one +WITH ins AS ( INSERT INTO packet_observations ( packet_hash, observer_id, @@ -2203,7 +2250,15 @@ INSERT INTO packet_observations ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ) ON CONFLICT (packet_hash, observer_id) DO NOTHING -RETURNING id, packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, path_bytes, rssi, snr, propagation_time_ms, radio_freq_mhz, spread_factor, bandwidth_khz, coding_rate, source_broker, payload_type, airtime_ms +RETURNING packet_hash +), bump AS ( + UPDATE packets p SET observation_count = p.observation_count + 1 + FROM ins WHERE p.packet_hash = ins.packet_hash + RETURNING p.observation_count +) +SELECT EXISTS (SELECT 1 FROM ins) AS inserted, + COALESCE((SELECT b.observation_count FROM bump b), + (SELECT pk.observation_count FROM packets pk WHERE pk.packet_hash = $1), 0)::bigint AS observation_count ` type InsertObservationParams struct { @@ -2227,10 +2282,16 @@ type InsertObservationParams struct { AirtimeMs *float32 `json:"airtime_ms"` } +type InsertObservationRow struct { + Inserted bool `json:"inserted"` + ObservationCount int64 `json:"observation_count"` +} + // ============================================================ // PACKET OBSERVATIONS // ============================================================ -func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationParams) (PacketObservation, error) { +// Bumps packets.observation_count only for a new row; a duplicate returns the current count. +func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationParams) (InsertObservationRow, error) { row := q.db.QueryRow(ctx, insertObservation, arg.PacketHash, arg.ObserverID, @@ -2251,28 +2312,8 @@ func (q *Queries) InsertObservation(ctx context.Context, arg InsertObservationPa arg.PayloadType, arg.AirtimeMs, ) - var i PacketObservation - err := row.Scan( - &i.ID, - &i.PacketHash, - &i.ObserverID, - &i.Iata, - &i.HeardAt, - &i.PathLengthByte, - &i.HashSize, - &i.HopCount, - &i.PathBytes, - &i.Rssi, - &i.Snr, - &i.PropagationTimeMs, - &i.RadioFreqMhz, - &i.SpreadFactor, - &i.BandwidthKhz, - &i.CodingRate, - &i.SourceBroker, - &i.PayloadType, - &i.AirtimeMs, - ) + var i InsertObservationRow + err := row.Scan(&i.Inserted, &i.ObservationCount) return i, err } @@ -2348,7 +2389,7 @@ func (q *Queries) ListAccounts(ctx context.Context) ([]Account, error) { const listAllChannelMessages = `-- name: ListAllChannelMessages :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -2462,7 +2503,7 @@ func (q *Queries) ListChannelCatalogues(ctx context.Context) ([]MeshmapperChanne const listChannelMessages = `-- name: ListChannelMessages :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -2547,7 +2588,7 @@ func (q *Queries) ListChannelMessages(ctx context.Context, arg ListChannelMessag const listChannelMessagesByHash = `-- name: ListChannelMessagesByHash :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count + p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -2873,7 +2914,7 @@ func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams const listMessagesAfterID = `-- name: ListMessagesAfterID :many SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, -(SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count +p.observation_count AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id JOIN packet_observations po ON po.packet_hash = cm.packet_hash @@ -3291,23 +3332,13 @@ COALESCE(CASE WHEN GREATEST(COALESCE(o.last_status_at, o.last_seen), o.last_seen) > NOW() - INTERVAL '5 minutes' THEN 'online' ELSE 'offline' END, 'offline')::text AS status, -COALESCE(( - SELECT po.iata - FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC - LIMIT 1 -), '')::text AS iata +COALESCE(o.last_iata, '')::text AS iata FROM observers o LEFT JOIN observer_brokers ob ON ob.observer_id = o.id LEFT JOIN observer_scopes os ON os.observer_id = o.id LEFT JOIN transport_scopes ts ON ts.id = os.scope_id WHERE - (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ( - SELECT po.iata FROM packet_observations po - WHERE po.observer_id = o.id - ORDER BY po.heard_at DESC LIMIT 1 - ) = ANY($1::bpchar[])) + (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR o.last_iata = ANY($1::bpchar[])) AND ($2 = '' OR o.observer_type = $2) AND ($3 = '' OR ob.broker_name = $3) AND ($4 = '' OR CASE @@ -3413,7 +3444,7 @@ SELECT p.last_heard_at, p.scope_id, ts.name AS scope_name, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, @@ -3545,7 +3576,7 @@ SELECT p.route_type, p.first_heard_at, p.last_heard_at, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, po.iata AS latest_observer_iata, @@ -3712,7 +3743,7 @@ SELECT sh.site_heard_at, sat.scan_saturated, sat.scan_floor, - (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = p.packet_hash) AS observation_count, + p.observation_count AS observation_count, -- sqlc loses LATERAL nullability; these scalar defaults are ignored when observer_id is NULL. po.observer_id AS latest_observer_id, o.display_name AS latest_observer_name, @@ -3958,43 +3989,27 @@ func (q *Queries) ListScopeCatalogues(ctx context.Context) ([]MeshmapperScopeCat const listTraceTags = `-- name: ListTraceTags :many -WITH tags AS ( - SELECT - p.trace_tag, - MIN(p.first_heard_at) AS first_heard_at, - MAX(p.last_heard_at) AS last_heard_at, - COUNT(*) AS packet_count, - MAX(p.parsed_payload->>'type') AS trace_type - FROM packets p - WHERE p.trace_tag IS NOT NULL - AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR p.trace_tag IN ( - SELECT ti.trace_tag FROM trace_iatas ti WHERE ti.iata = ANY($1::bpchar[]))) - AND ($2::text = '' OR p.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) - AND ($3::timestamptz IS NULL OR p.first_heard_at >= $3) - AND ($4::timestamptz IS NULL OR p.first_heard_at <= $4) - AND ($7::text = '' OR p.parsed_payload->>'type' = $7) - GROUP BY p.trace_tag - HAVING ($5::timestamptz IS NULL OR MAX(p.last_heard_at) < $5) - ORDER BY MAX(p.last_heard_at) DESC - LIMIT $6 -) SELECT encode(t.trace_tag, 'hex') AS trace_tag, - t.first_heard_at::timestamptz AS first_heard_at, - t.last_heard_at::timestamptz AS last_heard_at, + t.first_heard_at, + t.last_heard_at, t.packet_count, (SELECT COUNT(*) FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR ti.iata = ANY($1::bpchar[]))) AS iata_count, - t.trace_type::text AS trace_type, - (SELECT p3.parsed_payload - FROM packets p3 - WHERE p3.trace_tag = t.trace_tag - ORDER BY jsonb_array_length(p3.parsed_payload->'pathHashes') DESC - LIMIT 1) AS best_payload -FROM tags t + COALESCE(t.trace_type, '')::text AS trace_type, + t.best_payload +FROM trace_tags t +WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( + SELECT 1 FROM trace_iatas ti WHERE ti.trace_tag = t.trace_tag AND ti.iata = ANY($1::bpchar[]))) + AND ($2::text = '' OR t.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) + AND ($3::timestamptz IS NULL OR t.first_heard_at >= $3) + AND ($4::timestamptz IS NULL OR t.first_heard_at <= $4) + AND ($5::timestamptz IS NULL OR t.last_heard_at < $5) + AND ($7::text = '' OR t.trace_type = $7) ORDER BY t.last_heard_at DESC +LIMIT $6 ` type ListTraceTagsParams struct { @@ -4020,9 +4035,7 @@ type ListTraceTagsRow struct { // ============================================================ // TRACES // ============================================================ -// Returns distinct trace tags with summary info, ordered by most recent first. -// IATA membership comes from trace_iatas (joining observations here spilled the -// hash join). Per-tag details filled in only for the returned page. +// Tags, most recent first. Filters match the tag summary (see RecordTrace). func (q *Queries) ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([]ListTraceTagsRow, error) { rows, err := q.db.Query(ctx, listTraceTags, arg.Column1, @@ -4337,30 +4350,35 @@ func (q *Queries) ReconfirmRoutes(ctx context.Context, arg ReconfirmRoutesParams return count, err } -const refreshHourlyStats = `-- name: RefreshHourlyStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_hourly_iata_stats -` - -func (q *Queries) RefreshHourlyStats(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshHourlyStats) - return err -} - -const refreshObserverActivity = `-- name: RefreshObserverActivity :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_observer_activity_hourly -` - -func (q *Queries) RefreshObserverActivity(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshObserverActivity) - return err -} - -const refreshPayloadBreakdown = `-- name: RefreshPayloadBreakdown :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_payload_breakdown_by_iata -` - -func (q *Queries) RefreshPayloadBreakdown(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshPayloadBreakdown) +const recordTrace = `-- name: RecordTrace :exec +WITH iata AS ( + INSERT INTO trace_iatas (trace_tag, iata, last_heard) + VALUES ($1, $3, $2) + ON CONFLICT (trace_tag, iata) DO UPDATE SET + last_heard = EXCLUDED.last_heard + WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour' +) +INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, heard) +VALUES ($1, $2, $2, true) +ON CONFLICT (trace_tag) DO UPDATE SET + first_heard_at = CASE WHEN trace_tags.heard THEN LEAST(trace_tags.first_heard_at, EXCLUDED.first_heard_at) + ELSE EXCLUDED.first_heard_at END, + last_heard_at = CASE WHEN trace_tags.heard THEN GREATEST(trace_tags.last_heard_at, EXCLUDED.last_heard_at) + ELSE EXCLUDED.last_heard_at END, + heard = true +` + +type RecordTraceParams struct { + TraceTag []byte `json:"trace_tag"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + Iata string `json:"iata"` +} + +// One hearing of a TRACE packet: trace_iatas refreshes at most hourly and the tag's heard +// window widens (the first hearing replaces the packet's provisional times). Packet counts +// and payloads come from UpsertPacket. +func (q *Queries) RecordTrace(ctx context.Context, arg RecordTraceParams) error { + _, err := q.db.Exec(ctx, recordTrace, arg.TraceTag, arg.HeardAt, arg.Iata) return err } @@ -4373,42 +4391,6 @@ func (q *Queries) RefreshRadioPresets(ctx context.Context) error { return err } -const refreshTopAdvertisers = `-- name: RefreshTopAdvertisers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_advertisers_by_iata -` - -func (q *Queries) RefreshTopAdvertisers(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopAdvertisers) - return err -} - -const refreshTopNodes = `-- name: RefreshTopNodes :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_nodes_by_iata -` - -func (q *Queries) RefreshTopNodes(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopNodes) - return err -} - -const refreshTopObservers = `-- name: RefreshTopObservers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_observers_by_iata -` - -func (q *Queries) RefreshTopObservers(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopObservers) - return err -} - -const refreshTopTalkers = `-- name: RefreshTopTalkers :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_top_talkers_by_iata -` - -func (q *Queries) RefreshTopTalkers(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshTopTalkers) - return err -} - const resolveEndpointHashPairs = `-- name: ResolveEndpointHashPairs :many SELECT ns.iata, ns.prefix_1 AS hash, n.id AS node_id, n.name, n.latitude, n.longitude, n.public_key FROM node_short_ids ns @@ -5035,11 +5017,16 @@ func (q *Queries) TouchObserverBrokers(ctx context.Context, arg TouchObserverBro const touchObservers = `-- name: TouchObservers :exec UPDATE observers o SET last_seen = GREATEST(o.last_seen, v.seen), - observation_count = COALESCE(o.observation_count, 0) + v.delta + observation_count = COALESCE(o.observation_count, 0) + v.delta, + last_iata = CASE WHEN v.iata <> '' AND (o.last_iata_at IS NULL OR v.iata_at > o.last_iata_at) + THEN v.iata ELSE o.last_iata END, + last_iata_at = GREATEST(o.last_iata_at, v.iata_at) FROM ( SELECT unnest($1::uuid[]) AS id, unnest($2::timestamptz[]) AS seen, - unnest($3::int[]) AS delta + unnest($3::int[]) AS delta, + unnest($4::text[]) AS iata, + unnest($5::timestamptz[]) AS iata_at ) v WHERE o.id = v.id ` @@ -5048,12 +5035,20 @@ type TouchObserversParams struct { Column1 []uuid.UUID `json:"column_1"` Column2 []pgtype.Timestamptz `json:"column_2"` Column3 []int32 `json:"column_3"` + Column4 []string `json:"column_4"` + Column5 []pgtype.Timestamptz `json:"column_5"` } // Batched flush of coalesced presence bumps. GREATEST keeps a late flush // from regressing a newer write-through (e.g. a status update). func (q *Queries) TouchObservers(ctx context.Context, arg TouchObserversParams) error { - _, err := q.db.Exec(ctx, touchObservers, arg.Column1, arg.Column2, arg.Column3) + _, err := q.db.Exec(ctx, touchObservers, + arg.Column1, + arg.Column2, + arg.Column3, + arg.Column4, + arg.Column5, + ) return err } @@ -5160,18 +5155,16 @@ INSERT INTO channels (channel_hash, key_fingerprint, name, hashtag, is_hashtag, VALUES ($1, $2::bytea, $3, $4, $5, ($2 IS NOT NULL), NOW()) ON CONFLICT (channel_hash, key_fingerprint) DO UPDATE SET last_seen = NOW(), - name = COALESCE(EXCLUDED.name, channels.name), - message_count = CASE WHEN $6 THEN channels.message_count + 1 ELSE channels.message_count END + name = COALESCE(EXCLUDED.name, channels.name) RETURNING id, channel_hash, key_fingerprint, name, hashtag, is_hashtag, is_public, key_known, first_seen, last_seen, message_count ` type UpsertChannelParams struct { - ChannelHash []byte `json:"channel_hash"` - Column2 []byte `json:"column_2"` - Name *string `json:"name"` - Hashtag *string `json:"hashtag"` - IsHashtag *bool `json:"is_hashtag"` - MessageCount *int64 `json:"message_count"` + ChannelHash []byte `json:"channel_hash"` + Column2 []byte `json:"column_2"` + Name *string `json:"name"` + Hashtag *string `json:"hashtag"` + IsHashtag *bool `json:"is_hashtag"` } // ============================================================ @@ -5186,7 +5179,6 @@ func (q *Queries) UpsertChannel(ctx context.Context, arg UpsertChannelParams) (C arg.Name, arg.Hashtag, arg.IsHashtag, - arg.MessageCount, ) var i Channel err := row.Scan( @@ -5529,19 +5521,31 @@ func (q *Queries) UpsertNodeShortID(ctx context.Context, arg UpsertNodeShortIDPa const upsertObserver = `-- name: UpsertObserver :one -INSERT INTO observers (public_key, observer_type, last_seen) -VALUES ($1, 'unknown', NOW()) +INSERT INTO observers (public_key, observer_type, last_seen, last_iata, last_iata_at) +VALUES ($1, 'unknown', NOW(), NULLIF($2::text, ''), + CASE WHEN $2::text <> '' THEN $3::timestamptz END) ON CONFLICT (public_key) DO UPDATE SET last_seen = NOW(), - observation_count = observers.observation_count + 1 -RETURNING id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope + observation_count = observers.observation_count + 1, + last_iata = CASE WHEN EXCLUDED.last_iata IS NOT NULL + AND (observers.last_iata_at IS NULL OR EXCLUDED.last_iata_at > observers.last_iata_at) + THEN EXCLUDED.last_iata ELSE observers.last_iata END, + last_iata_at = GREATEST(observers.last_iata_at, EXCLUDED.last_iata_at) +RETURNING id, public_key, display_name, observer_type, software_version, hardware_model, firmware_version, firmware_build, radio_freq_mhz, radio_sf, radio_bw_khz, radio_cr, battery_level, uptime_seconds, status_metadata, last_status_at, first_seen, last_seen, observation_count, metadata, region_scope, last_iata, last_iata_at ` +type UpsertObserverParams struct { + PublicKey []byte `json:"public_key"` + Iata string `json:"iata"` + IataAt pgtype.Timestamptz `json:"iata_at"` +} + // ============================================================ // OBSERVERS // ============================================================ -func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observer, error) { - row := q.db.QueryRow(ctx, upsertObserver, publicKey) +// Empty iata (status/neighbors) leaves last_iata alone; otherwise the newest iata_at wins. +func (q *Queries) UpsertObserver(ctx context.Context, arg UpsertObserverParams) (Observer, error) { + row := q.db.QueryRow(ctx, upsertObserver, arg.PublicKey, arg.Iata, arg.IataAt) var i Observer err := row.Scan( &i.ID, @@ -5565,6 +5569,8 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe &i.ObservationCount, &i.Metadata, &i.RegionScope, + &i.LastIata, + &i.LastIataAt, ) return i, err } @@ -5611,6 +5617,7 @@ func (q *Queries) UpsertObserverScope(ctx context.Context, arg UpsertObserverSco const upsertPacket = `-- name: UpsertPacket :one +WITH up AS ( INSERT INTO packets ( packet_hash, payload_type, @@ -5633,8 +5640,25 @@ INSERT INTO packets ( ) ON CONFLICT (packet_hash) DO UPDATE SET last_heard_at = NOW() -RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, (xmax = 0) +RETURNING packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, trace_tag, scope_id, (xmax = 0) AS inserted +), trace AS ( + INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, packet_count, trace_type, scope_id, best_payload, best_path_len) + SELECT u.trace_tag, u.first_heard_at, u.last_heard_at, 1, u.parsed_payload->>'type', u.scope_id, u.parsed_payload, + CASE WHEN jsonb_typeof(u.parsed_payload->'pathHashes') = 'array' + THEN jsonb_array_length(u.parsed_payload->'pathHashes') ELSE 0 END + FROM up u + WHERE u.inserted AND u.trace_tag IS NOT NULL + ON CONFLICT (trace_tag) DO UPDATE SET + packet_count = trace_tags.packet_count + 1, + trace_type = GREATEST(trace_tags.trace_type, EXCLUDED.trace_type), + scope_id = COALESCE(trace_tags.scope_id, EXCLUDED.scope_id), + best_payload = CASE WHEN EXCLUDED.best_path_len > trace_tags.best_path_len + THEN EXCLUDED.best_payload ELSE trace_tags.best_payload END, + best_path_len = GREATEST(trace_tags.best_path_len, EXCLUDED.best_path_len) +) +SELECT packet_hash, payload_type, payload_version, route_type, transport_codes_present, region_code, sub_region_code, origin_pubkey, raw_payload, raw_header, parsed_payload, decrypted, channel_hash, first_heard_at, last_heard_at, inserted +FROM up ` type UpsertPacketParams struct { @@ -5676,6 +5700,8 @@ type UpsertPacketRow struct { // ============================================================ // PACKETS // ============================================================ +// A new TRACE packet adds itself to its tag summary in the same statement, so the summary +// can't miss a stored packet. 'TRACE' > 'PING'; the longest path keeps the best payload. func (q *Queries) UpsertPacket(ctx context.Context, arg UpsertPacketParams) (UpsertPacketRow, error) { row := q.db.QueryRow(ctx, upsertPacket, arg.PacketHash, @@ -5755,26 +5781,6 @@ func (q *Queries) UpsertRegion(ctx context.Context, arg UpsertRegionParams) (int return id, err } -const upsertTraceIATA = `-- name: UpsertTraceIATA :exec -INSERT INTO trace_iatas (trace_tag, iata, last_heard) -VALUES ($1, $2, $3) -ON CONFLICT (trace_tag, iata) DO UPDATE SET - last_heard = EXCLUDED.last_heard -WHERE EXCLUDED.last_heard > trace_iatas.last_heard + INTERVAL '1 hour' -` - -type UpsertTraceIATAParams struct { - TraceTag []byte `json:"trace_tag"` - Iata string `json:"iata"` - LastHeard pgtype.Timestamptz `json:"last_heard"` -} - -// Refreshes at most hourly so repeat hears don't churn the row. -func (q *Queries) UpsertTraceIATA(ctx context.Context, arg UpsertTraceIATAParams) error { - _, err := q.db.Exec(ctx, upsertTraceIATA, arg.TraceTag, arg.Iata, arg.LastHeard) - return err -} - const upsertTransportScope = `-- name: UpsertTransportScope :exec INSERT INTO transport_scopes (name, display_name, transport_key, key_fingerprint) diff --git a/db/sqlc/rollup.sql.go b/db/sqlc/rollup.sql.go new file mode 100644 index 00000000..db8ee8f0 --- /dev/null +++ b/db/sqlc/rollup.sql.go @@ -0,0 +1,501 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: rollup.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const deleteOldRollups = `-- name: DeleteOldRollups :exec +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour < $1::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour < $1::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour < $1::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour < $1::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour < $1::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour < $1::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour < $1::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour < $1::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour < $1::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour < $1::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour < $1::timestamptz), + l AS (DELETE FROM analytics_dirty_hours tl WHERE tl.hour < $1::timestamptz), + m AS (DELETE FROM analytics_rollup_hours tm WHERE tm.hour < $1::timestamptz RETURNING 1) +UPDATE analytics_raw_state SET coverage = coverage + 1 WHERE EXISTS (SELECT 1 FROM m) +` + +// Bumps coverage when hours go, so cached responses that included them are not reused. +func (q *Queries) DeleteOldRollups(ctx context.Context, cutoff pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, deleteOldRollups, cutoff) + return err +} + +const deleteRollupHour = `-- name: DeleteRollupHour :exec +WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour = $1::timestamptz), + b AS (DELETE FROM analytics_hourly_payload_breakdown tb WHERE tb.hour = $1::timestamptz), + c AS (DELETE FROM analytics_hourly_signal tc WHERE tc.hour = $1::timestamptz), + d AS (DELETE FROM analytics_hourly_paths td WHERE td.hour = $1::timestamptz), + e AS (DELETE FROM analytics_hourly_observer_activity te WHERE te.hour = $1::timestamptz), + f AS (DELETE FROM analytics_hourly_observer_identity tf WHERE tf.hour = $1::timestamptz), + g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour = $1::timestamptz), + h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour = $1::timestamptz), + i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour = $1::timestamptz), + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = $1::timestamptz) +DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = $1::timestamptz +` + +func (q *Queries) DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, deleteRollupHour, hour) + return err +} + +const deleteStaleDirtyHours = `-- name: DeleteStaleDirtyHours :exec +DELETE FROM analytics_dirty_hours d +WHERE EXISTS (SELECT 1 FROM analytics_rollup_hours h WHERE h.hour = d.hour AND h.status = 'partial') + OR d.hour <= (SELECT raw_deleted_before + INTERVAL '30 minutes' FROM analytics_raw_state) +` + +// Dirty hours that can no longer be re-rolled. +func (q *Queries) DeleteStaleDirtyHours(ctx context.Context) error { + _, err := q.db.Exec(ctx, deleteStaleDirtyHours) + return err +} + +const fillRollupObs = `-- name: FillRollupObs :exec +INSERT INTO rollup_obs (packet_hash, observer_id, iata, heard_at, path_length_byte, hash_size, hop_count, + path_len, rssi, snr, airtime_ms, payload_type, route_type, origin_pubkey, scope_id) +SELECT po.packet_hash, po.observer_id, po.iata, po.heard_at, po.path_length_byte, po.hash_size, po.hop_count, + COALESCE(octet_length(po.path_bytes), 0), po.rssi, po.snr, po.airtime_ms, po.payload_type, + p.route_type, p.origin_pubkey, p.scope_id +FROM packet_observations po +JOIN packets p ON p.packet_hash = po.packet_hash +WHERE po.heard_at >= $1::timestamptz AND po.heard_at < $1::timestamptz + INTERVAL '1 hour' +` + +func (q *Queries) FillRollupObs(ctx context.Context, dollar_1 pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, fillRollupObs, dollar_1) + return err +} + +const finishRollupHour = `-- name: FinishRollupHour :one +WITH prev AS ( + SELECT ph.content_hash FROM analytics_rollup_hours ph WHERE ph.hour = $1::timestamptz +), upd AS ( + UPDATE analytics_rollup_hours uh + SET status = 'complete', rolled_at = now(), content_hash = $2::text + WHERE uh.hour = $1::timestamptz +), undirty AS ( + DELETE FROM analytics_dirty_hours dh WHERE dh.hour = $1::timestamptz +), rev AS ( + UPDATE analytics_state SET revision = revision + 1 + WHERE (SELECT content_hash FROM prev) IS DISTINCT FROM $2::text + RETURNING revision +) +SELECT EXISTS (SELECT 1 FROM rev)::boolean AS changed +` + +type FinishRollupHourParams struct { + Hour pgtype.Timestamptz `json:"hour"` + ContentHash string `json:"content_hash"` +} + +// Marks the hour complete and bumps the revision only when its content changed. +func (q *Queries) FinishRollupHour(ctx context.Context, arg FinishRollupHourParams) (bool, error) { + row := q.db.QueryRow(ctx, finishRollupHour, arg.Hour, arg.ContentHash) + var changed bool + err := row.Scan(&changed) + return changed, err +} + +const getAnalyticsRevision = `-- name: GetAnalyticsRevision :one +SELECT (s.revision + r.coverage)::bigint FROM analytics_state s CROSS JOIN analytics_raw_state r +` + +// Both counters only grow, so their sum changes whenever rolled content or coverage does. +func (q *Queries) GetAnalyticsRevision(ctx context.Context) (int64, error) { + row := q.db.QueryRow(ctx, getAnalyticsRevision) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err +} + +const listDirtyRollupHours = `-- name: ListDirtyRollupHours :many +SELECT d.hour FROM analytics_dirty_hours d +JOIN analytics_rollup_hours h ON h.hour = d.hour AND h.status = 'complete' +CROSS JOIN analytics_raw_state r +WHERE r.raw_deleted_before IS NULL OR d.hour > r.raw_deleted_before + INTERVAL '30 minutes' +ORDER BY d.hour +LIMIT $1 +` + +// Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours +// wait: their first roll consumes the entry only if the entry predates its snapshot. +func (q *Queries) ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + rows, err := q.db.Query(ctx, listDirtyRollupHours, limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.Timestamptz{} + for rows.Next() { + var hour pgtype.Timestamptz + if err := rows.Scan(&hour); err != nil { + return nil, err + } + items = append(items, hour) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listMissingRollupHours = `-- name: ListMissingRollupHours :many +SELECT hour FROM analytics_rollup_hours +WHERE status = 'missing' +ORDER BY hour +LIMIT $1 +` + +func (q *Queries) ListMissingRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) { + rows, err := q.db.Query(ctx, listMissingRollupHours, limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.Timestamptz{} + for rows.Next() { + var hour pgtype.Timestamptz + if err := rows.Scan(&hour); err != nil { + return nil, err + } + items = append(items, hour) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const lockRollupHour = `-- name: LockRollupHour :one +SELECT CASE + WHEN r.raw_deleted_before IS NOT NULL AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + THEN 'partial' + ELSE h.status + END::text AS status +FROM analytics_rollup_hours h CROSS JOIN analytics_raw_state r +WHERE h.hour = $1 +FOR UPDATE OF h +` + +// Returns the hour's status, or 'partial' if raw deletion has reached it since registration. +func (q *Queries) LockRollupHour(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + row := q.db.QueryRow(ctx, lockRollupHour, hour) + var status string + err := row.Scan(&status) + return status, err +} + +const markPartialRollupHours = `-- name: MarkPartialRollupHours :exec +WITH marked AS ( + UPDATE analytics_rollup_hours h SET status = 'partial' + FROM analytics_raw_state r + WHERE h.status = 'missing' + AND r.raw_deleted_before IS NOT NULL + AND h.hour <= r.raw_deleted_before + INTERVAL '30 minutes' + RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked) +` + +// Hours whose raw rows cleanup already started deleting can never be rolled completely. +// Readers report the new status, so the revision moves too. +func (q *Queries) MarkPartialRollupHours(ctx context.Context) error { + _, err := q.db.Exec(ctx, markPartialRollupHours) + return err +} + +const oldestMissingRollupHour = `-- name: OldestMissingRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'missing' +` + +func (q *Queries) OldestMissingRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + row := q.db.QueryRow(ctx, oldestMissingRollupHour) + var column_1 pgtype.Timestamptz + err := row.Scan(&column_1) + return column_1, err +} + +const registerRollupHours = `-- name: RegisterRollupHours :exec +INSERT INTO analytics_rollup_hours (hour, status) +SELECT h, 'missing' +FROM (SELECT CASE WHEN MIN(heard_at) IS NOT NULL + THEN date_trunc('hour', GREATEST(MIN(heard_at), $1::timestamptz), 'UTC') END AS first_hour + FROM packet_observations) f, + generate_series(f.first_hour, date_trunc('hour', now() - INTERVAL '95 minutes', 'UTC'), INTERVAL '1 hour') h +WHERE f.first_hour IS NOT NULL +ON CONFLICT (hour) DO NOTHING +` + +// Adds a 'missing' row for every eligible hour since the earliest retained observation +// (and not before @since), zero-observation hours included. +// No observations yet means nothing to register (GREATEST would ignore the NULL MIN). +func (q *Queries) RegisterRollupHours(ctx context.Context, since pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, registerRollupHours, since) + return err +} + +const rollAdvertHearings = `-- name: RollAdvertHearings :exec +INSERT INTO analytics_hourly_advert_hearings (hour, iata, origin_pubkey, observations, last_heard, name, node_type) +SELECT $1::timestamptz, r.iata, r.origin_pubkey, r.n, r.last_heard, n.name, n.node_type +FROM (SELECT iata, origin_pubkey, count(*) AS n, max(heard_at) AS last_heard + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY iata, origin_pubkey) r +LEFT JOIN nodes n ON n.public_key = r.origin_pubkey +` + +func (q *Queries) RollAdvertHearings(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollAdvertHearings, hour) + return err +} + +const rollAdvertSets = `-- name: RollAdvertSets :exec +INSERT INTO analytics_hourly_advert_sets (hour, origin_pubkey, iatas, advert_packets, flood_packets, direct_packets) +SELECT $1::timestamptz, origin_pubkey, iatas, count(*), + count(*) FILTER (WHERE route_type IN (0, 1)), + count(*) FILTER (WHERE route_type IN (2, 3)) +FROM (SELECT origin_pubkey, route_type, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs + WHERE payload_type = 4 AND origin_pubkey IS NOT NULL + GROUP BY packet_hash, origin_pubkey, route_type) p +GROUP BY origin_pubkey, iatas +` + +// Each distinct ADVERT packet counts once, under the exact set of IATAs that heard it this hour. +func (q *Queries) RollAdvertSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollAdvertSets, hour) + return err +} + +const rollIATAObservations = `-- name: RollIATAObservations :exec +INSERT INTO analytics_hourly_iata_observations (hour, iata, observation_count) +SELECT $1::timestamptz, iata, count(*) FROM rollup_obs GROUP BY iata +` + +func (q *Queries) RollIATAObservations(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollIATAObservations, hour) + return err +} + +const rollObserverActivity = `-- name: RollObserverActivity :exec +INSERT INTO analytics_hourly_observer_activity (hour, observer_id, payload_type, observations, airtime_ms, airtime_n, + snr_sum, snr_n, snr_min, rssi_sum, rssi_n) +SELECT $1::timestamptz, observer_id, COALESCE(payload_type, -1)::smallint, + count(*), sum(airtime_ms)::real, count(airtime_ms), + sum(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + count(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), + min(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real, + sum(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint, + count(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)) +FROM rollup_obs +GROUP BY observer_id, COALESCE(payload_type, -1) +` + +func (q *Queries) RollObserverActivity(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollObserverActivity, hour) + return err +} + +const rollObserverIdentity = `-- name: RollObserverIdentity :exec +INSERT INTO analytics_hourly_observer_identity (hour, iata, observer_id, observation_count, display_name, observer_type) +SELECT $1::timestamptz, r.iata, r.observer_id, r.n, o.display_name, o.observer_type +FROM (SELECT iata, observer_id, count(*) AS n FROM rollup_obs GROUP BY iata, observer_id) r +LEFT JOIN observers o ON o.id = r.observer_id +` + +func (q *Queries) RollObserverIdentity(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollObserverIdentity, hour) + return err +} + +const rollPacketSets = `-- name: RollPacketSets :exec +INSERT INTO analytics_hourly_packet_sets (hour, iatas, packets) +SELECT $1::timestamptz, iatas, count(*) +FROM (SELECT array_agg(DISTINCT iata ORDER BY iata) AS iatas FROM rollup_obs GROUP BY packet_hash) p +GROUP BY iatas +` + +func (q *Queries) RollPacketSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollPacketSets, hour) + return err +} + +const rollPaths = `-- name: RollPaths :exec +INSERT INTO analytics_hourly_paths (hour, iata, category, hash_bytes, entries, receptions) +SELECT $1::timestamptz, c.iata, c.category, c.hash_bytes, c.entries, count(*) +FROM ( + SELECT k.iata, k.category, + CASE WHEN k.category = 0 THEN k.hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN k.category = 0 THEN k.hop_count ELSE 0 END::integer AS entries + FROM ( + SELECT iata, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND path_len = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM rollup_obs + ) k +) c +GROUP BY c.iata, c.category, c.hash_bytes, c.entries +` + +// category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). +func (q *Queries) RollPaths(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollPaths, hour) + return err +} + +const rollPayloadBreakdown = `-- name: RollPayloadBreakdown :exec +INSERT INTO analytics_hourly_payload_breakdown (hour, iata, payload_type, count) +SELECT $1::timestamptz, iata, payload_type, count(*) +FROM rollup_obs WHERE payload_type IS NOT NULL +GROUP BY iata, payload_type +` + +func (q *Queries) RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollPayloadBreakdown, hour) + return err +} + +const rollScopeSets = `-- name: RollScopeSets :exec +INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) +SELECT $1::timestamptz, iatas, scope_id, count(*) +FROM (SELECT scope_id, array_agg(DISTINCT iata ORDER BY iata) AS iatas + FROM rollup_obs WHERE scope_id IS NOT NULL + GROUP BY packet_hash, scope_id) p +GROUP BY iatas, scope_id +` + +func (q *Queries) RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollScopeSets, hour) + return err +} + +const rollSignal = `-- name: RollSignal :exec +INSERT INTO analytics_hourly_signal (hour, iata, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) +SELECT $1::timestamptz, b.iata, grouping(b.snr_bin, b.rssi_bin)::integer, + COALESCE(b.snr_bin, -1)::integer, COALESCE(b.rssi_bin, -1)::integer, + count(*), count(b.snr), COALESCE(sum(b.snr), 0)::double precision, + count(b.rssi), COALESCE(sum(b.rssi), 0)::double precision +FROM ( + SELECT s.iata, s.snr, s.rssi, + width_bucket(s.snr, -30, 30, 12) AS snr_bin, + width_bucket(s.rssi, -140, 0, 14) AS rssi_bin + FROM ( + SELECT ro.iata, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + AND ro.snr > '-Infinity'::real AND ro.snr < 'Infinity'::real + THEN ro.snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(ro.rssi, 0) = 0 AND COALESCE(ro.snr, 0) = 0) + THEN ro.rssi::double precision END AS rssi + FROM rollup_obs ro + ) s +) b +GROUP BY GROUPING SETS ((b.iata), (b.iata, b.snr_bin), (b.iata, b.rssi_bin)) +` + +// kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. +// rssi=0 AND snr=0 means "not reported". +func (q *Queries) RollSignal(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollSignal, hour) + return err +} + +const rollTalkerSets = `-- name: RollTalkerSets :exec +INSERT INTO analytics_hourly_talker_sets (hour, sender_name, iatas, messages, last_sent) +SELECT $1::timestamptz, sender_name, iatas, count(*), max(sent_at) +FROM (SELECT cm.sender_name, max(cm.sent_at) AS sent_at, array_agg(DISTINCT ro.iata ORDER BY ro.iata) AS iatas + FROM rollup_obs ro + JOIN channel_messages cm ON cm.packet_hash = ro.packet_hash + WHERE cm.sender_name IS NOT NULL + GROUP BY cm.id, cm.sender_name) m +GROUP BY sender_name, iatas +` + +func (q *Queries) RollTalkerSets(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollTalkerSets, hour) + return err +} + +const rollupContentHash = `-- name: RollupContentHash :one +SELECT md5(concat_ws('|', + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_iata_observations t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_payload_breakdown t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_signal t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_paths t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_activity t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_observer_identity t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_hearings t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_talker_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_packet_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = $1::timestamptz) +))::text AS content_hash +` + +// Order-independent fingerprint of every family row for the hour. +func (q *Queries) RollupContentHash(ctx context.Context, hour pgtype.Timestamptz) (string, error) { + row := q.db.QueryRow(ctx, rollupContentHash, hour) + var content_hash string + err := row.Scan(&content_hash) + return content_hash, err +} + +const rollupUnlock = `-- name: RollupUnlock :exec +SELECT pg_advisory_unlock(25189, 1) +` + +func (q *Queries) RollupUnlock(ctx context.Context) error { + _, err := q.db.Exec(ctx, rollupUnlock) + return err +} + +const setRollupHourPartial = `-- name: SetRollupHourPartial :exec +WITH marked AS ( + UPDATE analytics_rollup_hours SET status = 'partial' WHERE hour = $1 AND status = 'missing' RETURNING 1 +) +UPDATE analytics_state SET revision = revision + 1 WHERE EXISTS (SELECT 1 FROM marked) +` + +func (q *Queries) SetRollupHourPartial(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, setRollupHourPartial, hour) + return err +} + +const tryRollupLock = `-- name: TryRollupLock :one + +SELECT pg_try_advisory_lock(25189, 1) +` + +// ============================================================ +// HOURLY ANALYTICS ROLLUPS +// An hour [H, H+1h) is eligible once now() >= H + 95 min (ingest clamps heard_at to ±30 min). +// ============================================================ +func (q *Queries) TryRollupLock(ctx context.Context) (bool, error) { + row := q.db.QueryRow(ctx, tryRollupLock) + var pg_try_advisory_lock bool + err := row.Scan(&pg_try_advisory_lock) + return pg_try_advisory_lock, err +} diff --git a/db/sqlc/series.sql.go b/db/sqlc/series.sql.go new file mode 100644 index 00000000..cf4e6905 --- /dev/null +++ b/db/sqlc/series.sql.go @@ -0,0 +1,169 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: series.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getEarliestCompleteRollupHour = `-- name: GetEarliestCompleteRollupHour :one +SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete' +` + +func (q *Queries) GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + row := q.db.QueryRow(ctx, getEarliestCompleteRollupHour) + var column_1 pgtype.Timestamptz + err := row.Scan(&column_1) + return column_1, err +} + +const getStatsSeries = `-- name: GetStatsSeries :many +WITH hours AS ( + SELECT g.h::timestamptz AS hour, COALESCE(r.status, 'missing')::text AS status + FROM generate_series($1::timestamptz, $2::timestamptz, INTERVAL '1 hour') g(h) + LEFT JOIN analytics_rollup_hours r ON r.hour = g.h + WHERE g.h < $2::timestamptz +), complete AS ( + SELECT hour FROM hours WHERE status = 'complete' +), obs AS ( + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT o.hour, o.iata, o.observation_count FROM analytics_hourly_iata_observations o JOIN complete c ON c.hour = o.hour + WHERE cardinality($3::bpchar[]) > 0 AND o.iata = ANY($3::bpchar[]) +), obs_agg AS ( + SELECT hour, sum(observation_count)::bigint AS observations, count(DISTINCT iata)::bigint AS active_iatas + FROM obs GROUP BY GROUPING SETS ((), (hour)) +), pkt AS ( + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.packets FROM analytics_hourly_packet_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[] +), pkt_agg AS ( + SELECT hour, sum(packets)::bigint AS unique_packets FROM pkt GROUP BY GROUPING SETS ((), (hour)) +), obsr AS ( + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT i.hour, i.observer_id FROM analytics_hourly_observer_identity i JOIN complete c ON c.hour = i.hour + WHERE cardinality($3::bpchar[]) > 0 AND i.iata = ANY($3::bpchar[]) +), obsr_agg AS ( + SELECT hour, count(DISTINCT observer_id)::bigint AS active_observers FROM obsr GROUP BY GROUPING SETS ((), (hour)) +), scp AS ( + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.scope_id, s.packets FROM analytics_hourly_scope_sets s JOIN complete c ON c.hour = s.hour + WHERE cardinality($3::bpchar[]) > 0 AND s.iatas && $3::bpchar[] +), scp_agg AS ( + SELECT hour, sum(packets)::bigint AS scoped_packets, count(DISTINCT scope_id)::bigint AS active_scopes + FROM scp GROUP BY GROUPING SETS ((), (hour)) +), pth AS ( + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT p.hour, p.entries FROM analytics_hourly_paths p JOIN complete c ON c.hour = p.hour + WHERE p.category = 0 AND cardinality($3::bpchar[]) > 0 AND p.iata = ANY($3::bpchar[]) +), pth_agg AS ( + SELECT hour, max(entries)::integer AS max_path_entries FROM pth GROUP BY GROUPING SETS ((), (hour)) +), sig AS ( + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND COALESCE(cardinality($3::bpchar[]), 0) = 0 + UNION ALL + SELECT s.hour, s.snr_sum, s.snr_samples, s.rssi_sum, s.rssi_samples FROM analytics_hourly_signal s JOIN complete c ON c.hour = s.hour + WHERE s.kind = 3 AND cardinality($3::bpchar[]) > 0 AND s.iata = ANY($3::bpchar[]) +), sig_agg AS ( + SELECT hour, sum(snr_sum)::double precision AS snr_sum, sum(snr_samples)::bigint AS snr_samples, + sum(rssi_sum)::double precision AS rssi_sum, sum(rssi_samples)::bigint AS rssi_samples + FROM sig GROUP BY GROUPING SETS ((), (hour)) +), keys AS ( + SELECT hour, status FROM hours + UNION ALL + SELECT NULL::timestamptz, 'summary'::text +) +SELECT k.hour, k.status, + COALESCE(o.observations, 0)::bigint AS observations, + COALESCE(p.unique_packets, 0)::bigint AS unique_packets, + COALESCE(ob.active_observers, 0)::bigint AS active_observers, + COALESCE(o.active_iatas, 0)::bigint AS active_iatas, + COALESCE(s.scoped_packets, 0)::bigint AS scoped_packets, + COALESCE(s.active_scopes, 0)::bigint AS active_scopes, + COALESCE(pa.max_path_entries, 0)::integer AS max_path_entries, + COALESCE(sg.snr_sum, 0)::double precision AS snr_sum, + COALESCE(sg.snr_samples, 0)::bigint AS snr_samples, + COALESCE(sg.rssi_sum, 0)::double precision AS rssi_sum, + COALESCE(sg.rssi_samples, 0)::bigint AS rssi_samples +FROM keys k +LEFT JOIN obs_agg o ON o.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pkt_agg p ON p.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN obsr_agg ob ON ob.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN scp_agg s ON s.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN pth_agg pa ON pa.hour IS NOT DISTINCT FROM k.hour +LEFT JOIN sig_agg sg ON sg.hour IS NOT DISTINCT FROM k.hour +ORDER BY k.hour NULLS LAST +` + +type GetStatsSeriesParams struct { + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + Iatas []string `json:"iatas"` +} + +type GetStatsSeriesRow struct { + Hour pgtype.Timestamptz `json:"hour"` + Status string `json:"status"` + Observations int64 `json:"observations"` + UniquePackets int64 `json:"unique_packets"` + ActiveObservers int64 `json:"active_observers"` + ActiveIatas int64 `json:"active_iatas"` + ScopedPackets int64 `json:"scoped_packets"` + ActiveScopes int64 `json:"active_scopes"` + MaxPathEntries int32 `json:"max_path_entries"` + SnrSum float64 `json:"snr_sum"` + SnrSamples int64 `json:"snr_samples"` + RssiSum float64 `json:"rssi_sum"` + RssiSamples int64 `json:"rssi_samples"` +} + +// One row per hour in [since, until) plus a summary row (hour NULL) over its complete hours. +// Counts sum; observers, IATAs and scopes are distinct across the window; packets count +// once per hour heard. Each branch pair keeps "all IATAs" and "these IATAs" plans separate. +func (q *Queries) GetStatsSeries(ctx context.Context, arg GetStatsSeriesParams) ([]GetStatsSeriesRow, error) { + rows, err := q.db.Query(ctx, getStatsSeries, arg.Since, arg.Until, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetStatsSeriesRow{} + for rows.Next() { + var i GetStatsSeriesRow + if err := rows.Scan( + &i.Hour, + &i.Status, + &i.Observations, + &i.UniquePackets, + &i.ActiveObservers, + &i.ActiveIatas, + &i.ScopedPackets, + &i.ActiveScopes, + &i.MaxPathEntries, + &i.SnrSum, + &i.SnrSamples, + &i.RssiSum, + &i.RssiSamples, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/db/sqlc/signal.sql.go b/db/sqlc/signal.sql.go index 8a490490..abca971a 100644 --- a/db/sqlc/signal.sql.go +++ b/db/sqlc/signal.sql.go @@ -13,11 +13,11 @@ import ( const getSignalStats = `-- name: GetSignalStats :many WITH readings AS ( - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM mv_signal_stats_hourly + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND COALESCE(cardinality($3::bpchar[]), 0) = 0 UNION ALL - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM mv_signal_stats_hourly + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_hourly_signal WHERE hour >= $1::timestamptz AND hour < $2::timestamptz AND cardinality($3::bpchar[]) > 0 AND iata = ANY($3::bpchar[]) ) @@ -57,7 +57,7 @@ type GetSignalStatsRow struct { RssiAverage float64 `json:"rssi_average"` } -// Read compact hourly snapshots, never observations on an HTTP request. +// Read the hourly rollup, never observations on an HTTP request. // Weight averages by sample counts instead of averaging regional/hourly means. func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) { rows, err := q.db.Query(ctx, getSignalStats, arg.Since, arg.Until, arg.Iatas) @@ -88,12 +88,3 @@ func (q *Queries) GetSignalStats(ctx context.Context, arg GetSignalStatsParams) } return items, nil } - -const refreshSignalStats = `-- name: RefreshSignalStats :exec -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_signal_stats_hourly -` - -func (q *Queries) RefreshSignalStats(ctx context.Context) error { - _, err := q.db.Exec(ctx, refreshSignalStats) - return err -} diff --git a/db/sqlc_scratch/rollup_obs.sql b/db/sqlc_scratch/rollup_obs.sql new file mode 100644 index 00000000..660e325b --- /dev/null +++ b/db/sqlc_scratch/rollup_obs.sql @@ -0,0 +1,9 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- sqlc type-check stub only; db/rollup.go creates this as a TEMP table per rolled hour. +CREATE TABLE rollup_obs ( + packet_hash bytea NOT NULL, observer_id uuid NOT NULL, iata character(3) NOT NULL, + heard_at timestamptz NOT NULL, path_length_byte smallint NOT NULL, hash_size smallint NOT NULL, + hop_count smallint NOT NULL, path_len integer NOT NULL, rssi smallint, snr real, airtime_ms real, + payload_type smallint, route_type smallint NOT NULL, origin_pubkey bytea, scope_id integer); diff --git a/db/stats.go b/db/stats.go index 0d91df70..5bda12c1 100644 --- a/db/stats.go +++ b/db/stats.go @@ -11,32 +11,55 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) +// rollupSince snaps a window start to its UTC hour; zero means def before now. +func rollupSince(since time.Time, def time.Duration) pgtype.Timestamptz { + if since.IsZero() { + since = time.Now().Add(-def) + } + return ts(since.UTC().Truncate(time.Hour)) +} + +// optional maps the ” the rollup queries use for a missing name back to nil. +func optional(s string) *string { + if s == "" { + return nil + } + return &s +} + +func pgUUID(id pgtype.UUID) *uuid.UUID { + if !id.Valid { + return nil + } + u := uuid.UUID(id.Bytes) + return &u +} + +// GetStatsOverview summarises the 24 most recent hours that can have been rolled. func (s *Store) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { - row, err := s.q.GetStatsOverview(ctx, iatas) + until := time.Now().UTC().Add(-rollupDelay).Truncate(time.Hour).Add(time.Hour) + since := until.Add(-24 * time.Hour) + series, err := s.GetStatsSeries(ctx, since, until, iatas) if err != nil { return nil, err } return &api.StatsOverview{ - TotalPackets: row.TotalPackets, - TotalObservations: row.TotalObservations, - ActiveObservers: row.ActiveObservers, - ActiveIATAs: row.ActiveIatas, + TotalPackets: series.Summary.UniquePackets, + TotalObservations: series.Summary.Observations, + ActiveObservers: series.Summary.ActiveObservers, + ActiveIATAs: series.Summary.ActiveIATAs, WindowHours: 24, + Since: since.UnixMilli(), + Until: until.UnixMilli(), }, nil } func (s *Store) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { - if since.IsZero() { - since = time.Now().Add(-7 * 24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetHourlyStats(ctx, sqlc.GetHourlyStatsParams{ - Column1: iatas, - Column2: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - }) + rows, err := s.q.GetHourlyStats(ctx, sqlc.GetHourlyStatsParams{Iatas: iatas, Since: rollupSince(since, 7*24*time.Hour)}) if err != nil { return nil, err } @@ -46,60 +69,42 @@ func (s *Store) GetStatsObservations(ctx context.Context, iatas []string, since Hour: v.Hour.Time.UnixMilli(), IATA: v.Iata, ObservationCount: v.ObservationCount, - UniquePackets: v.UniquePackets, - ActiveObservers: v.ActiveObservers, }) } return points, nil } func (s *Store) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsPayloadBreakdown(ctx, sqlc.GetStatsPayloadBreakdownParams{ - Column1: iatas, - Column2: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - }) + rows, err := s.q.GetStatsPayloadBreakdown(ctx, sqlc.GetStatsPayloadBreakdownParams{Iatas: iatas, Since: rollupSince(since, 24*time.Hour)}) if err != nil { return nil, err } items := make([]api.PayloadBreakdownItem, 0, len(rows)) for _, v := range rows { - if v.PayloadType == nil { - continue - } items = append(items, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: v.PayloadType, + PayloadTypeName: api.PayloadTypeName(v.PayloadType), Count: v.Count, }) } return items, nil } -func (s *Store) GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { - rows, err := s.q.GetTopNodes(ctx, sqlc.GetTopNodesParams{ - Column1: iatas, - Limit: limit, - }) +func (s *Store) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { + rows, err := s.q.GetTopNodes(ctx, sqlc.GetTopNodesParams{Since: rollupSince(since, 7*24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } items := make([]api.TopNode, 0, len(rows)) for _, v := range rows { - var count int64 - if v.ObservationCount != nil { - count = *v.ObservationCount - } items = append(items, api.TopNode{ - NodeID: v.NodeID, - NodeName: v.Name, + NodeID: pgUUID(v.NodeID), + PublicKey: v.PublicKey, + NodeName: optional(v.Name), NodeType: v.NodeType, NodeTypeName: api.NodeTypeName(v.NodeType), IATA: v.Iata, - ObservationCount: count, + ObservationCount: v.ObservationCount, LastHeard: v.LastHeard.Time.UnixMilli(), }) } @@ -107,15 +112,7 @@ func (s *Store) GetStatsTopNodes(ctx context.Context, iatas []string, limit int3 } func (s *Store) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{ - Column1: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - Column2: iatas, - Limit: limit, - }) + rows, err := s.q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{Since: rollupSince(since, 24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } @@ -123,8 +120,8 @@ func (s *Store) GetStatsTopObservers(ctx context.Context, iatas []string, since for _, v := range rows { items = append(items, api.TopObserver{ ObserverID: v.ID, - DisplayName: v.DisplayName, - ObserverType: v.ObserverType, + DisplayName: optional(v.DisplayName), + ObserverType: optional(v.ObserverType), IATA: v.Iata, ObservationCount: v.ObservationCount, }) @@ -133,23 +130,16 @@ func (s *Store) GetStatsTopObservers(ctx context.Context, iatas []string, since } func (s *Store) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsTopAdvertisers(ctx, sqlc.GetStatsTopAdvertisersParams{ - Column1: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - Column2: iatas, - Limit: limit, - }) + rows, err := s.q.GetStatsTopAdvertisers(ctx, sqlc.GetStatsTopAdvertisersParams{Since: rollupSince(since, 24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } items := make([]api.TopAdvertiser, 0, len(rows)) for _, v := range rows { items = append(items, api.TopAdvertiser{ - NodeID: v.ID, - NodeName: v.Name, + NodeID: pgUUID(v.NodeID), + PublicKey: v.PublicKey, + NodeName: optional(v.Name), NodeType: v.NodeType, NodeTypeName: api.NodeTypeName(v.NodeType), IATA: v.Iata, @@ -196,26 +186,14 @@ func (s *Store) GetStatsClockDrift(ctx context.Context, iatas []string, limit in } func (s *Store) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { - if since.IsZero() { - since = time.Now().Add(-24 * time.Hour) - } - interval := time.Since(since) - rows, err := s.q.GetStatsTopTalkers(ctx, sqlc.GetStatsTopTalkersParams{ - Column1: pgtype.Interval{Microseconds: int64(interval.Hours()) * 3600 * 1e6, Valid: true}, - Column2: iatas, - Limit: limit, - }) + rows, err := s.q.GetStatsTopTalkers(ctx, sqlc.GetStatsTopTalkersParams{Since: rollupSince(since, 24*time.Hour), Iatas: iatas, RowLimit: limit}) if err != nil { return nil, err } items := make([]api.TopTalker, 0, len(rows)) for _, v := range rows { - var senderName string - if v.SenderName != nil { - senderName = *v.SenderName - } items = append(items, api.TopTalker{ - SenderName: senderName, + SenderName: v.SenderName, MessageCount: v.MessageCount, LastSent: v.LastSent.Time.UnixMilli(), }) @@ -233,9 +211,12 @@ func (s *Store) GetRadioPresets(ctx context.Context, preset string, iatas []stri } items := make([]api.RadioPreset, 0, len(rows)) for _, v := range rows { + if v.Iata == nil { + continue + } items = append(items, api.RadioPreset{ Preset: v.Preset, - IATA: v.Iata, + IATA: *v.Iata, SourceType: v.SourceType, Count: v.Count, }) @@ -243,8 +224,8 @@ func (s *Store) GetRadioPresets(ctx context.Context, preset string, iatas []stri return items, nil } -func (s *Store) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { - rows, err := s.q.GetScopeStats(ctx, iatas) +func (s *Store) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { + rows, err := s.q.GetScopeStats(ctx, sqlc.GetScopeStatsParams{Since: rollupSince(since, 7*24*time.Hour), Iatas: iatas}) if err != nil { return nil, err } @@ -276,34 +257,6 @@ func (s *Store) GetStatsNodeTypes(ctx context.Context, iatas []string) ([]api.No return result, nil } -func (s *Store) RefreshHourlyStats(ctx context.Context) error { - return s.q.RefreshHourlyStats(ctx) -} - -func (s *Store) RefreshTopNodes(ctx context.Context) error { - return s.q.RefreshTopNodes(ctx) -} - -func (s *Store) RefreshPayloadBreakdown(ctx context.Context) error { - return s.q.RefreshPayloadBreakdown(ctx) -} - -func (s *Store) RefreshTopTalkers(ctx context.Context) error { - return s.q.RefreshTopTalkers(ctx) -} - -func (s *Store) RefreshTopAdvertisers(ctx context.Context) error { - return s.q.RefreshTopAdvertisers(ctx) -} - -func (s *Store) RefreshTopObservers(ctx context.Context) error { - return s.q.RefreshTopObservers(ctx) -} - func (s *Store) RefreshRadioPresets(ctx context.Context) error { return s.q.RefreshRadioPresets(ctx) } - -func (s *Store) RefreshObserverActivity(ctx context.Context) error { - return s.q.RefreshObserverActivity(ctx) -} diff --git a/db/stats_series.go b/db/stats_series.go new file mode 100644 index 00000000..c905f559 --- /dev/null +++ b/db/stats_series.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func (s *Store) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { + // Read the revision first: a roll landing mid-request then shows up as a newer revision next time. + rev, err := s.q.GetAnalyticsRevision(ctx) + if err != nil { + return nil, err + } + earliest, err := s.q.GetEarliestCompleteRollupHour(ctx) + if err != nil { + return nil, err + } + rows, err := s.q.GetStatsSeries(ctx, sqlc.GetStatsSeriesParams{Since: ts(since), Until: ts(until), Iatas: iatas}) + if err != nil { + return nil, err + } + out := &api.StatsSeries{ + Since: since.UnixMilli(), + Until: until.UnixMilli(), + Revision: rev, + Hours: make([]api.StatsSeriesHour, 0, len(rows)), + } + if earliest.Valid { + ms := earliest.Time.UnixMilli() + out.EarliestComplete = &ms + } + for _, r := range rows { + v := api.StatsSeriesValues{ + Observations: r.Observations, + UniquePackets: r.UniquePackets, + ActiveObservers: r.ActiveObservers, + ActiveIATAs: r.ActiveIatas, + ScopedPackets: r.ScopedPackets, + ActiveScopes: r.ActiveScopes, + MaxPathEntries: r.MaxPathEntries, + SNRSum: r.SnrSum, + SNRSamples: r.SnrSamples, + RSSISum: r.RssiSum, + RSSISamples: r.RssiSamples, + } + if !r.Hour.Valid { + out.Summary = v + continue + } + h := api.StatsSeriesHour{Hour: r.Hour.Time.UnixMilli(), Status: r.Status} + if r.Status == "complete" { + h.Values = &v + out.CompleteHours++ + } + out.Hours = append(out.Hours, h) + } + return out, nil +} diff --git a/db/stats_series_integration_test.go b/db/stats_series_integration_test.go new file mode 100644 index 00000000..627d5e1f --- /dev/null +++ b/db/stats_series_integration_test.go @@ -0,0 +1,126 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// seriesOracle computes a summary straight from raw rows over [since, until). +const seriesOracle = ` +WITH po AS ( + SELECT po.*, date_trunc('hour', po.heard_at, 'UTC') AS hour, p.scope_id + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.heard_at >= $1 AND po.heard_at < $2 + AND (cardinality($3::bpchar[]) = 0 OR po.iata = ANY($3::bpchar[])) + AND date_trunc('hour', po.heard_at, 'UTC') <> ALL($4::timestamptz[]) +), sig AS ( + SELECT * FROM po WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) +) +SELECT (SELECT count(*) FROM po), + (SELECT count(*) FROM (SELECT DISTINCT hour, packet_hash FROM po) d), + (SELECT count(DISTINCT observer_id) FROM po), + (SELECT count(DISTINCT iata) FROM po), + (SELECT count(*) FROM (SELECT DISTINCT hour, packet_hash FROM po WHERE scope_id IS NOT NULL) d), + (SELECT count(DISTINCT scope_id) FROM po), + (SELECT COALESCE(sum(snr), 0)::double precision FROM sig), (SELECT count(snr) FROM sig), + (SELECT COALESCE(sum(rssi), 0)::double precision FROM sig), (SELECT count(rssi) FROM sig)` + +func TestStatsSeriesPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + h0 := rollupFixture(t, ctx, pool) + store := New(pool, 0, 0) + r, ok, err := store.BeginRollup(ctx) + if err != nil || !ok { + t.Fatalf("BeginRollup: ok=%v err=%v", ok, err) + } + if err := r.RegisterHours(ctx, h0); err != nil { + t.Fatal(err) + } + rollAll(t, ctx, r) + r.Close(ctx) + since, until := h0, h0.Add(3*time.Hour) + + for _, tc := range []struct { + name string + iatas []string + excluded []time.Time // hours not complete + maxPath int32 + }{ + {"all", nil, nil, 2}, + {"one IATA", []string{"YYZ"}, nil, 1}, + {"overlapping IATAs", []string{"YVR", "YYZ", "YVR"}, nil, 2}, + {"empty region", []string{""}, nil, 0}, + {"partial hour excluded", nil, []time.Time{h0.Add(time.Hour)}, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(ctx) + for _, h := range tc.excluded { + if _, err := tx.Exec(ctx, "UPDATE analytics_rollup_hours SET status = 'partial' WHERE hour = $1", h); err != nil { + t.Fatal(err) + } + } + got, err := (&Store{q: sqlc.New(tx)}).GetStatsSeries(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + var want api.StatsSeriesValues + excluded := tc.excluded + if excluded == nil { + excluded = []time.Time{} + } + iatas := tc.iatas + if iatas == nil { + iatas = []string{} + } + if err := tx.QueryRow(ctx, seriesOracle, since, until, iatas, excluded).Scan(&want.Observations, &want.UniquePackets, + &want.ActiveObservers, &want.ActiveIATAs, &want.ScopedPackets, &want.ActiveScopes, + &want.SNRSum, &want.SNRSamples, &want.RSSISum, &want.RSSISamples); err != nil { + t.Fatal(err) + } + want.MaxPathEntries = tc.maxPath + if got.Summary != want { + t.Errorf("summary\n got %+v\nwant %+v", got.Summary, want) + } + if len(got.Hours) != 3 || got.Since != since.UnixMilli() || got.Until != until.UnixMilli() || got.EarliestComplete == nil { + t.Fatalf("series shape: %+v", got) + } + var obs, packets int64 + for _, h := range got.Hours { + partial := false + for _, e := range tc.excluded { + partial = partial || e.UnixMilli() == h.Hour + } + if partial != (h.Values == nil) || (partial && h.Status != "partial") { + t.Errorf("hour %d status %s values %v", h.Hour, h.Status, h.Values) + } + if h.Values != nil { + obs += h.Values.Observations + packets += h.Values.UniquePackets + } + } + // Cards and sparklines must agree for additive metrics. + if obs != got.Summary.Observations || packets != got.Summary.UniquePackets { + t.Errorf("hours sum to %d observations / %d packets, summary %d / %d", obs, packets, got.Summary.Observations, got.Summary.UniquePackets) + } + if got.CompleteHours != 3-len(tc.excluded) { + t.Errorf("complete hours %d", got.CompleteHours) + } + }) + } + + // Hours never registered (or not yet eligible) are missing. + got, err := store.GetStatsSeries(ctx, h0.Add(-2*time.Hour), h0, nil) + if err != nil || len(got.Hours) != 2 || got.Hours[0].Status != "missing" || got.Hours[0].Values != nil || got.CompleteHours != 0 { + t.Errorf("unregistered hours: %+v %v", got, err) + } +} diff --git a/db/stats_test.go b/db/stats_test.go index e5843113..88e978b2 100644 --- a/db/stats_test.go +++ b/db/stats_test.go @@ -19,62 +19,63 @@ func TestGetStatsOverview(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetAnalyticsRevision(gomock.Any()).Return(int64(7), nil) + mock.EXPECT().GetEarliestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) + var params sqlc.GetStatsSeriesParams mock.EXPECT(). - GetStatsOverview(gomock.Any(), []string{"YVR"}). - Return(sqlc.GetStatsOverviewRow{ - TotalPackets: 100, - TotalObservations: 500, - ActiveObservers: 10, - ActiveIatas: 3, - }, nil) + GetStatsSeries(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, p sqlc.GetStatsSeriesParams) ([]sqlc.GetStatsSeriesRow, error) { + params = p + return []sqlc.GetStatsSeriesRow{{Status: "summary", Observations: 500, UniquePackets: 100, ActiveObservers: 10, ActiveIatas: 3}}, nil + }) store := &Store{q: mock} result, err := store.GetStatsOverview(context.Background(), []string{"YVR"}) if err != nil { t.Fatalf("unexpected error: %v", err) } - if result.TotalPackets != 100 { - t.Errorf("expected TotalPackets 100, got %d", result.TotalPackets) + if result.TotalPackets != 100 || result.TotalObservations != 500 || result.ActiveObservers != 10 || result.ActiveIATAs != 3 { + t.Errorf("unexpected summary mapping: %+v", result) + } + if result.WindowHours != 24 || result.Until-result.Since != (24*time.Hour).Milliseconds() { + t.Errorf("window %d..%d (%d hours)", result.Since, result.Until, result.WindowHours) } - if result.WindowHours != 24 { - t.Errorf("expected WindowHours 24, got %d", result.WindowHours) + // The window ends at the last hour that can have been rolled. + wantUntil := time.Now().UTC().Add(-rollupDelay).Truncate(time.Hour).Add(time.Hour) + if !params.Until.Time.Equal(wantUntil) || len(params.Iatas) != 1 { + t.Errorf("series params %+v, want until %s", params, wantUntil) } } -func TestGetStatsTopNodes_NilObservationCount(t *testing.T) { +func TestGetStatsTopNodes_MissingNodeRow(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) nodeID := uuid.MustParse("00000000-0000-0000-0000-000000000001") lastHeard := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} - name := "test-node" + since := time.Date(2026, 1, 1, 12, 34, 0, 0, time.UTC) mock.EXPECT(). GetTopNodes(gomock.Any(), sqlc.GetTopNodesParams{ - Column1: []string{"YVR"}, - Limit: 5, + Since: pgtype.Timestamptz{Time: since.Truncate(time.Hour), Valid: true}, + Iatas: []string{"YVR"}, + RowLimit: 5, }). - Return([]sqlc.MvTopNodesByIatum{ - { - NodeID: nodeID, - Name: &name, - NodeType: 1, - Iata: "YVR", - ObservationCount: nil, - LastHeard: lastHeard, - }, + Return([]sqlc.GetTopNodesRow{ + {NodeID: pgtype.UUID{Bytes: nodeID, Valid: true}, PublicKey: "aa", Name: "test-node", NodeType: 1, Iata: "YVR", ObservationCount: 9, LastHeard: lastHeard}, + {PublicKey: "bb", Iata: "YVR", ObservationCount: 3, LastHeard: lastHeard}, }, nil) store := &Store{q: mock} - items, err := store.GetStatsTopNodes(context.Background(), []string{"YVR"}, 5) + items, err := store.GetStatsTopNodes(context.Background(), []string{"YVR"}, since, 5) if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(items) != 1 { - t.Fatalf("expected 1 item, got %d", len(items)) + if len(items) != 2 || items[0].NodeID == nil || *items[0].NodeID != nodeID || *items[0].NodeName != "test-node" { + t.Fatalf("unexpected first node: %+v", items) } - if items[0].ObservationCount != 0 { - t.Errorf("expected ObservationCount 0 for nil, got %d", items[0].ObservationCount) + if items[1].NodeID != nil || items[1].NodeName != nil || items[1].PublicKey != "bb" { + t.Errorf("deleted node should keep its key and drop id/name: %+v", items[1]) } } @@ -91,8 +92,8 @@ func TestGetStatsTopObservers_IATATypeAssertion(t *testing.T) { Return([]sqlc.GetStatsTopObserversRow{ { ID: observerID, - DisplayName: &displayName, - ObserverType: &obsType, + DisplayName: displayName, + ObserverType: obsType, Iata: "YVR", ObservationCount: 42, }, @@ -126,8 +127,9 @@ func TestGetStatsTopAdvertisers_FloodDirectSplit(t *testing.T) { GetStatsTopAdvertisers(gomock.Any(), gomock.Any()). Return([]sqlc.GetStatsTopAdvertisersRow{ { - ID: nodeID, - Name: &name, + NodeID: pgtype.UUID{Bytes: nodeID, Valid: true}, + PublicKey: "cc", + Name: name, NodeType: 2, // repeater AdvertCount: 10, FloodAdvertCount: 7, @@ -227,13 +229,13 @@ func TestGetScopeStats(t *testing.T) { mock := mockdb.NewMockQuerier(ctrl) mock.EXPECT(). - GetScopeStats(gomock.Any(), []string{"YVR"}). + GetScopeStats(gomock.Any(), gomock.Any()). Return([]sqlc.GetScopeStatsRow{ {Name: "default", PacketCount: 100, ObserverCount: 5, NodeCount: 20}, }, nil) store := &Store{q: mock} - items, err := store.GetScopeStats(context.Background(), []string{"YVR"}) + items, err := store.GetScopeStats(context.Background(), []string{"YVR"}, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/db/store.go b/db/store.go index 2f9f35aa..c0f4c9f3 100644 --- a/db/store.go +++ b/db/store.go @@ -22,6 +22,7 @@ import ( // Store wraps the sqlc-generated Queries and implements both ingest.DB and api.Reader. type Store struct { q sqlc.Querier + pool *pgxpool.Pool // rollups need a dedicated connection; nil in mock-backed tests clockDriftThreshold time.Duration // see api.Node.ClockOutOfSync staleThreshold time.Duration // see api.NodeSummary.Stale } @@ -32,7 +33,7 @@ type Store struct { // staleThreshold is how long since last_seen before a node's Stale is reported true; see // internal/config.ResolvedConfig.NodeStaleThreshold. func New(pool *pgxpool.Pool, clockDriftThreshold, staleThreshold time.Duration) *Store { - return &Store{q: sqlc.New(pool), clockDriftThreshold: clockDriftThreshold, staleThreshold: staleThreshold} + return &Store{q: sqlc.New(pool), pool: pool, clockDriftThreshold: clockDriftThreshold, staleThreshold: staleThreshold} } func (s *Store) ResolvePathHashes(ctx context.Context, iata string, hashes [][]byte) (map[string][]api.ResolvedPathEntry, error) { diff --git a/db/trace_pagination_integration_test.go b/db/trace_pagination_integration_test.go index b0ca1405..991d74d4 100644 --- a/db/trace_pagination_integration_test.go +++ b/db/trace_pagination_integration_test.go @@ -5,6 +5,8 @@ package db import ( "context" + "encoding/hex" + "encoding/json" "os" "reflect" "strings" @@ -13,11 +15,11 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/jackc/pgx/v5" ) -// Trace tags, not individual packets, are the entities being paginated. All -// fixture writes use temporary tables and are rolled back on exit. +// Trace tags, not individual packets, are the entities being paginated. func TestTraceTagPaginationPostgres(t *testing.T) { dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") if dsn == "" { @@ -35,31 +37,10 @@ func TestTraceTagPaginationPostgres(t *testing.T) { t.Fatal(err) } defer tx.Rollback(context.Background()) - for _, table := range []string{"packets", "trace_iatas", "transport_scopes"} { - if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { - t.Fatal(err) - } - } - _, err = tx.Exec(ctx, ` -INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) VALUES + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO transport_scopes (id,name,transport_key,key_fingerprint) VALUES (1,'scope-one',decode(repeat('01',16),'hex'),decode(repeat('01',8),'hex')), - (2,'scope-two',decode(repeat('02',16),'hex'),decode(repeat('02',8),'hex')); -INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,trace_tag,scope_id,first_heard_at,last_heard_at,parsed_payload) -SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,1,'\x00','\x00',decode(tag,'hex'),scope, - '2026-01-01'::timestamptz+first*interval '1 second','2026-01-01'::timestamptz+last*interval '1 second',payload::jsonb -FROM (VALUES - (1,'aaaaaaaa',1,1,40,'{"type":"TRACE","pathHashes":["aa"],"snrValues":[1]}'), - (2,'aaaaaaaa',1,2,10,'{"type":"TRACE","pathHashes":["aa","bb"],"snrValues":[1,2]}'), - (3,'aaaaaaaa',2,3,25,'{"type":"PING","pathHashes":["aa","bb","cc"],"snrValues":[1,2,3]}'), - (4,'bbbbbbbb',1,4,30,'{"type":"TRACE","pathHashes":["bb","11","22","33"],"snrValues":[2,3,4,5]}'), - (5,'bbbbbbbb',2,5,5,'{"type":"PING","pathHashes":["bb"],"snrValues":[2]}'), - (6,'cccccccc',1,6,20,'{"type":"TRACE","pathHashes":["cc"],"snrValues":[3]}'), - (7,'dddddddd',2,7,15,'{"type":"PING","pathHashes":["dd"],"snrValues":[4]}'), - (8,NULL,1,8,50,'{"type":"TRACE","pathHashes":[],"snrValues":[]}') -) v(i,tag,scope,first,last,payload); -INSERT INTO trace_iatas (trace_tag,iata) VALUES - ('\xaaaaaaaa','YYZ'),('\xaaaaaaaa','YVR'),('\xbbbbbbbb','YYZ'),('\xcccccccc','YVR');`) - if err != nil { + (2,'scope-two',decode(repeat('02',16),'hex'),decode(repeat('02',8),'hex'))`); err != nil { t.Fatal(err) } store := &Store{q: sqlc.New(tx)} @@ -70,12 +51,47 @@ INSERT INTO trace_iatas (trace_tag,iata) VALUES } return anchor.Add(time.Duration(seconds) * time.Second) } + // Each packet is heard once when new and again (duplicate) at its last time. + for _, p := range []struct { + tag string + iata string + scope int32 + first, last int + kind string + hashes []string + }{ + {"aaaaaaaa", "YYZ", 1, 1, 40, "TRACE", []string{"aa"}}, + {"aaaaaaaa", "YVR", 1, 2, 10, "TRACE", []string{"aa", "bb"}}, + {"aaaaaaaa", "YVR", 2, 3, 25, "PING", []string{"aa", "bb", "cc"}}, + {"bbbbbbbb", "YYZ", 1, 4, 30, "TRACE", []string{"bb", "11", "22", "33"}}, + {"bbbbbbbb", "YYZ", 2, 5, 5, "PING", []string{"bb"}}, + {"cccccccc", "YVR", 1, 6, 20, "TRACE", []string{"cc"}}, + {"dddddddd", "YOW", 2, 7, 15, "PING", []string{"dd"}}, + } { + tag, _ := hex.DecodeString(p.tag) + snr := make([]float32, len(p.hashes)) + for i := range snr { + snr[i] = float32(i + int(p.tag[0]-'a') + 1) + } + payload, _ := json.Marshal(map[string]any{"type": p.kind, "pathHashes": p.hashes, "snrValues": snr}) + scope := p.scope + hash := append(append([]byte{}, tag...), byte(len(p.hashes)), byte(p.first)) + if _, err := store.UpsertPacket(ctx, ingest.UpsertPacketParams{PacketHash: hash, PayloadType: 9, RouteType: 1, + RawPayload: []byte{0}, RawHeader: []byte{0}, ParsedPayload: payload, ScopeID: &scope, TraceTag: tag}); err != nil { + t.Fatal(err) + } + for _, heard := range []int{p.first, p.last} { + if err := store.RecordTrace(ctx, ingest.TraceHearing{TraceTag: tag, IATA: p.iata, HeardAt: at(heard)}); err != nil { + t.Fatal(err) + } + } + } paths := map[string][]string{"a": {"aa", "bb", "cc"}, "b": {"bb", "11", "22", "33"}, "c": {"cc"}, "d": {"dd"}} snrs := map[string][]float32{"a": {1, 2, 3}, "b": {2, 3, 4, 5}, "c": {3}, "d": {4}} summary := func(tag string, first, last int, count, iatas int64, kind string) api.TraceTagSummary { return api.TraceTagSummary{TraceTag: strings.Repeat(tag, 8), FirstHeardAt: at(first).UnixMilli(), LastHeardAt: at(last).UnixMilli(), PacketCount: count, IATACount: iatas, TraceType: kind, PathHashes: paths[tag], SNRValues: snrs[tag]} } - a, b, c, d := summary("a", 1, 40, 3, 2, "TRACE"), summary("b", 4, 30, 2, 1, "TRACE"), summary("c", 6, 20, 1, 1, "TRACE"), summary("d", 7, 15, 1, 0, "PING") + a, b, c, d := summary("a", 1, 40, 3, 2, "TRACE"), summary("b", 4, 30, 2, 1, "TRACE"), summary("c", 6, 20, 1, 1, "TRACE"), summary("d", 7, 15, 1, 1, "PING") for _, tc := range []struct { name string iatas []string @@ -90,11 +106,12 @@ INSERT INTO trace_iatas (trace_tag,iata) VALUES {"region", []string{"YYZ"}, "", "", 0, 0, 35, []api.TraceTagSummary{b}}, {"multiple regions", []string{"YYZ", "YVR", "YYZ"}, "", "", 0, 0, 35, []api.TraceTagSummary{b, c}}, {"unknown region", []string{"ZZZ"}, "", "", 0, 0, 0, []api.TraceTagSummary{}}, + // Filters apply to the tag summary: its first scope, any-TRACE type, and first_heard_at. {"scope", nil, "scope-one", "", 0, 0, 30, []api.TraceTagSummary{c}}, - {"scope defines group", nil, "scope-two", "", 0, 0, 20, []api.TraceTagSummary{d, summary("b", 5, 5, 1, 1, "PING")}}, - {"type", nil, "", "TRACE", 0, 0, 35, []api.TraceTagSummary{summary("b", 4, 30, 1, 1, "TRACE"), c}}, - {"type defines group", nil, "", "PING", 0, 0, 30, []api.TraceTagSummary{summary("a", 3, 25, 1, 2, "PING"), d, summary("b", 5, 5, 1, 1, "PING")}}, - {"time window defines group", nil, "", "", 3, 6, 26, []api.TraceTagSummary{summary("a", 3, 25, 1, 2, "PING"), c}}, + {"first scope wins", nil, "scope-two", "", 0, 0, 0, []api.TraceTagSummary{d}}, + {"type", nil, "", "TRACE", 0, 0, 35, []api.TraceTagSummary{b, c}}, + {"mixed tag is TRACE", nil, "", "PING", 0, 0, 0, []api.TraceTagSummary{d}}, + {"time window on first heard", nil, "", "", 3, 6, 26, []api.TraceTagSummary{c}}, } { t.Run(tc.name, func(t *testing.T) { got, err := store.ListTraceTags(ctx, tc.iatas, tc.scope, tc.kind, at(tc.since), at(tc.until), at(tc.cursor), 20) @@ -135,4 +152,47 @@ INSERT INTO trace_iatas (trace_tag,iata) VALUES if total != 4 || len(seen) != 4 { t.Errorf("expected exactly four complete trace summaries") } + + if err := store.DeleteOldTraceTags(ctx, at(20)); err != nil { + t.Fatal(err) + } + if got, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, 20); err != nil || !reflect.DeepEqual(got, []api.TraceTagSummary{a, b, c}) { + t.Errorf("after cleanup got %+v, %v; want a, b, c", got, err) + } +} + +// Storing a TRACE packet records its contribution even if the hearing write that follows +// is lost; the next hearing replaces the packet's provisional times. +func TestTraceSummarySurvivesMissedHearingPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if err := RunMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + if _, err := pool.Exec(ctx, "INSERT INTO iata_codes (iata) VALUES ('YVR')"); err != nil { + t.Fatal(err) + } + store := New(pool, 0, 0) + tag := []byte{0xee, 0xee, 0xee, 0xee} + payload := json.RawMessage(`{"type":"TRACE","pathHashes":["aa","bb"],"snrValues":[1,2]}`) + upsert := func(hash byte) { + t.Helper() + if _, err := store.UpsertPacket(ctx, ingest.UpsertPacketParams{PacketHash: []byte{hash}, PayloadType: 9, RouteType: 1, + RawPayload: []byte{0}, RawHeader: []byte{0}, ParsedPayload: payload, TraceTag: tag}); err != nil { + t.Fatal(err) + } + } + upsert(1) // the RecordTrace that would follow never runs + upsert(1) // a broker duplicate must not count twice + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + if err != nil || len(rows) != 1 || rows[0].PacketCount != 1 || len(rows[0].PathHashes) != 2 || rows[0].TraceType != "TRACE" { + t.Fatalf("summary after a missed hearing: %+v %v", rows, err) + } + heard := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if err := store.RecordTrace(ctx, ingest.TraceHearing{TraceTag: tag, IATA: "YVR", HeardAt: heard}); err != nil { + t.Fatal(err) + } + rows, err = store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + if err != nil || len(rows) != 1 || rows[0].FirstHeardAt != heard.UnixMilli() || rows[0].LastHeardAt != heard.UnixMilli() || rows[0].IATACount != 1 { + t.Errorf("first hearing should replace provisional times: %+v %v", rows, err) + } } diff --git a/db/traces.go b/db/traces.go index d3cd6153..b33ebe3f 100644 --- a/db/traces.go +++ b/db/traces.go @@ -11,6 +11,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/jackc/pgx/v5/pgtype" ) @@ -20,14 +21,18 @@ type tracePayload struct { SNRValues []float32 `json:"snrValues"` } -func (s *Store) UpsertTraceIATA(ctx context.Context, traceTag []byte, iata string, heardAt time.Time) error { - return s.q.UpsertTraceIATA(ctx, sqlc.UpsertTraceIATAParams{ - TraceTag: traceTag, - Iata: iata, - LastHeard: pgtype.Timestamptz{Time: heardAt, Valid: true}, +func (s *Store) RecordTrace(ctx context.Context, h ingest.TraceHearing) error { + return s.q.RecordTrace(ctx, sqlc.RecordTraceParams{ + TraceTag: h.TraceTag, + Iata: h.IATA, + HeardAt: pgtype.Timestamptz{Time: h.HeardAt, Valid: true}, }) } +func (s *Store) DeleteOldTraceTags(ctx context.Context, cutoff time.Time) error { + return s.q.DeleteOldTraceTags(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) +} + func (s *Store) DeleteOldTraceIATAs(ctx context.Context, cutoff time.Time) error { return s.q.DeleteOldTraceIATAs(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } diff --git a/docs/docs.go b/docs/docs.go index 15dc6a7e..425a460f 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2280,6 +2280,7 @@ const docTemplate = `{ }, "/scopes/{name}": { "get": { + "description": "packetCount sums packets per hour across every retained rollup hour; iatas are the IATAs that heard the scope's packets. Observer and node counts are current memberships.", "produces": [ "application/json" ], @@ -2424,13 +2425,14 @@ const docTemplate = `{ }, "/stats/observations": { "get": { + "description": "Observation counts only; distinct packet and observer counts don't sum across IATAs, so read them from /stats/series.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Hourly observation time series", + "summary": "Hourly observation counts per IATA", "parameters": [ { "type": "string", @@ -2570,13 +2572,14 @@ const docTemplate = `{ }, "/stats/overview": { "get": { + "description": "Summarises the 24 most recent hours that can have been rolled (an hour rolls about 95 minutes after it closes); since/until report that window. Packets count once per hour heard. Use /stats/series for sparklines.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Network overview stats (last 24h)", + "summary": "Network overview stats (last 24 rolled hours)", "parameters": [ { "type": "string", @@ -2795,7 +2798,7 @@ const docTemplate = `{ }, "/stats/scopes": { "get": { - "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -2827,6 +2830,12 @@ const docTemplate = `{ "description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs", "name": "region", "in": "query" + }, + { + "type": "integer", + "description": "Start of packet window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" } ], "responses": { @@ -2854,6 +2863,78 @@ const docTemplate = `{ } } }, + "/stats/series": { + "get": { + "description": "One entry per UTC hour in [since, until) plus a summary over the complete hours. Both bounds round down to UTC hours and since is clamped to the rollup retention; the response reports the effective window. An hour is rolled about 95 minutes after it closes, so recent hours are \"missing\" until then; \"partial\" hours lost raw rows before they could be rolled and never get values. Counts (observations, packets) sum across hours, with a packet counted once in each hour it was heard; observers, IATAs and scopes are distinct across the whole window; averages are sum / samples. revision changes whenever any rolled hour changes.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Hourly network activity from the analytics rollups", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; the window is at most the rollup retention (default 90 days)", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/signal": { "get": { "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", @@ -2928,6 +3009,7 @@ const docTemplate = `{ }, "/stats/top-advertisers": { "get": { + "description": "Each advert counts once per hour heard, however many requested IATAs heard it. nodeId is null when the node row has been deleted.", "produces": [ "application/json" ], @@ -2990,13 +3072,14 @@ const docTemplate = `{ }, "/stats/top-nodes": { "get": { + "description": "nodeId is null when the node row has been deleted; publicKey always identifies it.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Top N nodes by observation count (from materialized view)", + "summary": "Top N nodes by advert hearings (last 7 days by default)", "parameters": [ { "type": "string", @@ -3016,6 +3099,12 @@ const docTemplate = `{ "name": "region", "in": "query" }, + { + "type": "integer", + "description": "Start of window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -3198,13 +3287,13 @@ const docTemplate = `{ }, { "type": "string", - "description": "Filter by transport scope name", + "description": "Filter by the tag's first transport scope name", "name": "scope", "in": "query" }, { "type": "string", - "description": "Filter by type: TRACE or PING (default: all)", + "description": "Filter by type: TRACE (any multi-hop packet) or PING (default: all)", "name": "type", "in": "query" }, @@ -3466,6 +3555,7 @@ const docTemplate = `{ "type": "integer" }, "messageCount": { + "description": "lifetime count; not reduced by retention", "type": "integer" }, "name": { @@ -3984,9 +4074,6 @@ const docTemplate = `{ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint": { "type": "object", "properties": { - "activeObservers": { - "type": "integer" - }, "hour": { "description": "epoch ms, start of the 1-hour bucket", "type": "integer" @@ -3996,9 +4083,6 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" - }, - "uniquePackets": { - "type": "integer" } } }, @@ -5194,10 +5278,19 @@ const docTemplate = `{ "activeObservers": { "type": "integer" }, + "since": { + "description": "epoch ms, start of the window", + "type": "integer" + }, "totalObservations": { "type": "integer" }, "totalPackets": { + "description": "distinct per hour (see StatsSeriesValues)", + "type": "integer" + }, + "until": { + "description": "epoch ms, exclusive end of the window", "type": "integer" }, "windowHours": { @@ -5206,6 +5299,109 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries": { + "type": "object", + "properties": { + "completeHours": { + "description": "hours contributing to summary", + "type": "integer" + }, + "earliestComplete": { + "description": "first complete hour held, epoch ms; null if none", + "type": "integer" + }, + "hours": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour" + } + }, + "revision": { + "description": "changes whenever rolled hours, their status or coverage change", + "type": "integer" + }, + "since": { + "description": "effective start, epoch ms on a UTC hour", + "type": "integer" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + }, + "until": { + "description": "effective exclusive end, epoch ms on a UTC hour", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the hour", + "type": "integer" + }, + "status": { + "description": "partial: raw rows deleted before it was rolled", + "type": "string", + "enum": [ + "complete", + "partial", + "missing" + ] + }, + "values": { + "description": "null unless complete", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + } + ] + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues": { + "type": "object", + "properties": { + "activeIatas": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeObservers": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeScopes": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "maxPathEntries": { + "type": "integer" + }, + "observations": { + "type": "integer" + }, + "rssiSamples": { + "type": "integer" + }, + "rssiSum": { + "description": "average = rssiSum / rssiSamples", + "type": "number" + }, + "scopedPackets": { + "type": "integer" + }, + "snrSamples": { + "type": "integer" + }, + "snrSum": { + "description": "average = snrSum / snrSamples", + "type": "number" + }, + "uniquePackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.TopAdvertiser": { "type": "object", "properties": { @@ -5227,6 +5423,7 @@ const docTemplate = `{ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -5237,6 +5434,10 @@ const docTemplate = `{ }, "nodeTypeName": { "type": "string" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, @@ -5251,6 +5452,7 @@ const docTemplate = `{ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -5264,6 +5466,10 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, diff --git a/docs/historical-stats.md b/docs/historical-stats.md new file mode 100644 index 00000000..72ad3ac8 --- /dev/null +++ b/docs/historical-stats.md @@ -0,0 +1,85 @@ +# Historical stats + +## The rule + +Historical stats never scan `packet_observations` per request or per refresh. They read +hourly rollup tables filled once per closed hour by one background task. Raw rows are read +only for per-entity drill-downs (a packet, a trace, a route), observer comparison and +sub-hour observer activity. + +## Rollup contract + +- **Hours.** `hour` is the UTC `date_trunc('hour', heard_at)` and leads every rollup table's + primary key. Rollup tables have no foreign keys: history outlives deleted observers, nodes + and IATAs. +- **Eligibility.** Hour `[H, H+1h)` can be rolled once `now() >= H + 95 min`. Ingest clamps + `heard_at` to ±30 minutes of server time, so nothing can still land in it. +- **Registration.** Every eligible hour since the earliest retained observation gets a row + in `analytics_rollup_hours`, including hours with no traffic. Status is `missing`, + `complete` or `partial`. +- **Rolling.** One process holds the advisory lock (`analytics_rollup` task, every 5 + minutes, plus a catch-up at boot). Each hour is rolled in one REPEATABLE READ transaction: + copy its observations (joined to packet dimensions) into a temp table, delete and reinsert + every family, hash the result, mark the hour complete. A serialization failure skips the + hour until the next pass. +- **Revision.** `analytics_state.revision` increases when an hour's content hash changes or + an hour turns partial; `analytics_raw_state.coverage` increases when retention deletes + rollup hours. Cache keys for rollup-backed endpoints include their sum (each counter only + grows), read through a 10 s memo. +- **Raw deletion.** Cleanup deletes packets older than `packets.retention`, but holds the + cutoff 35 minutes behind the oldest `missing` hour, for at most 24 hours (it logs a + warning past 6). Each batch advances `analytics_raw_state.raw_deleted_before` to the + newest deleted `last_heard_at`. Cleanup and the rollup never write the same row. +- **Partial hours.** A not-yet-complete hour starting at or before `raw_deleted_before + 30 min` + lost raw rows before it was rolled. It is marked `partial` and never rolled; readers + report it without values. +- **Re-rolls.** Hours are not re-rolled periodically. A message stored by channel backfill (at + startup, and when keys are imported at runtime) queues every hour its packet was heard in + `analytics_dirty_hours`, in the same statement as the message insert. Each pass re-rolls a + few complete ones while their raw rows remain. A queued hour still waiting for its first roll + keeps its entry unless that roll's snapshot already included it. +- **Retention.** Rollups are kept for `analytics.rollup_retention` (default 90 days), which + is also the longest window `/stats/series` accepts. + +## Combining hours + +| Metric | Across hours and IATAs | +| --- | --- | +| Observations, receptions, payload counts | Sum | +| Unique packets, adverts, messages, scoped packets | Sum of hourly IATA-set counts: each is counted once per hour it was heard, so one heard across an hour boundary counts twice (measured +1.1%) | +| Active observers, IATAs, scopes | Distinct across the window | +| Max path entries | Max | +| SNR / RSSI averages | Σ sum / Σ samples, never an average of averages | + +Windows snap to UTC hours on the server, and responses report the effective window. + +## IATA sets + +A packet heard in YVR and YYZ appears in both IATAs' per-IATA rows, so summing per-IATA +distinct counts overcounts (2.26× on production data). The `*_sets` tables instead group +each distinct packet, advert or message under the exact sorted set of IATAs that heard it +that hour (`iatas char(3)[]`, kept canonical by a CHECK). For any requested set S, the rows +where `iatas && S` count every item heard in S exactly once. This is exact for one IATA, a +region or all IATAs. + +## Raw or current, by design + +- Packet, trace and route detail, the packet list, observer comparison, and observer + activity below one hour. +- Current memberships: scope observer and node counts, node types, clock drift, radio presets. +- `/traces` reads `trace_tags`, a per-tag summary kept at ingest, not a rollup. A new packet + adds its count, type and payload in the statement that stores it; hearings set the times. + +## Adding a metric + +1. Add a family table to `db/migrations/001_baseline.sql` (on 2.x, a new migration): `hour` + first in the primary key, no foreign keys, and a set table with + `CHECK (iata_set_is_canonical(iatas))` plus a GIN index if it needs distinct counts. +2. Add a delete in `DeleteRollupHour`, a `Roll*` insert over `rollup_obs` in + `db/queries/rollup.sql`, a line in `RollupContentHash` and `DeleteOldRollups`, and the + call in `rollHour` (`db/rollup.go`). +3. Read it with an hour-bounded query, using the UNION-ALL IATA guard (see `series.sql`) for + per-IATA tables or `iatas && @iatas` for set tables. +4. Cache it with `cachedRollup` so its key carries the revision. +5. Add a raw-SQL oracle case to `db/rollup_integration_test.go`. +6. Run `sqlc generate`, regenerate mocks and swagger. diff --git a/docs/swagger.json b/docs/swagger.json index 25af0674..757caae0 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2278,6 +2278,7 @@ }, "/scopes/{name}": { "get": { + "description": "packetCount sums packets per hour across every retained rollup hour; iatas are the IATAs that heard the scope's packets. Observer and node counts are current memberships.", "produces": [ "application/json" ], @@ -2422,13 +2423,14 @@ }, "/stats/observations": { "get": { + "description": "Observation counts only; distinct packet and observer counts don't sum across IATAs, so read them from /stats/series.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Hourly observation time series", + "summary": "Hourly observation counts per IATA", "parameters": [ { "type": "string", @@ -2568,13 +2570,14 @@ }, "/stats/overview": { "get": { + "description": "Summarises the 24 most recent hours that can have been rolled (an hour rolls about 95 minutes after it closes); since/until report that window. Packets count once per hour heard. Use /stats/series for sparklines.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Network overview stats (last 24h)", + "summary": "Network overview stats (last 24 rolled hours)", "parameters": [ { "type": "string", @@ -2793,7 +2796,7 @@ }, "/stats/scopes": { "get": { - "description": "Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -2825,6 +2828,12 @@ "description": "Filter by region slug, expands to member IATAs; combined with explicit IATAs", "name": "region", "in": "query" + }, + { + "type": "integer", + "description": "Start of packet window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" } ], "responses": { @@ -2852,6 +2861,78 @@ } } }, + "/stats/series": { + "get": { + "description": "One entry per UTC hour in [since, until) plus a summary over the complete hours. Both bounds round down to UTC hours and since is clamped to the rollup retention; the response reports the effective window. An hour is rolled about 95 minutes after it closes, so recent hours are \"missing\" until then; \"partial\" hours lost raw rows before they could be rolled and never get values. Counts (observations, packets) sum across hours, with a packet counted once in each hour it was heard; observers, IATAs and scopes are distinct across the whole window; averages are sum / samples. revision changes whenever any rolled hour changes.", + "produces": [ + "application/json" + ], + "tags": [ + "Stats" + ], + "summary": "Hourly network activity from the analytics rollups", + "parameters": [ + { + "type": "integer", + "description": "Inclusive start, epoch milliseconds", + "name": "since", + "in": "query", + "required": true + }, + { + "type": "integer", + "description": "Exclusive end, epoch milliseconds; the window is at most the rollup retention (default 90 days)", + "name": "until", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Comma-separated reception IATA codes", + "name": "iatas", + "in": "query" + }, + { + "type": "integer", + "description": "Region ID, expands to member IATAs", + "name": "regionId", + "in": "query" + }, + { + "type": "string", + "description": "Region slug, expands to member IATAs", + "name": "region", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/stats/signal": { "get": { "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", @@ -2926,6 +3007,7 @@ }, "/stats/top-advertisers": { "get": { + "description": "Each advert counts once per hour heard, however many requested IATAs heard it. nodeId is null when the node row has been deleted.", "produces": [ "application/json" ], @@ -2988,13 +3070,14 @@ }, "/stats/top-nodes": { "get": { + "description": "nodeId is null when the node row has been deleted; publicKey always identifies it.", "produces": [ "application/json" ], "tags": [ "Stats" ], - "summary": "Top N nodes by observation count (from materialized view)", + "summary": "Top N nodes by advert hearings (last 7 days by default)", "parameters": [ { "type": "string", @@ -3014,6 +3097,12 @@ "name": "region", "in": "query" }, + { + "type": "integer", + "description": "Start of window epoch ms, rounded down to the hour (default 7 days ago)", + "name": "since", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -3196,13 +3285,13 @@ }, { "type": "string", - "description": "Filter by transport scope name", + "description": "Filter by the tag's first transport scope name", "name": "scope", "in": "query" }, { "type": "string", - "description": "Filter by type: TRACE or PING (default: all)", + "description": "Filter by type: TRACE (any multi-hop packet) or PING (default: all)", "name": "type", "in": "query" }, @@ -3464,6 +3553,7 @@ "type": "integer" }, "messageCount": { + "description": "lifetime count; not reduced by retention", "type": "integer" }, "name": { @@ -3982,9 +4072,6 @@ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint": { "type": "object", "properties": { - "activeObservers": { - "type": "integer" - }, "hour": { "description": "epoch ms, start of the 1-hour bucket", "type": "integer" @@ -3994,9 +4081,6 @@ }, "observationCount": { "type": "integer" - }, - "uniquePackets": { - "type": "integer" } } }, @@ -5192,10 +5276,19 @@ "activeObservers": { "type": "integer" }, + "since": { + "description": "epoch ms, start of the window", + "type": "integer" + }, "totalObservations": { "type": "integer" }, "totalPackets": { + "description": "distinct per hour (see StatsSeriesValues)", + "type": "integer" + }, + "until": { + "description": "epoch ms, exclusive end of the window", "type": "integer" }, "windowHours": { @@ -5204,6 +5297,109 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries": { + "type": "object", + "properties": { + "completeHours": { + "description": "hours contributing to summary", + "type": "integer" + }, + "earliestComplete": { + "description": "first complete hour held, epoch ms; null if none", + "type": "integer" + }, + "hours": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour" + } + }, + "revision": { + "description": "changes whenever rolled hours, their status or coverage change", + "type": "integer" + }, + "since": { + "description": "effective start, epoch ms on a UTC hour", + "type": "integer" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + }, + "until": { + "description": "effective exclusive end, epoch ms on a UTC hour", + "type": "integer" + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the hour", + "type": "integer" + }, + "status": { + "description": "partial: raw rows deleted before it was rolled", + "type": "string", + "enum": [ + "complete", + "partial", + "missing" + ] + }, + "values": { + "description": "null unless complete", + "allOf": [ + { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues" + } + ] + } + } + }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues": { + "type": "object", + "properties": { + "activeIatas": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeObservers": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "activeScopes": { + "description": "distinct across the hour or window", + "type": "integer" + }, + "maxPathEntries": { + "type": "integer" + }, + "observations": { + "type": "integer" + }, + "rssiSamples": { + "type": "integer" + }, + "rssiSum": { + "description": "average = rssiSum / rssiSamples", + "type": "number" + }, + "scopedPackets": { + "type": "integer" + }, + "snrSamples": { + "type": "integer" + }, + "snrSum": { + "description": "average = snrSum / snrSamples", + "type": "number" + }, + "uniquePackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.TopAdvertiser": { "type": "object", "properties": { @@ -5225,6 +5421,7 @@ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -5235,6 +5432,10 @@ }, "nodeTypeName": { "type": "string" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, @@ -5249,6 +5450,7 @@ "type": "integer" }, "nodeId": { + "description": "null once the node row has been deleted", "type": "string" }, "nodeName": { @@ -5262,6 +5464,10 @@ }, "observationCount": { "type": "integer" + }, + "publicKey": { + "description": "hex", + "type": "string" } } }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 3f601a76..b1431feb 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -113,6 +113,7 @@ definitions: description: epoch ms, time of most recent message type: integer messageCount: + description: lifetime count; not reduced by retention type: integer name: description: display name from config or nil @@ -489,8 +490,6 @@ definitions: type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObservationPoint: properties: - activeObservers: - type: integer hour: description: epoch ms, start of the 1-hour bucket type: integer @@ -498,8 +497,6 @@ definitions: type: string observationCount: type: integer - uniquePackets: - type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.Observer: properties: @@ -1318,14 +1315,92 @@ definitions: type: integer activeObservers: type: integer + since: + description: epoch ms, start of the window + type: integer totalObservations: type: integer totalPackets: + description: distinct per hour (see StatsSeriesValues) + type: integer + until: + description: epoch ms, exclusive end of the window type: integer windowHours: description: always 24 for now type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries: + properties: + completeHours: + description: hours contributing to summary + type: integer + earliestComplete: + description: first complete hour held, epoch ms; null if none + type: integer + hours: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour' + type: array + revision: + description: changes whenever rolled hours, their status or coverage change + type: integer + since: + description: effective start, epoch ms on a UTC hour + type: integer + summary: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues' + until: + description: effective exclusive end, epoch ms on a UTC hour + type: integer + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesHour: + properties: + hour: + description: epoch ms, start of the hour + type: integer + status: + description: 'partial: raw rows deleted before it was rolled' + enum: + - complete + - partial + - missing + type: string + values: + allOf: + - $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues' + description: null unless complete + type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeriesValues: + properties: + activeIatas: + description: distinct across the hour or window + type: integer + activeObservers: + description: distinct across the hour or window + type: integer + activeScopes: + description: distinct across the hour or window + type: integer + maxPathEntries: + type: integer + observations: + type: integer + rssiSamples: + type: integer + rssiSum: + description: average = rssiSum / rssiSamples + type: number + scopedPackets: + type: integer + snrSamples: + type: integer + snrSum: + description: average = snrSum / snrSamples + type: number + uniquePackets: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TopAdvertiser: properties: advertCount: @@ -1345,6 +1420,7 @@ definitions: description: epoch ms type: integer nodeId: + description: null once the node row has been deleted type: string nodeName: type: string @@ -1352,6 +1428,9 @@ definitions: type: integer nodeTypeName: type: string + publicKey: + description: hex + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TopNode: properties: @@ -1361,6 +1440,7 @@ definitions: description: epoch ms type: integer nodeId: + description: null once the node row has been deleted type: string nodeName: type: string @@ -1370,6 +1450,9 @@ definitions: type: string observationCount: type: integer + publicKey: + description: hex + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.TopObserver: properties: @@ -3060,6 +3143,9 @@ paths: - Scopes /scopes/{name}: get: + description: packetCount sums packets per hour across every retained rollup + hour; iatas are the IATAs that heard the scope's packets. Observer and node + counts are current memberships. parameters: - description: 'Scope name e.g. %23bc (URL-encoded #bc)' in: path @@ -3156,6 +3242,8 @@ paths: - Stats /stats/observations: get: + description: Observation counts only; distinct packet and observer counts don't + sum across IATAs, so read them from /stats/series. parameters: - description: Comma-separated IATA codes in: query @@ -3186,7 +3274,7 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: Hourly observation time series + summary: Hourly observation counts per IATA tags: - Stats /stats/observer-comparison: @@ -3256,6 +3344,9 @@ paths: - Stats /stats/overview: get: + description: Summarises the 24 most recent hours that can have been rolled (an + hour rolls about 95 minutes after it closes); since/until report that window. + Packets count once per hour heard. Use /stats/series for sparklines. parameters: - description: Comma-separated IATA codes in: query @@ -3280,7 +3371,7 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: Network overview stats (last 24h) + summary: Network overview stats (last 24 rolled hours) tags: - Stats /stats/paths: @@ -3412,12 +3503,13 @@ paths: - Stats /stats/scopes: get: - description: Counts each packet, observer and node once per scope. IATA filters - use retained observations for packets/observers and node IATA memberships - for nodes. Without filters, returns global totals for every stored scope. - With filters, lists only manual scopes configured for a matching region and - imported scopes whose current MeshMapper catalogue includes a matching IATA; - those remain listed with zero counts. An empty region returns an empty array. + description: 'Packets are those heard since the window start, each counted once + per hour however many of the requested IATAs heard it. Observers and nodes + are current memberships: observers filter by the IATA they last reported from, + nodes by their IATA memberships. With filters, lists only manual scopes configured + for a matching region and imported scopes whose current MeshMapper catalogue + includes a matching IATA; those remain listed with zero counts. An empty region + returns an empty array.' parameters: - description: Comma-separated IATA codes in: query @@ -3436,6 +3528,11 @@ paths: in: query name: region type: string + - description: Start of packet window epoch ms, rounded down to the hour (default + 7 days ago) + in: query + name: since + type: integer produces: - application/json responses: @@ -3456,6 +3553,63 @@ paths: summary: Scope statistics tags: - Stats + /stats/series: + get: + description: One entry per UTC hour in [since, until) plus a summary over the + complete hours. Both bounds round down to UTC hours and since is clamped to + the rollup retention; the response reports the effective window. An hour is + rolled about 95 minutes after it closes, so recent hours are "missing" until + then; "partial" hours lost raw rows before they could be rolled and never + get values. Counts (observations, packets) sum across hours, with a packet + counted once in each hour it was heard; observers, IATAs and scopes are distinct + across the whole window; averages are sum / samples. revision changes whenever + any rolled hour changes. + parameters: + - description: Inclusive start, epoch milliseconds + in: query + name: since + required: true + type: integer + - description: Exclusive end, epoch milliseconds; the window is at most the + rollup retention (default 90 days) + in: query + name: until + required: true + type: integer + - description: Comma-separated reception IATA codes + in: query + name: iatas + type: string + - description: Region ID, expands to member IATAs + in: query + name: regionId + type: integer + - description: Region slug, expands to member IATAs + in: query + name: region + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Hourly network activity from the analytics rollups + tags: + - Stats /stats/signal: get: description: Aggregates stored observer receptions in [since, until), at most @@ -3514,6 +3668,8 @@ paths: - Stats /stats/top-advertisers: get: + description: Each advert counts once per hour heard, however many requested + IATAs heard it. nodeId is null when the node row has been deleted. parameters: - description: Comma-separated IATA codes in: query @@ -3556,6 +3712,8 @@ paths: - Stats /stats/top-nodes: get: + description: nodeId is null when the node row has been deleted; publicKey always + identifies it. parameters: - description: Comma-separated IATA codes in: query @@ -3569,6 +3727,11 @@ paths: in: query name: region type: string + - description: Start of window epoch ms, rounded down to the hour (default 7 + days ago) + in: query + name: since + type: integer - description: Max results (default 10); must be positive, values above 200 are clamped in: query @@ -3589,7 +3752,7 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' - summary: Top N nodes by observation count (from materialized view) + summary: Top N nodes by advert hearings (last 7 days by default) tags: - Stats /stats/top-observers: @@ -3692,11 +3855,12 @@ paths: in: query name: regionId type: integer - - description: Filter by transport scope name + - description: Filter by the tag's first transport scope name in: query name: scope type: string - - description: 'Filter by type: TRACE or PING (default: all)' + - description: 'Filter by type: TRACE (any multi-hop packet) or PING (default: + all)' in: query name: type type: string diff --git a/go.mod b/go.mod index d309cbf9..0fc10fac 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/swaggo/http-swagger v1.3.4 github.com/swaggo/swag v1.16.6 go.uber.org/mock v0.6.0 + golang.org/x/sync v0.21.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -45,7 +46,6 @@ require ( golang.org/x/crypto v0.53.0 // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect - golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.39.0 // indirect golang.org/x/tools v0.47.0 // indirect diff --git a/internal/api/channels.go b/internal/api/channels.go index 00e9d85b..50ed66f4 100644 --- a/internal/api/channels.go +++ b/internal/api/channels.go @@ -101,5 +101,5 @@ type Channel struct { ChannelSummary Hashtag *string `json:"hashtag,omitempty"` // tag name without # prefix; non-nil only for hashtag channels KeyFingerprint *string `json:"keyFingerprint,omitempty"` // first 8 bytes of SHA256(key), hex-encoded - MessageCount int64 `json:"messageCount"` + MessageCount int64 `json:"messageCount"` // lifetime count; not reduced by retention } diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go index 3d7e3a28..f59c42c3 100644 --- a/internal/api/handlers/limits_test.go +++ b/internal/api/handlers/limits_test.go @@ -69,7 +69,7 @@ func TestListLimits(t *testing.T) { got = limit return nil, nil }, - getStatsTopNodes: func(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { + getStatsTopNodes: func(ctx context.Context, iatas []string, _ time.Time, limit int32) ([]api.TopNode, error) { calls++ got = limit return nil, nil @@ -113,7 +113,7 @@ func TestListLimits(t *testing.T) { router.Mount("/messages", MessagesRouter(reader)) router.Mount("/routes", RoutesRouter(reader)) router.Mount("/traces", TracesRouter(reader)) - router.Mount("/stats", StatsRouter(reader, nil)) + router.Mount("/stats", StatsRouter(reader, StatsOptions{})) for _, endpoint := range []struct { path string defaultLimit int32 diff --git a/internal/api/handlers/observer_comparison_test.go b/internal/api/handlers/observer_comparison_test.go index 0b11e6d1..78f7b539 100644 --- a/internal/api/handlers/observer_comparison_test.go +++ b/internal/api/handlers/observer_comparison_test.go @@ -28,7 +28,7 @@ func TestObserverComparisonRejectsInvalidQuery(t *testing.T) { } { t.Run(query, func(t *testing.T) { w := httptest.NewRecorder() - StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query, nil)) if w.Code != 400 { t.Fatalf("status %d; wanted validation error", w.Code) } @@ -42,7 +42,7 @@ func TestObserverComparisonHTTP(t *testing.T) { query := "observerA=" + a.String() + "&observerB=" + b.String() + "&since=0&until=1000" for _, suffix := range []string{"&since=2", "&observerA=" + b.String(), "&until=253402300800000", "&observerB=00000000-0000-0000-0000-000000000000"} { w := httptest.NewRecorder() - StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+suffix, nil)) if w.Code != 400 { t.Fatalf("invalid query returned %d", w.Code) } @@ -80,7 +80,7 @@ func TestObserverComparisonHTTP(t *testing.T) { }, } w := httptest.NewRecorder() - StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/observer-comparison?"+query+tc.suffix, nil)) if w.Code != tc.status { t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) } diff --git a/internal/api/handlers/paths.go b/internal/api/handlers/paths.go index 58f5acb4..d427d68e 100644 --- a/internal/api/handlers/paths.go +++ b/internal/api/handlers/paths.go @@ -32,7 +32,7 @@ import ( func getPathStats(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() - since, until, err := parseStatsWindow(r) + since, until, err := parseStatsWindow(r, rawStatsWindow) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return diff --git a/internal/api/handlers/paths_test.go b/internal/api/handlers/paths_test.go index 190fe69b..fb3b0eeb 100644 --- a/internal/api/handlers/paths_test.go +++ b/internal/api/handlers/paths_test.go @@ -28,7 +28,7 @@ func (s pathReader) GetPathStats(ctx context.Context, since, until time.Time, ia func TestPathsRejectInvalidWindow(t *testing.T) { for _, query := range []string{"", "since=0", "since=x&until=1", "since=-1&until=1", "since=1&until=1", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { w := httptest.NewRecorder() - StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/paths?"+query, nil)) if w.Code != 400 { t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) } @@ -66,7 +66,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) { return &api.PathStats{Until: until.UnixMilli(), Receptions: 4, Hashed: 1, Empty: 1, Trace: 1, Unclassified: 1}, tc.err }} w := httptest.NewRecorder() - StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=2592000000"+tc.query, nil)) if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { t.Fatalf("%d %s", w.Code, w.Body.String()) } @@ -86,7 +86,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) { return nil, ctx.Err() }} w := httptest.NewRecorder() - StatsRouter(r, nil).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) + StatsRouter(r, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/paths?since=0&until=1", nil).WithContext(ctx)) if !called || w.Body.Len() != 0 { t.Fatal("cancellation did not reach reader") } diff --git a/internal/api/handlers/regions.go b/internal/api/handlers/regions.go index 0b0f4008..f80df798 100644 --- a/internal/api/handlers/regions.go +++ b/internal/api/handlers/regions.go @@ -75,10 +75,13 @@ func getRegion(reader api.Reader) http.HandlerFunc { } } +// rawStatsWindow is the longest window for endpoints that still aggregate raw-era snapshots. +const rawStatsWindow = 30 * 24 * time.Hour + // parseStatsWindow rounds both endpoints down to UTC hours. Polls within an // hour share a cache key; the response reports the effective window. A valid // sub-hour request can contain no complete buckets and return an empty result. -func parseStatsWindow(r *http.Request) (time.Time, time.Time, error) { +func parseStatsWindow(r *http.Request, maxWindow time.Duration) (time.Time, time.Time, error) { q := r.URL.Query() for _, key := range []string{"since", "until"} { if len(q[key]) != 1 || q.Get(key) == "" { @@ -87,8 +90,8 @@ func parseStatsWindow(r *http.Request) (time.Time, time.Time, error) { } since, errSince := strconv.ParseInt(q.Get("since"), 10, 64) until, errUntil := strconv.ParseInt(q.Get("until"), 10, 64) - if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 || until-since > int64((30*24*time.Hour)/time.Millisecond) { - return time.Time{}, time.Time{}, fmt.Errorf("since and until must be epoch milliseconds between 0 and 253402300799999, with a positive window of at most 30 days") + if errSince != nil || errUntil != nil || since < 0 || until <= since || until > 253402300799999 || until-since > maxWindow.Milliseconds() { + return time.Time{}, time.Time{}, fmt.Errorf("since and until must be epoch milliseconds between 0 and 253402300799999, with a positive window of at most %d days", int(maxWindow/(24*time.Hour))) } return time.UnixMilli(since).UTC().Truncate(time.Hour), time.UnixMilli(until).UTC().Truncate(time.Hour), nil } diff --git a/internal/api/handlers/scope_stats_test.go b/internal/api/handlers/scope_stats_test.go index 111681dd..363b1c8b 100644 --- a/internal/api/handlers/scope_stats_test.go +++ b/internal/api/handlers/scope_stats_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "slices" "testing" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" @@ -71,7 +72,7 @@ func TestGetStatsScopes_Filters(t *testing.T) { } return &api.Region{IATAs: []string{"YVR", "YYJ"}}, nil }, - getScopeStats: func(_ context.Context, iatas []string) ([]api.ScopeStats, error) { + getScopeStats: func(_ context.Context, iatas []string, _ time.Time) ([]api.ScopeStats, error) { called = true if !slices.Equal(iatas, tc.want) { t.Fatalf("IATAs = %v, want %v", iatas, tc.want) @@ -104,7 +105,7 @@ func TestGetStatsScopes_Filters(t *testing.T) { func TestGetStatsScopes_ReaderError(t *testing.T) { w := httptest.NewRecorder() - getStatsScopes(stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) { + getStatsScopes(stubReader{getScopeStats: func(context.Context, []string, time.Time) ([]api.ScopeStats, error) { return nil, errors.New("database unavailable") }}, nil)(w, httptest.NewRequest(http.MethodGet, "/stats/scopes?iata=YVR", nil)) if w.Code != http.StatusInternalServerError { @@ -119,7 +120,7 @@ func testScopes(byIATA map[string][]string) *scopestore.ScopeStore { } func TestGetStatsScopes_FiltersToRegionMembers(t *testing.T) { - reader := stubReader{getScopeStats: func(context.Context, []string) ([]api.ScopeStats, error) { + reader := stubReader{getScopeStats: func(context.Context, []string, time.Time) ([]api.ScopeStats, error) { return []api.ScopeStats{{Name: "#elsewhere", PacketCount: 9}, {Name: "#on"}, {Name: "#yow", PacketCount: 4}}, nil }} members := testScopes(map[string][]string{"YOW": {"#yow", "#on"}}) diff --git a/internal/api/handlers/scopes.go b/internal/api/handlers/scopes.go index 41cca54d..b6a2ba9f 100644 --- a/internal/api/handlers/scopes.go +++ b/internal/api/handlers/scopes.go @@ -73,6 +73,7 @@ func listScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc { // getScope godoc // // @Summary Get scope detail by name +// @Description packetCount sums packets per hour across every retained rollup hour; iatas are the IATAs that heard the scope's packets. Observer and node counts are current memberships. // @Tags Scopes // @Produce json // @Param name path string true "Scope name e.g. %23bc (URL-encoded #bc)" diff --git a/internal/api/handlers/signal.go b/internal/api/handlers/signal.go index b1029a98..a5fd8648 100644 --- a/internal/api/handlers/signal.go +++ b/internal/api/handlers/signal.go @@ -32,7 +32,7 @@ import ( func getSignalStats(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() - since, until, err := parseStatsWindow(r) + since, until, err := parseStatsWindow(r, rawStatsWindow) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go index f9146fa8..578372b7 100644 --- a/internal/api/handlers/signal_test.go +++ b/internal/api/handlers/signal_test.go @@ -28,7 +28,7 @@ func (s signalReader) GetSignalStats(ctx context.Context, since, until time.Time func TestSignalRejectsInvalidWindow(t *testing.T) { for _, query := range []string{"", "since=0", "until=1", "since=x&until=1", "since=-1&until=1", "since=0&until=0", "since=2&until=1", "since=0&until=2592000001", "since=0&since=0&until=1", "since=0&until=1&until=2", "since=253402300799998&until=253402300800000"} { w := httptest.NewRecorder() - StatsRouter(nil, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) + StatsRouter(nil, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/signal?"+query, nil)) if w.Code != 400 { t.Fatalf("query %q: %d %s", query, w.Code, w.Body.String()) } @@ -70,7 +70,7 @@ func TestSignalFiltersAndErrors(t *testing.T) { return &api.SignalStats{Until: until.UnixMilli(), Receptions: 5, Hourly: []api.SignalHour{}}, tc.err }} w := httptest.NewRecorder() - StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=2592000000"+tc.query, nil)) if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { t.Fatalf("%d %s", w.Code, w.Body.String()) } @@ -93,7 +93,7 @@ func TestSignalCancellationReachesReader(t *testing.T) { return nil, ctx.Err() }} w := httptest.NewRecorder() - StatsRouter(reader, nil).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/signal?since=0&until=1", nil).WithContext(ctx)) if !called || w.Body.Len() != 0 { t.Fatalf("cancellation: called=%v body=%s", called, w.Body.String()) } @@ -101,12 +101,12 @@ func TestSignalCancellationReachesReader(t *testing.T) { func TestStatsWindowSnapsPollingTimes(t *testing.T) { for _, query := range []string{"since=123&until=86400123", "since=3599999&until=89999999"} { - since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?"+query, nil)) + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?"+query, nil), rawStatsWindow) if err != nil || since.UnixMilli() != 0 || until.UnixMilli() != 86400000 { t.Fatalf("%v %v %v", since, until, err) } } - since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?since=1&until=2", nil)) + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?since=1&until=2", nil), rawStatsWindow) if err != nil || !since.Equal(until) { t.Fatal("sub-hour normalization") } diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index ca064233..dc6216f9 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -4,6 +4,7 @@ package handlers import ( + "errors" "log/slog" "net/http" "slices" @@ -27,8 +28,29 @@ import ( // GET /stats/scopes → GetStatsScopes // // All endpoints accept an optional iata= filter (case-insensitive). -func StatsRouter(reader api.Reader, scopes ScopeMembership) http.Handler { +// StatsOptions configures StatsRouter. +type StatsOptions struct { + Scopes ScopeMembership // region-filtered scope lists; nil lists none + SeriesWindow time.Duration // longest /stats/series window, the rollup retention; 0 means 90 days +} + +// parseSince reads an optional epoch-ms since; zero means the endpoint's default window. +func parseSince(r *http.Request) (time.Time, error) { + p := r.URL.Query().Get("since") + if p == "" { + return time.Time{}, nil + } + ms, err := strconv.ParseInt(p, 10, 64) + if err != nil { + return time.Time{}, errors.New("since must be epoch milliseconds") + } + return time.UnixMilli(ms), nil +} + +func StatsRouter(reader api.Reader, opts StatsOptions) http.Handler { + scopes := opts.Scopes r := chi.NewRouter() + r.Get("/series", getStatsSeries(reader, opts.SeriesWindow)) r.Get("/overview", getStatsOverview(reader)) r.Get("/observations", getStatsObservations(reader)) r.Get("/signal", getSignalStats(reader)) @@ -48,7 +70,8 @@ func StatsRouter(reader api.Reader, scopes ScopeMembership) http.Handler { // getStatsOverview godoc // -// @Summary Network overview stats (last 24h) +// @Summary Network overview stats (last 24 rolled hours) +// @Description Summarises the 24 most recent hours that can have been rolled (an hour rolls about 95 minutes after it closes); since/until report that window. Packets count once per hour heard. Use /stats/series for sparklines. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -67,6 +90,9 @@ func getStatsOverview(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } overview, err := reader.GetStatsOverview(r.Context(), iatas) if err != nil { @@ -80,7 +106,8 @@ func getStatsOverview(reader api.Reader) http.HandlerFunc { // getStatsObservations godoc // -// @Summary Hourly observation time series +// @Summary Hourly observation counts per IATA +// @Description Observation counts only; distinct packet and observer counts don't sum across IATAs, so read them from /stats/series. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -100,15 +127,14 @@ func getStatsObservations(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } points, err := reader.GetStatsObservations(r.Context(), iatas, since) if err != nil { @@ -142,15 +168,14 @@ func getStatsPayloadBreakdown(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } breakdown, err := reader.GetStatsPayloadBreakdown(r.Context(), iatas, since) if err != nil { @@ -164,12 +189,14 @@ func getStatsPayloadBreakdown(reader api.Reader) http.HandlerFunc { // getStatsTopNodes godoc // -// @Summary Top N nodes by observation count (from materialized view) +// @Summary Top N nodes by advert hearings (last 7 days by default) +// @Description nodeId is null when the node row has been deleted; publicKey always identifies it. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs" +// @Param since query int false "Start of window epoch ms, rounded down to the hour (default 7 days ago)" // @Param limit query int false "Max results (default 10); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Success 200 {array} api.TopNode // @Failure 500 {object} handlers.APIError @@ -184,13 +211,21 @@ func getStatsTopNodes(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } limit, err := parseLimit(r, 10) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return } - nodes, err := reader.GetStatsTopNodes(r.Context(), iatas, limit) + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + nodes, err := reader.GetStatsTopNodes(r.Context(), iatas, since, limit) if err != nil { slog.Error("api: GetStatsTopNodes failed", "component", "api", "error", err) respondError(w, http.StatusInternalServerError, "internal server error") @@ -223,15 +258,14 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } limit, err := parseLimit(r, 10) if err != nil { @@ -251,6 +285,7 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { // getStatsTopAdvertisers godoc // // @Summary Top N nodes by distinct ADVERT packet count (last 24h by default) +// @Description Each advert counts once per hour heard, however many requested IATAs heard it. nodeId is null when the node row has been deleted. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" @@ -271,15 +306,14 @@ func getStatsTopAdvertisers(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } limit, err := parseLimit(r, 10) if err != nil { @@ -318,6 +352,9 @@ func getStatsClockDrift(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } limit, err := parseLimit(r, 10) if err != nil { @@ -357,15 +394,14 @@ func getStatsTopTalkers(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) - } - var since time.Time - if p := r.URL.Query().Get("since"); p != "" { - ms, err := strconv.ParseInt(p, 10, 64) - if err != nil { - respondError(w, http.StatusBadRequest, "since must be epoch milliseconds") - return + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything } - since = time.UnixMilli(ms) + } + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return } limit, err := parseLimit(r, 10) if err != nil { @@ -405,6 +441,9 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } presets, err := reader.GetRadioPresets(r.Context(), preset, iatas) if err != nil { @@ -418,13 +457,14 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // getStatsScopes godoc // // @Summary Scope statistics -// @Description Counts each packet, observer and node once per scope. IATA filters use retained observations for packets/observers and node IATA memberships for nodes. Without filters, returns global totals for every stored scope. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. +// @Description Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" // @Param iata query string false "Single IATA code; used when iatas is absent" // @Param regionId query int false "Filter by region ID, expands to member IATAs" // @Param region query string false "Filter by region slug, expands to member IATAs; combined with explicit IATAs" +// @Param since query int false "Start of packet window epoch ms, rounded down to the hour (default 7 days ago)" // @Success 200 {object} []api.ScopeStats // @Failure 400 {object} handlers.APIError // @Failure 500 {object} handlers.APIError @@ -444,7 +484,12 @@ func getStatsScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc return } } - stats, err := reader.GetScopeStats(r.Context(), iatas) + since, err := parseSince(r) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + stats, err := reader.GetScopeStats(r.Context(), iatas, since) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return @@ -481,6 +526,9 @@ func getStatsNodeTypes(reader api.Reader) http.HandlerFunc { return } iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } } result, err := reader.GetStatsNodeTypes(r.Context(), iatas) if err != nil { diff --git a/internal/api/handlers/stats_region_test.go b/internal/api/handlers/stats_region_test.go new file mode 100644 index 00000000..4ed483f4 --- /dev/null +++ b/internal/api/handlers/stats_region_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// iataRecorder records the IATA filter each stats read receives. +type iataRecorder struct { + stubReader + got *[]string +} + +func (r iataRecorder) rec(iatas []string) { *r.got = append([]string{}, iatas...) } + +func (r iataRecorder) GetStatsOverview(_ context.Context, iatas []string) (*api.StatsOverview, error) { + r.rec(iatas) + return &api.StatsOverview{}, nil +} +func (r iataRecorder) GetStatsObservations(_ context.Context, iatas []string, _ time.Time) ([]api.ObservationPoint, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsPayloadBreakdown(_ context.Context, iatas []string, _ time.Time) ([]api.PayloadBreakdownItem, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopNodes(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopNode, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopObservers(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopObserver, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopAdvertisers(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopAdvertiser, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsClockDrift(_ context.Context, iatas []string, _ int32) ([]api.ClockDriftEntry, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsTopTalkers(_ context.Context, iatas []string, _ time.Time, _ int32) ([]api.TopTalker, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetRadioPresets(_ context.Context, _ string, iatas []string) ([]api.RadioPreset, error) { + r.rec(iatas) + return nil, nil +} +func (r iataRecorder) GetStatsNodeTypes(_ context.Context, iatas []string) ([]api.NodeTypeCount, error) { + r.rec(iatas) + return nil, nil +} + +// A region with no IATAs must filter to nothing, not fall back to every IATA. +func TestStatsEmptyRegionMatchesNothing(t *testing.T) { + for _, path := range []string{"overview", "observations", "payload-breakdown", "top-nodes", "top-observers", + "top-advertisers", "clock-drift", "top-talkers", "radio-presets", "node-types"} { + for _, tc := range []struct { + region string + want []string + }{ + {"empty", []string{""}}, + {"west", []string{"YVR"}}, + } { + var got []string + reader := iataRecorder{got: &got, stubReader: stubReader{getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + if slug == "empty" { + return &api.Region{}, nil + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }}} + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", "/"+path+"?region="+tc.region, nil)) + if w.Code != 200 || !reflect.DeepEqual(got, tc.want) { + t.Errorf("/stats/%s?region=%s: status %d, iatas %q; want 200, %q", path, tc.region, w.Code, got, tc.want) + } + } + } +} diff --git a/internal/api/handlers/stats_series.go b/internal/api/handlers/stats_series.go new file mode 100644 index 00000000..564a7013 --- /dev/null +++ b/internal/api/handlers/stats_series.go @@ -0,0 +1,79 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "log/slog" + "net/http" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +const defaultSeriesWindow = 90 * 24 * time.Hour + +// getStatsSeries godoc +// +// @Summary Hourly network activity from the analytics rollups +// @Description One entry per UTC hour in [since, until) plus a summary over the complete hours. Both bounds round down to UTC hours and since is clamped to the rollup retention; the response reports the effective window. An hour is rolled about 95 minutes after it closes, so recent hours are "missing" until then; "partial" hours lost raw rows before they could be rolled and never get values. Counts (observations, packets) sum across hours, with a packet counted once in each hour it was heard; observers, IATAs and scopes are distinct across the whole window; averages are sum / samples. revision changes whenever any rolled hour changes. +// @Tags Stats +// @Produce json +// @Param since query int true "Inclusive start, epoch milliseconds" +// @Param until query int true "Exclusive end, epoch milliseconds; the window is at most the rollup retention (default 90 days)" +// @Param iatas query string false "Comma-separated reception IATA codes" +// @Param regionId query int false "Region ID, expands to member IATAs" +// @Param region query string false "Region slug, expands to member IATAs" +// @Success 200 {object} api.StatsSeries +// @Failure 400 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Failure 503 {object} handlers.APIError +// @Router /stats/series [get] +func getStatsSeries(reader api.Reader, maxWindow time.Duration) http.HandlerFunc { + if maxWindow <= 0 { + maxWindow = defaultSeriesWindow + } + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + since, until, err := parseStatsWindow(r, maxWindow) + if err != nil { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + if oldest := time.Now().UTC().Add(-maxWindow).Truncate(time.Hour); since.Before(oldest) { + since = oldest + } + if !since.Before(until) { + since = until + } + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + iatas := parseIATAs(r) + if q.Get("regionId") != "" || q.Get("region") != "" { + regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) + if err != nil { + respondError(w, http.StatusBadRequest, "region not found") + return + } + iatas = append(iatas, regionIATAs...) + if len(iatas) == 0 { + iatas = []string{""} // an empty region matches nothing, not everything + } + } + series, err := reader.GetStatsSeries(ctx, since, until, iatas) + switch { + case errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded): + respondError(w, http.StatusServiceUnavailable, "series query timed out; try a shorter time period") + case err != nil: + if r.Context().Err() != nil { + return + } + slog.Error("stats series failed", "error", err) + respondError(w, http.StatusInternalServerError, "internal server error") + default: + respond(w, http.StatusOK, series) + } + } +} diff --git a/internal/api/handlers/stats_series_test.go b/internal/api/handlers/stats_series_test.go new file mode 100644 index 00000000..3ba2624e --- /dev/null +++ b/internal/api/handlers/stats_series_test.go @@ -0,0 +1,76 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "errors" + "fmt" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +func TestGetStatsSeries(t *testing.T) { + now := time.Now().UTC() + until := now.Truncate(time.Hour) + since := until.Add(-48 * time.Hour) + window := fmt.Sprintf("since=%d&until=%d", since.UnixMilli()+1234, until.UnixMilli()+1234) + for _, tc := range []struct { + name, query string + maxWindow time.Duration + wantSince time.Time + iatas []string + err error + status int + }{ + {"global", window, 0, since, nil, nil, 200}, + {"IATAs", window + "&iatas=yvr,yyj", 0, since, []string{"YVR", "YYJ"}, nil, 200}, + {"region", window + "®ion=west", 0, since, []string{"YVR"}, nil, 200}, + {"empty region matches nothing", window + "®ion=empty", 0, since, []string{""}, nil, 200}, + {"missing region", window + "®ion=missing", 0, since, nil, nil, 400}, + {"missing until", fmt.Sprintf("since=%d", since.UnixMilli()), 0, since, nil, nil, 400}, + {"window over retention", window, 24 * time.Hour, since, nil, nil, 400}, + // The window fits, but its start predates what the rollups still hold. + {"since clamped to retention", fmt.Sprintf("since=%d&until=%d", until.Add(-50*time.Hour).UnixMilli(), until.Add(-2*time.Hour).UnixMilli()), + 48 * time.Hour, until.Add(-48 * time.Hour), nil, nil, 200}, + {"timeout", window, 0, since, nil, context.DeadlineExceeded, 503}, + {"database error", window, 0, since, nil, errors.New("private detail"), 500}, + } { + t.Run(tc.name, func(t *testing.T) { + reader := stubReader{ + getRegionBySlug: func(_ context.Context, slug string) (*api.Region, error) { + switch slug { + case "empty": + return &api.Region{}, nil + case "missing": + return nil, errors.New("not found") + } + return &api.Region{IATAs: []string{"YVR"}}, nil + }, + getStatsSeries: func(ctx context.Context, gotSince, gotUntil time.Time, iatas []string) (*api.StatsSeries, error) { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > 15*time.Second { + t.Error("missing bounded query deadline") + } + if !gotSince.Equal(tc.wantSince) || !reflect.DeepEqual(iatas, tc.iatas) { + t.Errorf("args %s %v; want %s %v", gotSince, iatas, tc.wantSince, tc.iatas) + } + if gotUntil.Sub(gotSince) > 48*time.Hour || gotUntil.Minute() != 0 { + t.Errorf("until %s not snapped within the window", gotUntil) + } + return &api.StatsSeries{Since: gotSince.UnixMilli()}, tc.err + }, + } + w := httptest.NewRecorder() + StatsRouter(reader, StatsOptions{SeriesWindow: tc.maxWindow}).ServeHTTP(w, httptest.NewRequest("GET", "/series?"+tc.query, nil)) + if w.Code != tc.status || strings.Contains(w.Body.String(), "private detail") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + }) + } +} diff --git a/internal/api/handlers/stats_test.go b/internal/api/handlers/stats_test.go index 86d9d9b9..d6019596 100644 --- a/internal/api/handlers/stats_test.go +++ b/internal/api/handlers/stats_test.go @@ -117,7 +117,7 @@ func TestGetStatsPayloadBreakdown_OK(t *testing.T) { func TestGetStatsTopNodes_OK(t *testing.T) { r := chi.NewRouter() r.Get("/stats/top-nodes", getStatsTopNodes(stubReader{ - getStatsTopNodes: func(_ context.Context, _ []string, _ int32) ([]api.TopNode, error) { + getStatsTopNodes: func(_ context.Context, _ []string, _ time.Time, _ int32) ([]api.TopNode, error) { return []api.TopNode{{IATA: "YVR", ObservationCount: 50}}, nil }, })) @@ -282,7 +282,7 @@ func TestGetStatsRadioPresets_OK(t *testing.T) { func TestGetStatsScopes_OK(t *testing.T) { r := chi.NewRouter() r.Get("/stats/scopes", getStatsScopes(stubReader{ - getScopeStats: func(_ context.Context, _ []string) ([]api.ScopeStats, error) { + getScopeStats: func(_ context.Context, _ []string, _ time.Time) ([]api.ScopeStats, error) { return []api.ScopeStats{{Name: "#bc", PacketCount: 100}}, nil }, }, nil)) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 3d3626cc..d4e148b6 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -16,6 +16,7 @@ import ( // Unset fields return zero values. Use it for both validation tests // (leave all fields nil) and happy path tests (set only what you need). type stubReader struct { + getStatsSeries func(context.Context, time.Time, time.Time, []string) (*api.StatsSeries, error) getRouteEvidence func(context.Context, string, string, api.RouteEvidenceQuery) (*api.RouteEvidence, error) getObserverComparison func(context.Context, uuid.UUID, uuid.UUID, time.Time, time.Time, []string) (*api.ObserverComparison, error) listIATAs func(ctx context.Context) ([]api.IATA, error) @@ -47,12 +48,12 @@ type stubReader struct { getStatsOverview func(ctx context.Context, iatas []string) (*api.StatsOverview, error) getStatsObservations func(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) getStatsPayloadBreakdown func(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) - getStatsTopNodes func(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) + getStatsTopNodes func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) getStatsTopObservers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) getStatsTopAdvertisers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) getStatsClockDrift func(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) getStatsTopTalkers func(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) - getScopeStats func(ctx context.Context, iatas []string) ([]api.ScopeStats, error) + getScopeStats func(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) getStatsNodeTypes func(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) getScopeNames func(ctx context.Context) ([]string, error) getScopeByName func(ctx context.Context, name string) (*api.ScopeDetail, error) @@ -70,6 +71,13 @@ func (s stubReader) GetSignalStats(context.Context, time.Time, time.Time, []stri return nil, nil } +func (s stubReader) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { + if s.getStatsSeries != nil { + return s.getStatsSeries(ctx, since, until, iatas) + } + return &api.StatsSeries{}, nil +} + func (s stubReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { return nil, nil } @@ -284,9 +292,9 @@ func (s stubReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string return nil, nil } -func (s stubReader) GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { +func (s stubReader) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { if s.getStatsTopNodes != nil { - return s.getStatsTopNodes(ctx, iatas, limit) + return s.getStatsTopNodes(ctx, iatas, since, limit) } return nil, nil } @@ -319,9 +327,9 @@ func (s stubReader) GetStatsTopTalkers(ctx context.Context, iatas []string, sinc return nil, nil } -func (s stubReader) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { +func (s stubReader) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { if s.getScopeStats != nil { - return s.getScopeStats(ctx, iatas) + return s.getScopeStats(ctx, iatas, since) } return nil, nil } diff --git a/internal/api/handlers/traces.go b/internal/api/handlers/traces.go index 7b2a7d47..54fd96dd 100644 --- a/internal/api/handlers/traces.go +++ b/internal/api/handlers/traces.go @@ -32,8 +32,8 @@ func TracesRouter(reader api.Reader) http.Handler { // @Param iatas query string false "Filter by IATA code(s), comma-separated" // @Param region query string false "Filter by region slug" // @Param regionId query int false "Filter by region ID" -// @Param scope query string false "Filter by transport scope name" -// @Param type query string false "Filter by type: TRACE or PING (default: all)" +// @Param scope query string false "Filter by the tag's first transport scope name" +// @Param type query string false "Filter by type: TRACE (any multi-hop packet) or PING (default: all)" // @Param since query int false "Filter by first_heard_at >= since (epoch ms)" // @Param until query int false "Filter by first_heard_at <= until (epoch ms)" // @Param cursor query int false "last_heard_at epoch ms of last item for pagination" diff --git a/internal/api/reader.go b/internal/api/reader.go index ba6c7cd5..e74a2534 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -163,6 +163,9 @@ type Reader interface { // since defines the start of the window; pass zero time for default (last 7 days). GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]ObservationPoint, error) + // GetStatsSeries returns hourly rollup metrics for [since, until), both on UTC hours. + GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*StatsSeries, error) + // GetSignalStats aggregates retained reception readings in [since, until). GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*SignalStats, error) @@ -176,7 +179,7 @@ type Reader interface { // GetStatsTopNodes returns the top N nodes by observation count. // Pass nil for iatas to return stats across all IATAs. - GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]TopNode, error) + GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]TopNode, error) // GetStatsTopObservers returns the top N observers by observation count. // Pass nil for iatas to return stats across all IATAs. @@ -201,7 +204,7 @@ type Reader interface { // GetScopeStats returns aggregate packet, observer and node counts per transport scope. // Pass empty iatas for global totals. IATAs must be uppercase. - GetScopeStats(ctx context.Context, iatas []string) ([]ScopeStats, error) + GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]ScopeStats, error) // GetStatsNodeTypes returns node counts grouped by type, optionally filtered by IATA. GetStatsNodeTypes(ctx context.Context, iatas []string) ([]NodeTypeCount, error) diff --git a/internal/api/router/options.go b/internal/api/router/options.go index 0d9076e1..e838834e 100644 --- a/internal/api/router/options.go +++ b/internal/api/router/options.go @@ -5,6 +5,7 @@ package router import ( "net/http" + "time" "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" "github.com/MeshCore-Beacon/beacon-server/internal/config" @@ -21,6 +22,7 @@ type Options struct { Auth config.AuthConfig RateLimit config.ResolvedRateLimitConfig Scopes handlers.ScopeMembership // region-filtered scope lists; nil lists none + RollupRetention time.Duration // longest historical stats window // AdminRoutes mounts operator-only subrouters at literal paths such as // "/accounts". Each feature owns its dependencies, methods and handlers. // The router applies authentication to every path and method in this group. diff --git a/internal/api/router/router.go b/internal/api/router/router.go index b8a5ceab..d6bbbc97 100644 --- a/internal/api/router/router.go +++ b/internal/api/router/router.go @@ -108,7 +108,7 @@ func New(h *hub.Hub, reader api.Reader, workers []*ingest.Worker, opts Options) r.Mount("/regions", handlers.RegionsRouter(reader)) r.Mount("/routes", handlers.RoutesRouter(reader)) r.Mount("/scopes", handlers.ScopesRouter(reader, opts.Scopes)) - r.Mount("/stats", handlers.StatsRouter(reader, opts.Scopes)) + r.Mount("/stats", handlers.StatsRouter(reader, handlers.StatsOptions{Scopes: opts.Scopes, SeriesWindow: opts.RollupRetention})) r.Mount("/traces", handlers.TracesRouter(reader)) }) diff --git a/internal/api/stats.go b/internal/api/stats.go index 1485639d..bb1e8eb3 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -29,21 +29,58 @@ type RadioPreset struct { } // StatsOverview is the top-level network summary for the overview endpoint. +// Covers the last 24 hours that can have been rolled; hours still missing contribute nothing. type StatsOverview struct { - TotalPackets int64 `json:"totalPackets"` + TotalPackets int64 `json:"totalPackets"` // distinct per hour (see StatsSeriesValues) TotalObservations int64 `json:"totalObservations"` ActiveObservers int64 `json:"activeObservers"` ActiveIATAs int64 `json:"activeIatas"` WindowHours int `json:"windowHours"` // always 24 for now + Since int64 `json:"since"` // epoch ms, start of the window + Until int64 `json:"until"` // epoch ms, exclusive end of the window +} + +// StatsSeries is hourly network activity read from the analytics rollups. Only complete +// hours carry values; the summary combines them (see docs/historical-stats.md). +type StatsSeries struct { + Since int64 `json:"since"` // effective start, epoch ms on a UTC hour + Until int64 `json:"until"` // effective exclusive end, epoch ms on a UTC hour + Revision int64 `json:"revision"` // changes whenever rolled hours, their status or coverage change + EarliestComplete *int64 `json:"earliestComplete"` // first complete hour held, epoch ms; null if none + CompleteHours int `json:"completeHours"` // hours contributing to summary + Hours []StatsSeriesHour `json:"hours"` + Summary StatsSeriesValues `json:"summary"` +} + +// StatsSeriesHour is one UTC hour of a StatsSeries. +type StatsSeriesHour struct { + Hour int64 `json:"hour"` // epoch ms, start of the hour + Status string `json:"status" enums:"complete,partial,missing"` // partial: raw rows deleted before it was rolled + Values *StatsSeriesValues `json:"values"` // null unless complete +} + +// StatsSeriesValues are the card metrics for an hour or a whole series. Packet counts are +// distinct per hour, so a packet heard across an hour boundary counts in both hours. +type StatsSeriesValues struct { + Observations int64 `json:"observations"` + UniquePackets int64 `json:"uniquePackets"` + ActiveObservers int64 `json:"activeObservers"` // distinct across the hour or window + ActiveIATAs int64 `json:"activeIatas"` // distinct across the hour or window + ScopedPackets int64 `json:"scopedPackets"` + ActiveScopes int64 `json:"activeScopes"` // distinct across the hour or window + MaxPathEntries int32 `json:"maxPathEntries"` + SNRSum float64 `json:"snrSum"` // average = snrSum / snrSamples + SNRSamples int64 `json:"snrSamples"` + RSSISum float64 `json:"rssiSum"` // average = rssiSum / rssiSamples + RSSISamples int64 `json:"rssiSamples"` } // ObservationPoint is a single time-bucketed observation count for charting. +// Distinct counts don't sum across IATAs, so use /stats/series for those. type ObservationPoint struct { Hour int64 `json:"hour"` // epoch ms, start of the 1-hour bucket IATA string `json:"iata"` ObservationCount int64 `json:"observationCount"` - UniquePackets int64 `json:"uniquePackets"` - ActiveObservers int64 `json:"activeObservers"` } // PayloadBreakdownItem is a single payload type with its observation count. @@ -61,15 +98,16 @@ type ScopeStats struct { NodeCount int64 `json:"nodeCount"` // distinct nodes with this as their default scope } -// TopNode is a node ranked by observation count from the mv_top_nodes_by_iata materialized view. +// TopNode is a node ranked by how often its adverts were heard in the window. type TopNode struct { - NodeID uuid.UUID `json:"nodeId"` - NodeName *string `json:"nodeName,omitempty"` - NodeType int16 `json:"nodeType"` - NodeTypeName string `json:"nodeTypeName"` - IATA string `json:"iata"` - ObservationCount int64 `json:"observationCount"` - LastHeard int64 `json:"lastHeard"` // epoch ms + NodeID *uuid.UUID `json:"nodeId"` // null once the node row has been deleted + PublicKey string `json:"publicKey"` // hex + NodeName *string `json:"nodeName,omitempty"` + NodeType int16 `json:"nodeType"` + NodeTypeName string `json:"nodeTypeName"` + IATA string `json:"iata"` + ObservationCount int64 `json:"observationCount"` + LastHeard int64 `json:"lastHeard"` // epoch ms } // TopObserver is an observer ranked by observation count. @@ -84,12 +122,13 @@ type TopObserver struct { // TopAdvertiser is a node ranked by distinct ADVERT packet count within the requested // window. Count is per-advert, not per-hearing -- see GetStatsTopAdvertisers. type TopAdvertiser struct { - NodeID uuid.UUID `json:"nodeId"` - NodeName *string `json:"nodeName,omitempty"` - NodeType int16 `json:"nodeType"` - NodeTypeName string `json:"nodeTypeName"` - IATA string `json:"iata"` - AdvertCount int64 `json:"advertCount"` + NodeID *uuid.UUID `json:"nodeId"` // null once the node row has been deleted + PublicKey string `json:"publicKey"` // hex + NodeName *string `json:"nodeName,omitempty"` + NodeType int16 `json:"nodeType"` + NodeTypeName string `json:"nodeTypeName"` + IATA string `json:"iata"` + AdvertCount int64 `json:"advertCount"` // FloodAdvertCount/DirectAdvertCount split AdvertCount by how the advert was routed: // flood = route type 0 (transport_flood) or 1 (flood), broadcast with no known path; // direct = route type 2 (direct) or 3 (transport_direct), routed along a known path. diff --git a/internal/background/rollup.go b/internal/background/rollup.go new file mode 100644 index 00000000..a23fe30c --- /dev/null +++ b/internal/background/rollup.go @@ -0,0 +1,161 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" +) + +// Per-pass budgets keep one pass short; the boot catch-up loops passes instead. +const ( + rollupInterval = 5 * time.Minute + rollupMissingBudget = 48 + rollupDirtyBatch = 24 + rollupDirtyBudget = 168 // a key import can dirty a week of hours at once +) + +// RollupSession is the lock-holding connection a pass works through (see db.RollupSession). +type RollupSession interface { + RegisterHours(ctx context.Context, since time.Time) error + MarkPartialHours(ctx context.Context) error + MissingHours(ctx context.Context, limit int32) ([]time.Time, error) + DirtyHours(ctx context.Context, limit int32) ([]time.Time, error) + RollHour(ctx context.Context, hour time.Time) (db.RollOutcome, bool, error) + Close(ctx context.Context) +} + +// Rollup rolls closed hours into the analytics rollup tables. Only the process holding +// the advisory lock does any work; the others skip the pass. +type Rollup struct { + begin func(context.Context) (RollupSession, bool, error) + packetRetention time.Duration + rollupRetention time.Duration + now func() time.Time +} + +// NewRollup rolls hours still inside both retentions using store's rollup lock. +func NewRollup(store *db.Store, packetRetention, rollupRetention time.Duration) *Rollup { + return &Rollup{ + begin: func(ctx context.Context) (RollupSession, bool, error) { + s, ok, err := store.BeginRollup(ctx) + if s == nil { + return nil, ok, err + } + return s, ok, err + }, + packetRetention: packetRetention, + rollupRetention: rollupRetention, + now: time.Now, + } +} + +// Task runs a pass every five minutes. +func (r *Rollup) Task() Task { + return Task{ + Name: "analytics_rollup", + Interval: rollupInterval, + Run: func(ctx context.Context) error { + _, _, err := r.Pass(ctx) + return err + }, + } +} + +// CatchUp runs passes until the backlog fits in one, so a restart fills gaps without +// waiting for the ticker. +func (r *Rollup) CatchUp(ctx context.Context) error { + for { + _, more, err := r.Pass(ctx) + if err != nil || !more { + return err + } + } +} + +// registerSince is the first hour worth rolling: raw rows must still exist, and cleanup +// (DeleteOldRollups, hour < now - rollupRetention) must not delete it again next run. +func (r *Rollup) registerSince() time.Time { + now := r.now() + since := now.Add(-r.packetRetention) + if rc := now.Add(-r.rollupRetention); r.rollupRetention > 0 && rc.After(since) { + since = rc + } + if h := since.Truncate(time.Hour); h.Before(since) { + return h.Add(time.Hour) + } + return since +} + +// Pass registers and rolls missing hours oldest first, then re-rolls dirty ones. A failed +// hour doesn't stop the pass; its error is returned with the others. rolled counts hours +// attempted; more reports a budget ran out with work likely left. +func (r *Rollup) Pass(ctx context.Context) (rolled int, more bool, err error) { + s, ok, err := r.begin(ctx) + if err != nil || !ok { + return 0, false, err + } + defer func() { + closeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + s.Close(closeCtx) + cancel() + }() + + if err := s.RegisterHours(ctx, r.registerSince()); err != nil { + return 0, false, fmt.Errorf("register hours: %w", err) + } + if err := s.MarkPartialHours(ctx); err != nil { + return 0, false, fmt.Errorf("mark partial hours: %w", err) + } + var errs []error + roll := func(hours []time.Time) { + for _, h := range hours { + if ctx.Err() != nil { + return + } + if _, _, err := s.RollHour(ctx, h); err != nil { + errs = append(errs, fmt.Errorf("roll %s: %w", h.UTC().Format(time.RFC3339), err)) + } + } + } + missing, err := s.MissingHours(ctx, rollupMissingBudget) + if err != nil { + return 0, false, fmt.Errorf("missing hours: %w", err) + } + roll(missing) + rolled, more = len(missing), len(missing) == rollupMissingBudget + // Hours that fail stay queued; never retry one within a pass. + tried := map[time.Time]bool{} + for budget := rollupDirtyBudget; budget > 0 && ctx.Err() == nil; { + dirty, err := s.DirtyHours(ctx, min(rollupDirtyBatch, int32(budget))) + if err != nil { + errs = append(errs, fmt.Errorf("dirty hours: %w", err)) + break + } + var fresh []time.Time + for _, h := range dirty { + if !tried[h] { + tried[h] = true + fresh = append(fresh, h) + } + } + if len(fresh) == 0 { + break + } + roll(fresh) + rolled += len(fresh) + budget -= len(fresh) + if budget == 0 { + more = true + } + } + if err := ctx.Err(); err != nil { + errs = append(errs, err) + } + return rolled, more, errors.Join(errs...) +} diff --git a/internal/background/rollup_test.go b/internal/background/rollup_test.go new file mode 100644 index 00000000..5b6d6ad6 --- /dev/null +++ b/internal/background/rollup_test.go @@ -0,0 +1,195 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package background + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/db" +) + +type rollupStub struct { + calls []string + since time.Time + missing [][]time.Time // one batch per pass + dirty []time.Time + failAt map[time.Time]error + done map[time.Time]bool // rolled dirty hours leave the queue + closed int +} + +func (s *rollupStub) RegisterHours(_ context.Context, since time.Time) error { + s.calls = append(s.calls, "register") + s.since = since + return nil +} +func (s *rollupStub) MarkPartialHours(context.Context) error { + s.calls = append(s.calls, "partial") + return nil +} +func (s *rollupStub) MissingHours(_ context.Context, limit int32) ([]time.Time, error) { + s.calls = append(s.calls, fmt.Sprintf("missing(%d)", limit)) + if len(s.missing) == 0 { + return nil, nil + } + batch := s.missing[0] + s.missing = s.missing[1:] + return batch, nil +} +func (s *rollupStub) DirtyHours(_ context.Context, limit int32) ([]time.Time, error) { + s.calls = append(s.calls, fmt.Sprintf("dirty(%d)", limit)) + var out []time.Time + for _, h := range s.dirty { + if !s.done[h] && int32(len(out)) < limit { + out = append(out, h) + } + } + return out, nil +} +func (s *rollupStub) RollHour(_ context.Context, h time.Time) (db.RollOutcome, bool, error) { + s.calls = append(s.calls, "roll "+h.Format("15")) + if err := s.failAt[h]; err != nil { + return db.RollSkipped, false, err + } + if s.done == nil { + s.done = map[time.Time]bool{} + } + s.done[h] = true + return db.RollComplete, true, nil +} +func (s *rollupStub) Close(context.Context) { s.closed++ } + +func newTestRollup(s *rollupStub, locked bool, beginErr error) *Rollup { + now := time.Date(2026, 1, 8, 12, 0, 0, 0, time.UTC) + return &Rollup{ + begin: func(context.Context) (RollupSession, bool, error) { + if beginErr != nil || !locked { + return nil, false, beginErr + } + return s, true, nil + }, + packetRetention: 7 * 24 * time.Hour, + rollupRetention: 90 * 24 * time.Hour, + now: func() time.Time { return now }, + } +} + +func hours(hs ...int) []time.Time { + out := make([]time.Time, len(hs)) + for i, h := range hs { + out[i] = time.Date(2026, 1, 8, h, 0, 0, 0, time.UTC) + } + return out +} + +func TestRollupPass(t *testing.T) { + s := &rollupStub{missing: [][]time.Time{hours(1, 2, 3)}, dirty: hours(0, 4)} + s.failAt = map[time.Time]error{hours(2)[0]: errors.New("boom"), hours(4)[0]: errors.New("stuck")} + rolled, more, err := newTestRollup(s, true, nil).Pass(context.Background()) + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Errorf("err = %v, want the failed hour's error", err) + } + if rolled != 5 || more { + t.Errorf("rolled = %d more = %v, want 5, false", rolled, more) + } + // A failing dirty hour stays queued but isn't retried within the pass. + want := []string{"register", "partial", "missing(48)", "roll 01", "roll 02", "roll 03", "dirty(24)", "roll 00", "roll 04", "dirty(24)"} + if !reflect.DeepEqual(s.calls, want) { + t.Errorf("calls = %v, want %v", s.calls, want) + } + if !s.since.Equal(time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)) { + t.Errorf("registered since %s, want now - packet retention", s.since) + } + if s.closed != 1 { + t.Errorf("session closed %d times", s.closed) + } +} + +func TestRollupRegisterSince(t *testing.T) { + now := time.Date(2026, 1, 8, 12, 20, 0, 0, time.UTC) + for _, tc := range []struct { + name string + packet, rollups time.Duration + want time.Time + }{ + {"packet retention shorter", 7 * 24 * time.Hour, 90 * 24 * time.Hour, time.Date(2026, 1, 1, 13, 0, 0, 0, time.UTC)}, + // Hours cleanup would delete next run must not be registered again. + {"rollup retention shorter", 30 * 24 * time.Hour, 24 * time.Hour, time.Date(2026, 1, 7, 13, 0, 0, 0, time.UTC)}, + } { + t.Run(tc.name, func(t *testing.T) { + r := &Rollup{packetRetention: tc.packet, rollupRetention: tc.rollups, now: func() time.Time { return now }} + got := r.registerSince() + if !got.Equal(tc.want) { + t.Errorf("registerSince = %s, want %s", got, tc.want) + } + if got.Before(now.Add(-tc.rollups)) { + t.Error("registers an hour DeleteOldRollups would remove") + } + }) + } +} + +func TestRollupPassLockHeldElsewhere(t *testing.T) { + s := &rollupStub{} + if rolled, _, err := newTestRollup(s, false, nil).Pass(context.Background()); err != nil || rolled != 0 || len(s.calls) != 0 { + t.Errorf("rolled %d err %v calls %v; want a silent skip", rolled, err, s.calls) + } + failure := errors.New("no connection") + if _, _, err := newTestRollup(s, true, failure).Pass(context.Background()); !errors.Is(err, failure) { + t.Errorf("err = %v, want %v", err, failure) + } +} + +func TestRollupCatchUp(t *testing.T) { + full := make([]int, rollupMissingBudget) + for i := range full { + full[i] = i % 24 + } + s := &rollupStub{missing: [][]time.Time{hours(full...), hours(full...), hours(5)}} + if err := newTestRollup(s, true, nil).CatchUp(context.Background()); err != nil { + t.Fatal(err) + } + passes := 0 + for _, c := range s.calls { + if c == "register" { + passes++ + } + } + if passes != 3 || s.closed != 3 { + t.Errorf("passes %d, closes %d; want 3 (two full batches, then a short one)", passes, s.closed) + } +} + +func TestRollupDrainsDirtyBacklog(t *testing.T) { + var backlog []time.Time + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i := range 200 { + backlog = append(backlog, start.Add(time.Duration(i)*time.Hour)) + } + s := &rollupStub{dirty: backlog} + r := newTestRollup(s, true, nil) + rolled, more, err := r.Pass(context.Background()) + if err != nil || rolled != rollupDirtyBudget || !more { + t.Fatalf("first pass rolled %d more %v err %v; want %d, true", rolled, more, err, rollupDirtyBudget) + } + if err := r.CatchUp(context.Background()); err != nil { + t.Fatal(err) + } + if len(s.done) != len(backlog) { + t.Errorf("drained %d of %d dirty hours", len(s.done), len(backlog)) + } +} + +func TestRollupTaskInterval(t *testing.T) { + task := newTestRollup(&rollupStub{}, true, nil).Task() + if task.Name != "analytics_rollup" || task.Interval != 5*time.Minute { + t.Errorf("task %q every %s", task.Name, task.Interval) + } +} diff --git a/internal/background/tasks.go b/internal/background/tasks.go index 951dca75..3fabc56f 100644 --- a/internal/background/tasks.go +++ b/internal/background/tasks.go @@ -5,8 +5,8 @@ package background import ( "context" - "errors" "fmt" + "log/slog" "time" "github.com/MeshCore-Beacon/beacon-server/db" @@ -14,78 +14,111 @@ import ( ) type viewRefresher interface { - RefreshHourlyStats(context.Context) error - RefreshTopNodes(context.Context) error - RefreshTopObservers(context.Context) error - RefreshPayloadBreakdown(context.Context) error - RefreshTopTalkers(context.Context) error - RefreshTopAdvertisers(context.Context) error RefreshRadioPresets(context.Context) error - RefreshObserverActivity(context.Context) error - RefreshSignalStats(context.Context) error - RefreshPathStats(context.Context) error } -// ViewRefreshTask returns a Task that refreshes all materialized views. +// ViewRefreshTask rebuilds the radio preset view; historical stats come from hourly rollups. func ViewRefreshTask(store viewRefresher, interval time.Duration) Task { return Task{ Name: "view_refresh", Interval: interval, Run: func(ctx context.Context) error { - var errs []error - for _, view := range []struct { - name string - refresh func(context.Context) error - }{ - {"hourly stats", store.RefreshHourlyStats}, - {"top nodes", store.RefreshTopNodes}, - {"top observers", store.RefreshTopObservers}, - {"payload breakdown", store.RefreshPayloadBreakdown}, - {"top talkers", store.RefreshTopTalkers}, - {"top advertisers", store.RefreshTopAdvertisers}, - {"radio presets", store.RefreshRadioPresets}, - {"observer activity", store.RefreshObserverActivity}, - {"signal stats", store.RefreshSignalStats}, - {"path stats", store.RefreshPathStats}, - } { - if err := view.refresh(ctx); err != nil { - errs = append(errs, fmt.Errorf("%s: %w", view.name, err)) - } + if err := store.RefreshRadioPresets(ctx); err != nil { + return fmt.Errorf("radio presets: %w", err) } - return errors.Join(errs...) + return nil }, } } -// CleanupTask returns a Task that prunes old telemetry, packet, and node rows. -func CleanupTask(store *db.Store, telemetryRetention, packetRetention, nodeDeleteAfter, interval time.Duration) Task { +type cleanupStore interface { + DeleteOldTelemetry(ctx context.Context, cutoff time.Time) error + OldestMissingRollupHour(ctx context.Context) (time.Time, bool, error) + DeleteOldPackets(ctx context.Context, cutoff time.Time) error + DeleteOldChannelIATAs(ctx context.Context, cutoff time.Time) error + DeleteOldTraceIATAs(ctx context.Context, cutoff time.Time) error + DeleteOldTraceTags(ctx context.Context, cutoff time.Time) error + DeleteOldRollups(ctx context.Context, cutoff time.Time) error + DeleteOldNodes(ctx context.Context, cutoff time.Time) error +} + +// CleanupConfig holds the retention windows CleanupTask enforces. +type CleanupConfig struct { + TelemetryRetention time.Duration + PacketRetention time.Duration + RollupRetention time.Duration + NodeDeleteAfter time.Duration + Interval time.Duration +} + +// Raw rows wait for unrolled hours, but not forever: past the cap those hours become partial. +const ( + rawHoldbackMargin = 35 * time.Minute // ingest clamps heard_at to ±30 min of last_heard_at + rawHoldbackWarn = 6 * time.Hour + rawHoldbackCap = 24 * time.Hour + traceTagLag = 30 * time.Minute +) + +// CleanupTask returns a Task that prunes old telemetry, packets, rollups and nodes. +func CleanupTask(store cleanupStore, cfg CleanupConfig) Task { + return cleanupTask(store, cfg, time.Now) +} + +func cleanupTask(store cleanupStore, cfg CleanupConfig, now func() time.Time) Task { return Task{ Name: "cleanup", - Interval: interval, + Interval: cfg.Interval, Run: func(ctx context.Context) error { - if err := store.DeleteOldTelemetry(ctx, time.Now().Add(-telemetryRetention)); err != nil { + t := now() + if err := store.DeleteOldTelemetry(ctx, t.Add(-cfg.TelemetryRetention)); err != nil { return err } - // One cutoff for all three so the IATA tables stay in step - // with the packets they mirror. - cutoff := time.Now().Add(-packetRetention) - if err := store.DeleteOldPackets(ctx, cutoff); err != nil { + cutoff, err := rawCutoff(ctx, store, t.Add(-cfg.PacketRetention)) + if err != nil { return err } - if err := store.DeleteOldChannelIATAs(ctx, cutoff); err != nil { - return err + // One cutoff so the IATA and trace tables stay in step with the packets they mirror. + for _, del := range []func(context.Context, time.Time) error{ + store.DeleteOldPackets, store.DeleteOldChannelIATAs, store.DeleteOldTraceIATAs, + } { + if err := del(ctx, cutoff); err != nil { + return err + } } - if err := store.DeleteOldTraceIATAs(ctx, cutoff); err != nil { + // Tag times are hearing times, up to the clamp behind their packets' last_heard_at; + // trailing by it keeps a re-heard packet's tag from being deleted out from under it. + if err := store.DeleteOldTraceTags(ctx, cutoff.Add(-traceTagLag)); err != nil { return err } - if err := store.DeleteOldNodes(ctx, time.Now().Add(-nodeDeleteAfter)); err != nil { + if err := store.DeleteOldRollups(ctx, t.Add(-cfg.RollupRetention)); err != nil { return err } - return nil + return store.DeleteOldNodes(ctx, t.Add(-cfg.NodeDeleteAfter)) }, } } +// rawCutoff holds packet deletion behind the oldest unrolled hour, up to rawHoldbackCap. +func rawCutoff(ctx context.Context, store cleanupStore, retentionCutoff time.Time) (time.Time, error) { + oldest, ok, err := store.OldestMissingRollupHour(ctx) + if err != nil || !ok { + return retentionCutoff, err + } + held := oldest.Add(-rawHoldbackMargin) + if !held.Before(retentionCutoff) { + return retentionCutoff, nil + } + holdback := retentionCutoff.Sub(held) + if holdback > rawHoldbackCap { + held, holdback = retentionCutoff.Add(-rawHoldbackCap), rawHoldbackCap + } + if holdback > rawHoldbackWarn { + slog.Warn("packet cleanup held back by unrolled analytics hours", "component", "background", + "holdback", holdback.Round(time.Minute), "oldest_missing_hour", oldest.UTC()) + } + return held, nil +} + // Limit lock lifetime while retaining the per-run work budget. const ( reconfirmRunLimit = 750_000 diff --git a/internal/background/tasks_test.go b/internal/background/tasks_test.go index 483f8586..7c668de4 100644 --- a/internal/background/tasks_test.go +++ b/internal/background/tasks_test.go @@ -10,6 +10,7 @@ import ( "errors" "log" "log/slog" + "reflect" "strings" "testing" "testing/synctest" @@ -96,65 +97,30 @@ func (s *refreshStub) refresh(ctx context.Context, name string) error { } return s.errs[name] } -func (s *refreshStub) RefreshHourlyStats(ctx context.Context) error { - return s.refresh(ctx, "hourly stats") -} -func (s *refreshStub) RefreshTopNodes(ctx context.Context) error { return s.refresh(ctx, "top nodes") } -func (s *refreshStub) RefreshTopObservers(ctx context.Context) error { - return s.refresh(ctx, "top observers") -} -func (s *refreshStub) RefreshPayloadBreakdown(ctx context.Context) error { - return s.refresh(ctx, "payload breakdown") -} -func (s *refreshStub) RefreshTopTalkers(ctx context.Context) error { - return s.refresh(ctx, "top talkers") -} -func (s *refreshStub) RefreshTopAdvertisers(ctx context.Context) error { - return s.refresh(ctx, "top advertisers") -} func (s *refreshStub) RefreshRadioPresets(ctx context.Context) error { return s.refresh(ctx, "radio presets") } -func (s *refreshStub) RefreshObserverActivity(ctx context.Context) error { - return s.refresh(ctx, "observer activity") -} - -func (s *refreshStub) RefreshSignalStats(ctx context.Context) error { - return s.refresh(ctx, "signal stats") -} - -func (s *refreshStub) RefreshPathStats(ctx context.Context) error { - return s.refresh(ctx, "path stats") -} func TestViewRefreshTask(t *testing.T) { - first, second := errors.New("first failure"), errors.New("second failure") + failure := errors.New("refresh failed") for _, tc := range []struct { name string errs map[string]error }{ {"success", nil}, - {"partial failure", map[string]error{"hourly stats": first, "top talkers": second}}, - {"signal failure", map[string]error{"signal stats": first}}, - {"path failure", map[string]error{"path stats": second}}, + {"failure", map[string]error{"radio presets": failure}}, } { t.Run(tc.name, func(t *testing.T) { store := &refreshStub{errs: tc.errs} - task := ViewRefreshTask(store, time.Minute) - err := task.Run(context.Background()) - if len(store.calls) != 10 { - t.Fatalf("refreshed %d views, want 10", len(store.calls)) + err := ViewRefreshTask(store, time.Minute).Run(context.Background()) + if len(store.calls) != 1 { + t.Fatalf("refreshed %d views, want 1", len(store.calls)) } if len(tc.errs) == 0 && err != nil { t.Fatal(err) } - for name, cause := range tc.errs { - if !errors.Is(err, cause) { - t.Errorf("missing cause %v in %v", cause, err) - } - if err == nil || !strings.Contains(err.Error(), name) { - t.Errorf("missing view name %q in %v", name, err) - } + if len(tc.errs) > 0 && (!errors.Is(err, failure) || !strings.Contains(err.Error(), "radio presets")) { + t.Errorf("err = %v", err) } }) } @@ -201,3 +167,82 @@ func TestSchedulerFailureIsNotComplete(t *testing.T) { } }) } + +type cleanupStub struct { + calls []string + cutoffs map[string]time.Time + oldest time.Time // zero = no missing hour +} + +func (s *cleanupStub) record(name string, cutoff time.Time) error { + s.calls = append(s.calls, name) + if s.cutoffs == nil { + s.cutoffs = map[string]time.Time{} + } + s.cutoffs[name] = cutoff + return nil +} +func (s *cleanupStub) DeleteOldTelemetry(_ context.Context, c time.Time) error { + return s.record("telemetry", c) +} +func (s *cleanupStub) OldestMissingRollupHour(context.Context) (time.Time, bool, error) { + s.calls = append(s.calls, "oldest") + return s.oldest, !s.oldest.IsZero(), nil +} +func (s *cleanupStub) DeleteOldPackets(_ context.Context, c time.Time) error { + return s.record("packets", c) +} +func (s *cleanupStub) DeleteOldChannelIATAs(_ context.Context, c time.Time) error { + return s.record("channel iatas", c) +} +func (s *cleanupStub) DeleteOldTraceIATAs(_ context.Context, c time.Time) error { + return s.record("trace iatas", c) +} +func (s *cleanupStub) DeleteOldTraceTags(_ context.Context, c time.Time) error { + return s.record("trace tags", c) +} +func (s *cleanupStub) DeleteOldRollups(_ context.Context, c time.Time) error { + return s.record("rollups", c) +} +func (s *cleanupStub) DeleteOldNodes(_ context.Context, c time.Time) error { + return s.record("nodes", c) +} + +func TestCleanupTask(t *testing.T) { + now := time.Date(2026, 1, 10, 12, 0, 0, 0, time.UTC) + cfg := CleanupConfig{TelemetryRetention: 31 * 24 * time.Hour, PacketRetention: 7 * 24 * time.Hour, + RollupRetention: 90 * 24 * time.Hour, NodeDeleteAfter: 30 * 24 * time.Hour, Interval: time.Hour} + retention := now.Add(-cfg.PacketRetention) + for _, tc := range []struct { + name string + oldest time.Time + want time.Time + }{ + {"no unrolled hours", time.Time{}, retention}, + {"unrolled hour after the cutoff", retention.Add(time.Hour), retention}, + {"held behind an unrolled hour", retention.Add(-2 * time.Hour), retention.Add(-2*time.Hour - 35*time.Minute)}, + {"holdback capped", retention.Add(-72 * time.Hour), retention.Add(-24 * time.Hour)}, + } { + t.Run(tc.name, func(t *testing.T) { + s := &cleanupStub{oldest: tc.oldest} + if err := cleanupTask(s, cfg, func() time.Time { return now }).Run(context.Background()); err != nil { + t.Fatal(err) + } + want := []string{"telemetry", "oldest", "packets", "channel iatas", "trace iatas", "trace tags", "rollups", "nodes"} + if !reflect.DeepEqual(s.calls, want) { + t.Errorf("calls %v, want %v", s.calls, want) + } + for _, name := range []string{"packets", "channel iatas", "trace iatas"} { + if !s.cutoffs[name].Equal(tc.want) { + t.Errorf("%s cutoff %s, want %s", name, s.cutoffs[name], tc.want) + } + } + if !s.cutoffs["trace tags"].Equal(tc.want.Add(-30 * time.Minute)) { + t.Errorf("trace tags cutoff %s, want 30 min behind packets", s.cutoffs["trace tags"]) + } + if !s.cutoffs["rollups"].Equal(now.Add(-cfg.RollupRetention)) || !s.cutoffs["telemetry"].Equal(now.Add(-cfg.TelemetryRetention)) { + t.Errorf("rollup/telemetry cutoffs %v", s.cutoffs) + } + }) + } +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 48fb5694..90cf83f5 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -16,11 +16,13 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/config" redis "github.com/redis/go-redis/v9" + "golang.org/x/sync/singleflight" ) // Client wraps a Redis client with helper methods used by CachedReader. type Client struct { rdb *redis.Client + sf singleflight.Group // one fetch per key across concurrent misses } // NewClient creates a new Redis client from the given address, password, and @@ -101,42 +103,62 @@ func (c *Client) delPrefix(ctx context.Context, prefix string) { // fails to unmarshal (corrupt or schema-changed), the entry is overwritten // with a fresh fetch. Errors from Set are ignored so a Redis hiccup never // fails a request. -func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Duration, fetch func() (T, error)) (T, error) { +func getOrSet[T any](ctx context.Context, c *Client, key string, ttl time.Duration, fetch func(context.Context) (T, error)) (T, error) { raw, err := c.rdb.Get(ctx, key).Bytes() if err != nil && !errors.Is(err, redis.Nil) { // real Redis error, degrade gracefully slog.DebugContext(ctx, "cache bypass", "component", "cache", "reason", "read_error") - return fetch() + return fetch(ctx) } var zero, out T - if errors.Is(err, redis.Nil) { - // cache miss — fetch, store, return + if err == nil { + if err = json.Unmarshal(raw, &out); err == nil { + slog.DebugContext(ctx, "cache hit", "component", "cache") + return out, nil + } + slog.DebugContext(ctx, "cache invalid entry", "component", "cache") + } else { slog.DebugContext(ctx, "cache miss", "component", "cache") - val, err := fetch() + } + raw, err = fetchShared(ctx, c, key, ttl, func(ctx context.Context) (any, error) { return fetch(ctx) }) + if err != nil { + return zero, err + } + // Each caller decodes its own copy: handlers may decorate the value they get. + if err := json.Unmarshal(raw, &out); err != nil { + return zero, err + } + return out, nil +} + +// sharedFetchTimeout bounds a fetch that no single request owns. +const sharedFetchTimeout = 30 * time.Second + +// fetchShared runs one fetch per key across concurrent misses and returns its JSON. The fetch +// gets its own bounded context so one caller disconnecting can't fail the others, and each +// caller still stops waiting when its own context ends. +func fetchShared(ctx context.Context, c *Client, key string, ttl time.Duration, fetch func(context.Context) (any, error)) ([]byte, error) { + ch := c.sf.DoChan(key, func() (any, error) { + fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), sharedFetchTimeout) + defer cancel() + val, err := fetch(fctx) if err != nil { - return zero, err - } - data, jsonErr := json.Marshal(val) - if jsonErr != nil { - return val, nil + return nil, err } - _ = c.rdb.Set(ctx, key, data, ttl) - return val, nil - } - if err = json.Unmarshal(raw, &out); err != nil { - // corrupt cache entry — overwrite it - slog.DebugContext(ctx, "cache invalid entry", "component", "cache") - val, err := fetch() + data, err := json.Marshal(val) if err != nil { - return zero, err + return nil, err } - data, jsonErr := json.Marshal(val) - if jsonErr != nil { - return val, nil + _ = c.rdb.Set(fctx, key, data, ttl) + return data, nil + }) + select { + case r := <-ch: + if r.Err != nil { + return nil, r.Err } - _ = c.rdb.Set(ctx, key, data, ttl) - return val, nil + return r.Val.([]byte), nil + case <-ctx.Done(): + return nil, ctx.Err() } - slog.DebugContext(ctx, "cache hit", "component", "cache") - return out, nil } diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 467cfcf4..2f912f07 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -31,6 +31,10 @@ type stubReader struct { calls int } +func (s *stubReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } +func (s *stubReader) GetStatsSeries(context.Context, time.Time, time.Time, []string) (*api.StatsSeries, error) { + return &api.StatsSeries{}, nil +} func (s *stubReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { return nil, s.err } @@ -73,7 +77,7 @@ func (s *stubReader) GetRegionBySlug(_ context.Context, _ string) (*api.Region, return nil, nil } func (s *stubReader) GetScopeNames(_ context.Context) ([]string, error) { return nil, nil } -func (s *stubReader) GetScopeStats(_ context.Context, _ []string) ([]api.ScopeStats, error) { +func (s *stubReader) GetScopeStats(_ context.Context, _ []string, _ time.Time) ([]api.ScopeStats, error) { return nil, nil } func (s *stubReader) GetScopeByName(_ context.Context, _ string) (*api.ScopeDetail, error) { @@ -92,7 +96,7 @@ func (s *stubReader) GetStatsPayloadBreakdown(_ context.Context, _ []string, _ t return nil, nil } -func (s *stubReader) GetStatsTopNodes(_ context.Context, _ []string, _ int32) ([]api.TopNode, error) { +func (s *stubReader) GetStatsTopNodes(_ context.Context, _ []string, _ time.Time, _ int32) ([]api.TopNode, error) { return nil, nil } @@ -258,7 +262,7 @@ func TestGetOrSet_CacheMiss_FetchesAndStores(t *testing.T) { c, _ := newTestClient(t) stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -277,11 +281,11 @@ func TestGetOrSet_CacheHit_DoesNotFetch(t *testing.T) { stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} // prime the cache - _, _ = getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + _, _ = getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) // second call should hit cache - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -300,7 +304,7 @@ func TestGetOrSet_RedisError_DegradeGracefully(t *testing.T) { mr.Close() // kill Redis stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -316,7 +320,7 @@ func TestGetOrSet_CorruptEntry_Overwrites(t *testing.T) { mr.Set("test:key", "not-valid-json") stub := &stubReader{iatas: []api.IATA{{IATA: "YVR"}}} - result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + result, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err != nil { @@ -334,7 +338,7 @@ func TestGetOrSet_FetchError_Propagates(t *testing.T) { c, _ := newTestClient(t) stub := &stubReader{err: errors.New("db error")} - _, err := getOrSet(context.Background(), c, "test:key", time.Minute, func() ([]api.IATA, error) { + _, err := getOrSet(context.Background(), c, "test:key", time.Minute, func(ctx context.Context) ([]api.IATA, error) { return stub.ListIATAs(context.Background()) }) if err == nil { @@ -357,14 +361,14 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { } // call with unsorted IATAs - getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func() (*api.StatsOverview, error) { + getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func(ctx context.Context) (*api.StatsOverview, error) { calls++ return &api.StatsOverview{TotalPackets: 42}, nil }) // call CachedReader with reversed order — should hit same key _ = cr - result, err := getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func() (*api.StatsOverview, error) { + result, err := getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func(ctx context.Context) (*api.StatsOverview, error) { calls++ return &api.StatsOverview{TotalPackets: 42}, nil }) diff --git a/internal/cache/logging_test.go b/internal/cache/logging_test.go index d16eed5e..94a45c68 100644 --- a/internal/cache/logging_test.go +++ b/internal/cache/logging_test.go @@ -22,7 +22,7 @@ func TestCacheDiagnostics(t *testing.T) { c, mr := newTestClient(t) key, value := "private-cache-key", "private-cache-value" calls := 0 - fetch := func() (string, error) { calls++; return value, nil } + fetch := func(context.Context) (string, error) { calls++; return value, nil } for i := 0; i < 2; i++ { got, err := getOrSet(context.Background(), c, key, time.Minute, fetch) if err != nil || got != value { diff --git a/internal/cache/paths.go b/internal/cache/paths.go deleted file mode 100644 index f5951cb3..00000000 --- a/internal/cache/paths.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later - -package cache - -import ( - "context" - "encoding/json" - "fmt" - "slices" - "time" - - "github.com/MeshCore-Beacon/beacon-server/internal/api" -) - -func (cr *CachedReader) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { - sorted := append([]string{}, iatas...) - slices.Sort(sorted) - sorted = slices.Compact(sorted) - segment, _ := json.Marshal(sorted) // [] and [""] must remain distinct (global vs empty region). - key := fmt.Sprintf("%s%s:%d:%d", keyPathStatsPrefix, segment, since.UnixMilli(), until.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.PathStats, error) { - return cr.inner.GetPathStats(ctx, since, until, iatas) - }) -} diff --git a/internal/cache/paths_test.go b/internal/cache/paths_test.go index fdffe97e..2c55bf5c 100644 --- a/internal/cache/paths_test.go +++ b/internal/cache/paths_test.go @@ -17,6 +17,8 @@ type pathStatsReader struct { calls int64 } +func (r *pathStatsReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } + func (r *pathStatsReader) GetPathStats(context.Context, time.Time, time.Time, []string) (*api.PathStats, error) { r.calls++ return &api.PathStats{Receptions: r.calls}, nil diff --git a/internal/cache/reader.go b/internal/cache/reader.go index e845e622..cac83ed4 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "fmt" - "slices" "sort" "strings" "time" @@ -29,6 +28,7 @@ const ( keyStatsOverviewPrefix = "beacon:stats:overview:" keyStatsObservationsPrefix = "beacon:stats:observations:" keySignalStatsPrefix = "beacon:stats:signal:" + keyStatsSeriesPrefix = "beacon:stats:series:" keyPathStatsPrefix = "beacon:stats:paths:" keyStatsBreakdownPrefix = "beacon:stats:breakdown:" keyStatsTopNodesPrefix = "beacon:stats:top-nodes:" @@ -56,6 +56,7 @@ type CachedReader struct { inner api.Reader c *Client ttl CacheTTLs + rev revisionMemo } // CacheTTLs holds the resolved per-category TTLs for the cache layer. @@ -83,6 +84,7 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { cr.c.del(ctx, keyScopeNames) cr.c.delPrefix(ctx, keyScopeByNamePrefix) + cr.c.delPrefix(ctx, keyScopeStats) } // InvalidateIATABorder makes an imported or removed MeshMapper boundary visible. @@ -118,49 +120,49 @@ func (cr *CachedReader) InvalidateObserver(ctx context.Context, observerID uuid. // ListIATAs implements [api.Reader]. func (cr *CachedReader) ListIATAs(ctx context.Context) ([]api.IATA, error) { - return getOrSet(ctx, cr.c, keyIATAs, cr.ttl.Reference, func() ([]api.IATA, error) { + return getOrSet(ctx, cr.c, keyIATAs, cr.ttl.Reference, func(ctx context.Context) ([]api.IATA, error) { return cr.inner.ListIATAs(ctx) }) } // GetIATA implements [api.Reader]. func (cr *CachedReader) GetIATA(ctx context.Context, iata string) (*api.IATA, error) { - return getOrSet(ctx, cr.c, keyIATAPrefix+iata, cr.ttl.Reference, func() (*api.IATA, error) { + return getOrSet(ctx, cr.c, keyIATAPrefix+iata, cr.ttl.Reference, func(ctx context.Context) (*api.IATA, error) { return cr.inner.GetIATA(ctx, iata) }) } // GetIATABorder implements [api.Reader]. func (cr *CachedReader) GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) { - return getOrSet(ctx, cr.c, keyIATABorderPrefix+iata, cr.ttl.Reference, func() (json.RawMessage, error) { + return getOrSet(ctx, cr.c, keyIATABorderPrefix+iata, cr.ttl.Reference, func(ctx context.Context) (json.RawMessage, error) { return cr.inner.GetIATABorder(ctx, iata) }) } // ListRegions implements [api.Reader]. func (cr *CachedReader) ListRegions(ctx context.Context) ([]api.RegionSummary, error) { - return getOrSet(ctx, cr.c, keyRegions, cr.ttl.Reference, func() ([]api.RegionSummary, error) { + return getOrSet(ctx, cr.c, keyRegions, cr.ttl.Reference, func(ctx context.Context) ([]api.RegionSummary, error) { return cr.inner.ListRegions(ctx) }) } // GetRegion implements [api.Reader]. func (cr *CachedReader) GetRegion(ctx context.Context, regionID int32) (*api.Region, error) { - return getOrSet(ctx, cr.c, fmt.Sprintf("%s%d", keyRegionPrefix, regionID), cr.ttl.Reference, func() (*api.Region, error) { + return getOrSet(ctx, cr.c, fmt.Sprintf("%s%d", keyRegionPrefix, regionID), cr.ttl.Reference, func(ctx context.Context) (*api.Region, error) { return cr.inner.GetRegion(ctx, regionID) }) } // GetRegionBySlug implements [api.Reader]. func (cr *CachedReader) GetRegionBySlug(ctx context.Context, slug string) (*api.Region, error) { - return getOrSet(ctx, cr.c, keyRegionSlugPrefix+slug, cr.ttl.Reference, func() (*api.Region, error) { + return getOrSet(ctx, cr.c, keyRegionSlugPrefix+slug, cr.ttl.Reference, func(ctx context.Context) (*api.Region, error) { return cr.inner.GetRegionBySlug(ctx, slug) }) } // GetScopeNames implements [api.Reader]. func (cr *CachedReader) GetScopeNames(ctx context.Context) ([]string, error) { - return getOrSet(ctx, cr.c, keyScopeNames, cr.ttl.Reference, func() ([]string, error) { + return getOrSet(ctx, cr.c, keyScopeNames, cr.ttl.Reference, func(ctx context.Context) ([]string, error) { return cr.inner.GetScopeNames(ctx) }) } @@ -171,82 +173,6 @@ func (cr *CachedReader) GetObserverComparison(ctx context.Context, a, b uuid.UUI return cr.inner.GetObserverComparison(ctx, a, b, since, until, iatas) } -// GetScopeStats implements [api.Reader]. -func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string) ([]api.ScopeStats, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(slices.Compact(sorted), ",") - } - return getOrSet(ctx, cr.c, keyScopeStats+":"+segment, cr.ttl.Reference, func() ([]api.ScopeStats, error) { - return cr.inner.GetScopeStats(ctx, iatas) - }) -} - -// GetScopeByName implements [api.Reader]. -func (cr *CachedReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { - return getOrSet(ctx, cr.c, keyScopeByNamePrefix+name, cr.ttl.Reference, func() (*api.ScopeDetail, error) { - return cr.inner.GetScopeByName(ctx, name) - }) -} - -// GetStatsOverview implements [api.Reader]. -func (cr *CachedReader) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s", keyStatsOverviewPrefix, segment) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.StatsOverview, error) { - return cr.inner.GetStatsOverview(ctx, iatas) - }) -} - -// GetStatsObservations implements [api.Reader]. -func (cr *CachedReader) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d", keyStatsObservationsPrefix, segment, since.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.ObservationPoint, error) { - return cr.inner.GetStatsObservations(ctx, iatas, since) - }) -} - -// GetStatsPayloadBreakdown implements [api.Reader]. -func (cr *CachedReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d", keyStatsBreakdownPrefix, segment, since.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.PayloadBreakdownItem, error) { - return cr.inner.GetStatsPayloadBreakdown(ctx, iatas, since) - }) -} - -// GetStatsTopNodes implements [api.Reader]. -func (cr *CachedReader) GetStatsTopNodes(ctx context.Context, iatas []string, limit int32) ([]api.TopNode, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d", keyStatsTopNodesPrefix, segment, limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopNode, error) { - return cr.inner.GetStatsTopNodes(ctx, iatas, limit) - }) -} - // GetStatsNodeTypes implements [api.Reader]. func (cr *CachedReader) GetStatsNodeTypes(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) { segment := "all" @@ -256,39 +182,11 @@ func (cr *CachedReader) GetStatsNodeTypes(ctx context.Context, iatas []string) ( segment = strings.Join(sorted, ",") } key := fmt.Sprintf("%s%s", keyStatsNodeTypes, segment) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.NodeTypeCount, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func(ctx context.Context) ([]api.NodeTypeCount, error) { return cr.inner.GetStatsNodeTypes(ctx, iatas) }) } -// GetStatsTopObservers implements [api.Reader]. -func (cr *CachedReader) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d:%d", keyStatsTopObsPrefix, segment, since.UnixMilli(), limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopObserver, error) { - return cr.inner.GetStatsTopObservers(ctx, iatas, since, limit) - }) -} - -// GetStatsTopAdvertisers implements [api.Reader]. -func (cr *CachedReader) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d:%d", keyStatsTopAdvPrefix, segment, since.UnixMilli(), limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopAdvertiser, error) { - return cr.inner.GetStatsTopAdvertisers(ctx, iatas, since, limit) - }) -} - // GetStatsClockDrift implements [api.Reader]. func (cr *CachedReader) GetStatsClockDrift(ctx context.Context, iatas []string, limit int32) ([]api.ClockDriftEntry, error) { segment := "all" @@ -298,25 +196,11 @@ func (cr *CachedReader) GetStatsClockDrift(ctx context.Context, iatas []string, segment = strings.Join(sorted, ",") } key := fmt.Sprintf("%s%s:%d", keyStatsClockDriftPrefix, segment, limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.ClockDriftEntry, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func(ctx context.Context) ([]api.ClockDriftEntry, error) { return cr.inner.GetStatsClockDrift(ctx, iatas, limit) }) } -// GetStatsTopTalkers implements [api.Reader]. -func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { - segment := "all" - if len(iatas) > 0 { - sorted := append([]string(nil), iatas...) - sort.Strings(sorted) - segment = strings.Join(sorted, ",") - } - key := fmt.Sprintf("%s%s:%d:%d", keyStatsTopTalkersPrefix, segment, since.UnixMilli(), limit) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.TopTalker, error) { - return cr.inner.GetStatsTopTalkers(ctx, iatas, since, limit) - }) -} - // GetRadioPresets implements [api.Reader]. func (cr *CachedReader) GetRadioPresets(ctx context.Context, preset string, iatas []string) ([]api.RadioPreset, error) { segment := "all" @@ -326,21 +210,21 @@ func (cr *CachedReader) GetRadioPresets(ctx context.Context, preset string, iata segment = strings.Join(sorted, ",") } key := fmt.Sprintf("%s%s:%s", keyRadioPresetsPrefix, preset, segment) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() ([]api.RadioPreset, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func(ctx context.Context) ([]api.RadioPreset, error) { return cr.inner.GetRadioPresets(ctx, preset, iatas) }) } // GetNode implements [api.Reader]. func (cr *CachedReader) GetNode(ctx context.Context, nodeID uuid.UUID) (*api.Node, error) { - return getOrSet(ctx, cr.c, keyNodePrefix+nodeID.String(), cr.ttl.Nodes, func() (*api.Node, error) { + return getOrSet(ctx, cr.c, keyNodePrefix+nodeID.String(), cr.ttl.Nodes, func(ctx context.Context) (*api.Node, error) { return cr.inner.GetNode(ctx, nodeID) }) } // GetNodeNeighbors implements [api.Reader]. func (cr *CachedReader) GetNodeNeighbors(ctx context.Context, nodeID uuid.UUID) ([]api.NodeNeighbor, error) { - return getOrSet(ctx, cr.c, keyNodeNeighborsPrefix+nodeID.String(), cr.ttl.Nodes, func() ([]api.NodeNeighbor, error) { + return getOrSet(ctx, cr.c, keyNodeNeighborsPrefix+nodeID.String(), cr.ttl.Nodes, func(ctx context.Context) ([]api.NodeNeighbor, error) { return cr.inner.GetNodeNeighbors(ctx, nodeID) }) } @@ -353,21 +237,21 @@ func (cr *CachedReader) GetNodesByIDs(ctx context.Context, ids []uuid.UUID) (map } sort.Strings(strs) key := keyNodesByIDsPrefix + strings.Join(strs, ",") - return getOrSet(ctx, cr.c, key, cr.ttl.Nodes, func() (map[uuid.UUID]*api.ResolvedNode, error) { + return getOrSet(ctx, cr.c, key, cr.ttl.Nodes, func(ctx context.Context) (map[uuid.UUID]*api.ResolvedNode, error) { return cr.inner.GetNodesByIDs(ctx, ids) }) } // GetObserver implements [api.Reader]. func (cr *CachedReader) GetObserver(ctx context.Context, observerID uuid.UUID) (*api.Observer, error) { - return getOrSet(ctx, cr.c, keyObserverPrefix+observerID.String(), cr.ttl.Observers, func() (*api.Observer, error) { + return getOrSet(ctx, cr.c, keyObserverPrefix+observerID.String(), cr.ttl.Observers, func(ctx context.Context) (*api.Observer, error) { return cr.inner.GetObserver(ctx, observerID) }) } // GetObserverScopes implements [api.Reader]. func (cr *CachedReader) GetObserverScopes(ctx context.Context, observerID uuid.UUID) ([]string, error) { - return getOrSet(ctx, cr.c, keyObserverScopesPrefix+observerID.String(), cr.ttl.Observers, func() ([]string, error) { + return getOrSet(ctx, cr.c, keyObserverScopesPrefix+observerID.String(), cr.ttl.Observers, func(ctx context.Context) ([]string, error) { return cr.inner.GetObserverScopes(ctx, observerID) }) } @@ -388,7 +272,7 @@ func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid until = until.UTC().Truncate(interval) } key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano) - return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) { + return getOrSet(ctx, cr.c, key, observerActivityTTL, func(ctx context.Context) (*api.ObserverActivity, error) { return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until) }) } diff --git a/internal/cache/rollup_stats.go b/internal/cache/rollup_stats.go new file mode 100644 index 00000000..6272fb19 --- /dev/null +++ b/internal/cache/rollup_stats.go @@ -0,0 +1,171 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sync" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +// revisionTTL bounds how stale a rollup revision read can be before a new roll shows up. +const revisionTTL = 10 * time.Second + +type revisionSource interface { + AnalyticsRevision(ctx context.Context) (int64, error) +} + +// revisionMemo spares Postgres a revision read on every cached stats request. +type revisionMemo struct { + mu sync.Mutex + rev int64 + at time.Time + now func() time.Time +} + +// get returns the analytics revision; ok is false when the inner reader can't supply one. +func (m *revisionMemo) get(ctx context.Context, inner api.Reader) (rev int64, ok bool) { + src, isSource := inner.(revisionSource) + if !isSource { + return 0, false + } + now := time.Now + if m.now != nil { + now = m.now + } + m.mu.Lock() + defer m.mu.Unlock() + if !m.at.IsZero() && now().Sub(m.at) < revisionTTL { + return m.rev, true + } + rev, err := src.AnalyticsRevision(ctx) + if err != nil { + return 0, false + } + m.rev, m.at = rev, now() + return rev, true +} + +// rollupKey builds a stats key that changes with the rollup revision. [] and [""] stay +// distinct (all IATAs vs an empty region). +func (cr *CachedReader) rollupKey(ctx context.Context, prefix string, iatas []string, parts ...any) (string, bool) { + rev, ok := cr.rev.get(ctx, cr.inner) + if !ok { + return "", false + } + sorted := append([]string{}, iatas...) + slices.Sort(sorted) + segment, _ := json.Marshal(slices.Compact(sorted)) + key := fmt.Sprintf("%sr%d:%s", prefix, rev, segment) + for _, p := range parts { + key += fmt.Sprintf(":%v", p) + } + return key, true +} + +// cachedRollup caches a rollup-backed read under a revision key, or reads through without one. +func cachedRollup[T any](ctx context.Context, cr *CachedReader, prefix string, iatas []string, fetch func(context.Context) (T, error), parts ...any) (T, error) { + key, ok := cr.rollupKey(ctx, prefix, iatas, parts...) + if !ok { + return fetch(ctx) + } + return getOrSet(ctx, cr.c, key, cr.ttl.Stats, fetch) +} + +// hourKey matches the store's hour snapping so polls within an hour share a key; 0 = default window. +func hourKey(t time.Time) int64 { + if t.IsZero() { + return 0 + } + return t.UTC().Truncate(time.Hour).UnixMilli() +} + +// GetStatsSeries implements [api.Reader]. +func (cr *CachedReader) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { + return cachedRollup(ctx, cr, keyStatsSeriesPrefix, iatas, func(ctx context.Context) (*api.StatsSeries, error) { + return cr.inner.GetStatsSeries(ctx, since, until, iatas) + }, since.UnixMilli(), until.UnixMilli()) +} + +// GetSignalStats implements [api.Reader]. +func (cr *CachedReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { + return cachedRollup(ctx, cr, keySignalStatsPrefix, iatas, func(ctx context.Context) (*api.SignalStats, error) { + return cr.inner.GetSignalStats(ctx, since, until, iatas) + }, since.UnixMilli(), until.UnixMilli()) +} + +// GetPathStats implements [api.Reader]. +func (cr *CachedReader) GetPathStats(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { + return cachedRollup(ctx, cr, keyPathStatsPrefix, iatas, func(ctx context.Context) (*api.PathStats, error) { + return cr.inner.GetPathStats(ctx, since, until, iatas) + }, since.UnixMilli(), until.UnixMilli()) +} + +// GetStatsOverview implements [api.Reader]. Its window follows the clock, so the current hour is part of the key. +func (cr *CachedReader) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { + return cachedRollup(ctx, cr, keyStatsOverviewPrefix, iatas, func(ctx context.Context) (*api.StatsOverview, error) { + return cr.inner.GetStatsOverview(ctx, iatas) + }, hourKey(time.Now())) +} + +// GetStatsObservations implements [api.Reader]. +func (cr *CachedReader) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { + return cachedRollup(ctx, cr, keyStatsObservationsPrefix, iatas, func(ctx context.Context) ([]api.ObservationPoint, error) { + return cr.inner.GetStatsObservations(ctx, iatas, since) + }, hourKey(since)) +} + +// GetStatsPayloadBreakdown implements [api.Reader]. +func (cr *CachedReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { + return cachedRollup(ctx, cr, keyStatsBreakdownPrefix, iatas, func(ctx context.Context) ([]api.PayloadBreakdownItem, error) { + return cr.inner.GetStatsPayloadBreakdown(ctx, iatas, since) + }, hourKey(since)) +} + +// GetStatsTopNodes implements [api.Reader]. +func (cr *CachedReader) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { + return cachedRollup(ctx, cr, keyStatsTopNodesPrefix, iatas, func(ctx context.Context) ([]api.TopNode, error) { + return cr.inner.GetStatsTopNodes(ctx, iatas, since, limit) + }, hourKey(since), limit) +} + +// GetStatsTopObservers implements [api.Reader]. +func (cr *CachedReader) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { + return cachedRollup(ctx, cr, keyStatsTopObsPrefix, iatas, func(ctx context.Context) ([]api.TopObserver, error) { + return cr.inner.GetStatsTopObservers(ctx, iatas, since, limit) + }, hourKey(since), limit) +} + +// GetStatsTopAdvertisers implements [api.Reader]. +func (cr *CachedReader) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { + return cachedRollup(ctx, cr, keyStatsTopAdvPrefix, iatas, func(ctx context.Context) ([]api.TopAdvertiser, error) { + return cr.inner.GetStatsTopAdvertisers(ctx, iatas, since, limit) + }, hourKey(since), limit) +} + +// GetStatsTopTalkers implements [api.Reader]. +func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { + return cachedRollup(ctx, cr, keyStatsTopTalkersPrefix, iatas, func(ctx context.Context) ([]api.TopTalker, error) { + return cr.inner.GetStatsTopTalkers(ctx, iatas, since, limit) + }, hourKey(since), limit) +} + +// GetScopeStats implements [api.Reader]. +func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { + return cachedRollup(ctx, cr, keyScopeStats+":", iatas, func(ctx context.Context) ([]api.ScopeStats, error) { + return cr.inner.GetScopeStats(ctx, iatas, since) + }, hourKey(since)) +} + +// GetScopeByName implements [api.Reader]. +func (cr *CachedReader) GetScopeByName(ctx context.Context, name string) (*api.ScopeDetail, error) { + return cachedRollup(ctx, cr, keyScopeByNamePrefix, nil, func(ctx context.Context) (*api.ScopeDetail, error) { + return cr.inner.GetScopeByName(ctx, name) + }, name) +} diff --git a/internal/cache/scope_stats_test.go b/internal/cache/scope_stats_test.go index 188a5b19..2975ad6b 100644 --- a/internal/cache/scope_stats_test.go +++ b/internal/cache/scope_stats_test.go @@ -17,11 +17,13 @@ type scopeStatsReader struct { calls int64 } -func (r *scopeStatsReader) GetScopeStats(context.Context, []string) ([]api.ScopeStats, error) { +func (r *scopeStatsReader) GetScopeStats(context.Context, []string, time.Time) ([]api.ScopeStats, error) { r.calls++ return []api.ScopeStats{{Name: "#test", PacketCount: r.calls}}, nil } +func (r *scopeStatsReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } + func TestScopeStatsCacheSeparatesIATAs(t *testing.T) { c, _ := newTestClient(t) inner := &scopeStatsReader{} @@ -42,7 +44,7 @@ func TestScopeStatsCacheSeparatesIATAs(t *testing.T) { {nil, 1}, } { before := slices.Clone(tc.iatas) - rows, err := reader.GetScopeStats(context.Background(), tc.iatas) + rows, err := reader.GetScopeStats(context.Background(), tc.iatas, time.Time{}) if err != nil { t.Fatal(err) } diff --git a/internal/cache/series_test.go b/internal/cache/series_test.go new file mode 100644 index 00000000..24ed0826 --- /dev/null +++ b/internal/cache/series_test.go @@ -0,0 +1,204 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" +) + +type seriesReader struct { + api.Reader + calls atomic.Int64 + revision atomic.Int64 + revReads atomic.Int64 + gate chan struct{} // when set, fetches block until closed +} + +func (r *seriesReader) GetStatsSeries(context.Context, time.Time, time.Time, []string) (*api.StatsSeries, error) { + if r.gate != nil { + <-r.gate + } + return &api.StatsSeries{Revision: r.calls.Add(1)}, nil +} + +func (r *seriesReader) AnalyticsRevision(context.Context) (int64, error) { + r.revReads.Add(1) + return r.revision.Load(), nil +} + +// seriesOnly hides AnalyticsRevision. +type seriesOnly struct{ *seriesReader } + +func (seriesOnly) AnalyticsRevision() {} + +func TestSeriesCacheKeyedByRevision(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}).(*CachedReader) + clock := time.Unix(0, 0) + cr.rev.now = func() time.Time { return clock } + get := func(iatas []string) int64 { + t.Helper() + got, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), iatas) + if err != nil { + t.Fatal(err) + } + return got.Revision + } + if get(nil) != 1 || get(nil) != 1 || get([]string{""}) != 2 || get([]string{"YVR", "YYJ"}) != 3 || get([]string{"YYJ", "YVR"}) != 3 { + t.Fatalf("unexpected cache behaviour: %d fetches", inner.calls.Load()) + } + // A new roll is invisible until the memo expires, then gets a fresh key. + inner.revision.Store(1) + if get(nil) != 1 { + t.Fatal("revision re-read inside the memo window") + } + clock = clock.Add(revisionTTL) + if get(nil) != 4 { + t.Fatal("new revision did not miss the cache") + } + if n := inner.revReads.Load(); n != 2 { + t.Errorf("revision read %d times, want 2", n) + } +} + +func TestSeriesCacheWithoutRevisionIsUncached(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{} + cr := NewCachedReader(seriesOnly{inner}, c, CacheTTLs{Stats: time.Hour}) + for range 2 { + if _, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil); err != nil { + t.Fatal(err) + } + } + if inner.calls.Load() != 2 { + t.Errorf("fetches = %d, want 2", inner.calls.Load()) + } +} + +func TestSeriesCacheSingleflight(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{gate: make(chan struct{})} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + if _, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil); err != nil { + t.Error(err) + } + }) + } + time.Sleep(50 * time.Millisecond) // let every caller reach the miss + close(inner.gate) + wg.Wait() + if n := inner.calls.Load(); n != 1 { + t.Errorf("concurrent misses ran %d fetches, want 1", n) + } +} + +// ctxSeriesReader blocks until released and fails if its fetch context was cancelled. +type ctxSeriesReader struct { + seriesReader + started chan struct{} + release chan struct{} +} + +func (r *ctxSeriesReader) GetStatsSeries(ctx context.Context, _, _ time.Time, _ []string) (*api.StatsSeries, error) { + r.calls.Add(1) + close(r.started) + <-r.release + if err := ctx.Err(); err != nil { + return nil, err + } + return &api.StatsSeries{Hours: []api.StatsSeriesHour{{Status: "complete"}}}, nil +} + +func TestSharedFetchSurvivesLeaderCancel(t *testing.T) { + c, _ := newTestClient(t) + inner := &ctxSeriesReader{started: make(chan struct{}), release: make(chan struct{})} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + leaderErr := make(chan error, 1) + go func() { + _, err := cr.GetStatsSeries(leaderCtx, time.UnixMilli(0), time.UnixMilli(3600000), nil) + leaderErr <- err + }() + <-inner.started + waiter := make(chan *api.StatsSeries, 1) + go func() { + got, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil) + if err != nil { + t.Error(err) + } + waiter <- got + }() + time.Sleep(20 * time.Millisecond) // let the waiter join the in-flight fetch + cancelLeader() + if err := <-leaderErr; !errors.Is(err, context.Canceled) { + t.Errorf("leader err = %v, want its own cancellation", err) + } + close(inner.release) + if got := <-waiter; got == nil || len(got.Hours) != 1 { + t.Errorf("waiter got %+v; the leader's cancellation leaked into the shared fetch", got) + } + if n := inner.calls.Load(); n != 1 { + t.Errorf("fetches = %d, want 1", n) + } +} + +func TestSharedFetchWaiterHonoursOwnDeadline(t *testing.T) { + c, _ := newTestClient(t) + inner := &ctxSeriesReader{started: make(chan struct{}), release: make(chan struct{})} + defer close(inner.release) + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + go func() { + _, _ = cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil) + }() + <-inner.started + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + if _, err := cr.GetStatsSeries(ctx, time.UnixMilli(0), time.UnixMilli(3600000), nil); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("err = %v, want the waiter's deadline", err) + } + if time.Since(start) > time.Second { + t.Error("waiter stayed blocked on the leader") + } +} + +func TestSharedFetchReturnsIndependentCopies(t *testing.T) { + c, _ := newTestClient(t) + inner := &seriesReader{gate: make(chan struct{})} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}) + results := make(chan *api.StatsSeries, 2) + var wg sync.WaitGroup + for range 2 { + wg.Go(func() { + got, err := cr.GetStatsSeries(context.Background(), time.UnixMilli(0), time.UnixMilli(3600000), nil) + if err != nil { + t.Error(err) + return + } + got.Since = time.Now().UnixNano() // a handler decorating its own response + results <- got + }) + } + time.Sleep(20 * time.Millisecond) + close(inner.gate) + wg.Wait() + a, b := <-results, <-results + if a == b { + t.Fatal("concurrent callers share one response object") + } + if inner.calls.Load() != 1 { + t.Errorf("fetches = %d, want 1", inner.calls.Load()) + } +} diff --git a/internal/cache/signal.go b/internal/cache/signal.go deleted file mode 100644 index d1dfaa51..00000000 --- a/internal/cache/signal.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2026 Beacon Contributors -// SPDX-License-Identifier: AGPL-3.0-or-later - -package cache - -import ( - "context" - "encoding/json" - "fmt" - "slices" - "time" - - "github.com/MeshCore-Beacon/beacon-server/internal/api" -) - -func (cr *CachedReader) GetSignalStats(ctx context.Context, since, until time.Time, iatas []string) (*api.SignalStats, error) { - sorted := append([]string{}, iatas...) - slices.Sort(sorted) - sorted = slices.Compact(sorted) - segment, _ := json.Marshal(sorted) // [] and [""] must remain distinct (global vs empty region). - key := fmt.Sprintf("%s%s:%d:%d", keySignalStatsPrefix, segment, since.UnixMilli(), until.UnixMilli()) - return getOrSet(ctx, cr.c, key, cr.ttl.Stats, func() (*api.SignalStats, error) { - return cr.inner.GetSignalStats(ctx, since, until, iatas) - }) -} diff --git a/internal/cache/signal_test.go b/internal/cache/signal_test.go index 89b739b6..5e8f3244 100644 --- a/internal/cache/signal_test.go +++ b/internal/cache/signal_test.go @@ -17,6 +17,8 @@ type signalStatsReader struct { calls int64 } +func (r *signalStatsReader) AnalyticsRevision(context.Context) (int64, error) { return 0, nil } + func (r *signalStatsReader) GetSignalStats(context.Context, time.Time, time.Time, []string) (*api.SignalStats, error) { r.calls++ return &api.SignalStats{Receptions: r.calls}, nil diff --git a/internal/config/config.go b/internal/config/config.go index 65abd77a..1ad309e2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -41,6 +41,7 @@ type Config struct { Presence PresenceConfig `yaml:"presence"` Nodes NodesConfig `yaml:"nodes"` Observers ObserversConfig `yaml:"observers"` + Analytics AnalyticsConfig `yaml:"analytics"` } // BackupConfig enables protected downloads. Disabled by default. @@ -90,6 +91,7 @@ type ResolvedConfig struct { RateLimit ResolvedRateLimitConfig TelemetryResolution time.Duration TelemetryRetention time.Duration + RollupRetention time.Duration PacketRetention time.Duration RouteRetention time.Duration RouteGrace time.Duration @@ -257,6 +259,12 @@ type PacketsConfig struct { Retention duration `yaml:"retention"` } +// AnalyticsConfig controls hourly rollup retention behaviour. +type AnalyticsConfig struct { + // RollupRetention is how long hourly rollups are kept and the longest window /stats/series accepts. Defaults to 2160h (90 days). + RollupRetention duration `yaml:"rollup_retention"` +} + // RoutesConfig controls known-route retention behaviour. type RoutesConfig struct { // Retention is how long a route is kept after it was last observed. @@ -411,6 +419,12 @@ func Load(path string) (*Config, error) { return nil, fmt.Errorf("websocket.allowed_origins[%d]: %w", i, err) } } + if d := cfg.Packets.Retention.Duration; d != 0 && d < 24*time.Hour { + return nil, fmt.Errorf("packets.retention must be at least 24h") + } + if d := cfg.Analytics.RollupRetention.Duration; d != 0 && d < 24*time.Hour { + return nil, fmt.Errorf("analytics.rollup_retention must be at least 24h") + } configDir := filepath.Dir(path) if err := cfg.validateMeshMapper(); err != nil { return nil, err @@ -465,6 +479,7 @@ func Resolve(cfg *Config) ResolvedConfig { }, TelemetryResolution: cfg.Telemetry.Resolution.Duration, TelemetryRetention: cfg.Telemetry.Retention.Duration, + RollupRetention: cfg.Analytics.RollupRetention.Duration, PacketRetention: cfg.Packets.Retention.Duration, RouteRetention: cfg.Routes.Retention.Duration, RouteGrace: cfg.Routes.Grace.Duration, @@ -493,7 +508,10 @@ func Resolve(cfg *Config) ResolvedConfig { r.TelemetryResolution = time.Hour } if r.TelemetryRetention == 0 { - r.TelemetryRetention = 28 * 24 * time.Hour + r.TelemetryRetention = 31 * 24 * time.Hour + } + if r.RollupRetention == 0 { + r.RollupRetention = 90 * 24 * time.Hour } if r.PacketRetention == 0 { r.PacketRetention = 7 * 24 * time.Hour @@ -542,8 +560,8 @@ func Resolve(cfg *Config) ResolvedConfig { func (r ResolvedConfig) String() string { return fmt.Sprintf( - "telemetryResolution=%s telemetryRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d maxConnectsPerMinute=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d", - r.TelemetryResolution, r.TelemetryRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, + "telemetryResolution=%s telemetryRetention=%s rollupRetention=%s packetRetention=%s routeRetention=%s routeGrace=%s routeMinObs=%d maxConnsPerIP=%d maxConnectsPerMinute=%d viewRefresh=%s reconfirm=%s cleanup=%s presenceFlush=%s presencePacketTTL=%s clockDriftThreshold=%s nodeStaleThreshold=%s nodeDeleteAfter=%s observerDeleteAfter=%s rateLimitEnabled=%t requestsPerMinute=%d burst=%d", + r.TelemetryResolution, r.TelemetryRetention, r.RollupRetention, r.PacketRetention, r.RouteRetention, r.RouteGrace, r.RouteMinObservations, r.MaxConnsPerIP, r.MaxConnectsPerMinute, r.ViewRefreshInterval, r.ReconfirmInterval, r.CleanupInterval, r.PresenceFlushInterval, r.PresencePacketTTL, r.ClockDriftThreshold, r.NodeStaleThreshold, r.NodeDeleteAfter, r.ObserverDeleteAfter, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4ba8b238..f13bbf61 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -127,8 +127,8 @@ func TestResolve_Defaults(t *testing.T) { if r.TelemetryResolution != time.Hour { t.Errorf("expected TelemetryResolution 1h, got %v", r.TelemetryResolution) } - if r.TelemetryRetention != 28*24*time.Hour { - t.Errorf("expected TelemetryRetention 672h, got %v", r.TelemetryRetention) + if r.TelemetryRetention != 31*24*time.Hour { + t.Errorf("expected TelemetryRetention 744h, got %v", r.TelemetryRetention) } if r.PacketRetention != 7*24*time.Hour { t.Errorf("expected PacketRetention 168h, got %v", r.PacketRetention) @@ -211,3 +211,64 @@ func TestResolve_RouteDefaults(t *testing.T) { t.Errorf("RouteMinObservations = %d, want 3", r.RouteMinObservations) } } + +func TestResolve_RollupRetentionDefaults(t *testing.T) { + r := Resolve(&Config{}) + if r.RollupRetention != 90*24*time.Hour { + t.Errorf("expected RollupRetention 2160h (90 days), got %v", r.RollupRetention) + } +} + +func TestResolve_TelemetryRetentionDefault(t *testing.T) { + r := Resolve(&Config{}) + if r.TelemetryRetention != 31*24*time.Hour { + t.Errorf("expected TelemetryRetention 744h (31 days), got %v", r.TelemetryRetention) + } +} + +func TestResolve_RollupRetentionExplicitOverride(t *testing.T) { + cfg := &Config{} + cfg.Analytics.RollupRetention.Duration = 60 * 24 * time.Hour + r := Resolve(cfg) + if r.RollupRetention != 60*24*time.Hour { + t.Errorf("expected RollupRetention 1440h (60 days), got %v", r.RollupRetention) + } +} + +func TestLoad_PacketsRetentionTooShort(t *testing.T) { + f, err := os.CreateTemp("", "beacon-config-*.yaml") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer os.Remove(f.Name()) + + _, _ = f.WriteString("packets:\n retention: 1h\n") + f.Close() + + _, err = Load(f.Name()) + if err == nil { + t.Fatal("expected error for packets.retention < 24h, got nil") + } + if !strings.Contains(err.Error(), "packets.retention must be at least 24h") { + t.Errorf("expected error about packets.retention, got: %v", err) + } +} + +func TestLoad_AnalyticsRollupRetentionTooShort(t *testing.T) { + f, err := os.CreateTemp("", "beacon-config-*.yaml") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + defer os.Remove(f.Name()) + + _, _ = f.WriteString("analytics:\n rollup_retention: 1h\n") + f.Close() + + _, err = Load(f.Name()) + if err == nil { + t.Fatal("expected error for analytics.rollup_retention < 24h, got nil") + } + if !strings.Contains(err.Error(), "analytics.rollup_retention must be at least 24h") { + t.Errorf("expected error about analytics.rollup_retention, got: %v", err) + } +} diff --git a/internal/ingest/advert_endpoint_update_test.go b/internal/ingest/advert_endpoint_update_test.go index aea19c4d..a0899522 100644 --- a/internal/ingest/advert_endpoint_update_test.go +++ b/internal/ingest/advert_endpoint_update_test.go @@ -41,8 +41,8 @@ func (s *advertEndpointDB) UpsertNode(_ context.Context, n UpsertNodeParams, _ R return s.node.ID, nil } -func (s *advertEndpointDB) InsertObservation(context.Context, InsertObservationParams) (bool, error) { - return !s.duplicate, nil +func (s *advertEndpointDB) InsertObservation(context.Context, InsertObservationParams) (bool, int64, error) { + return !s.duplicate, 1, nil } func TestAdvertEndpointUsesUpdatedName(t *testing.T) { @@ -99,7 +99,7 @@ func TestDuplicateAdvertSkipsLiveEndpointLookup(t *testing.T) { func TestRepeatAdvertUsesCurrentIdentityWithoutReapplyingAdvert(t *testing.T) { r := newRepeatHarness(t, true) - store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: r.db.stubDB, node: api.ResolvedNode{ID: uuid.New(), PublicKey: "aa"}}} + store := &advertEndpointDB{endpointCaptureDB: &endpointCaptureDB{stubDB: r.db, node: api.ResolvedNode{ID: uuid.New(), PublicKey: "aa"}}} r.w.db = store packet := buildAdvertPacketWithData(t, append([]byte{meshcore.AdvertTypeRepeater | meshcore.AdvertNameMask}, []byte("Advert name")...), false) hear := func(path byte) []packetObservationEvent { diff --git a/internal/ingest/backfill.go b/internal/ingest/backfill.go index 1a412839..65402ec7 100644 --- a/internal/ingest/backfill.go +++ b/internal/ingest/backfill.go @@ -41,7 +41,7 @@ type DecryptGroupTextResult struct { // decides what to do with that. The live path falls back to UpsertChannelHashOnly itself; // BackfillChannelMessages, which only scans packets already recorded that way, just leaves // them as they are and tries again next boot. -func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHash, rawPayload []byte) (*DecryptGroupTextResult, error) { +func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHash, rawPayload []byte, historical bool) (*DecryptGroupTextResult, error) { grpTxt, err := meshcore.GroupTextFromBytes(rawPayload) if err != nil { return nil, err @@ -70,6 +70,7 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa PacketHash: packetHash, SenderName: strings.ReplaceAll(strings.ToValidUTF8(payload.Sender, "\uFFFD"), "\x00", ""), SentAt: time.Unix(int64(payload.Timestamp), 0), + Historical: historical, Content: strings.ReplaceAll(strings.ToValidUTF8(payload.Text, "\uFFFD"), "\x00", ""), } newMsg, err := db.InsertChannelMessage(ctx, params) @@ -122,7 +123,7 @@ func BackfillChannelHashes(ctx context.Context, db DB, keys ChannelKeyStore, has func backfill(ctx context.Context, db DB, keys ChannelKeyStore, packets []UndecryptedPacket) int { decrypted := 0 for _, p := range packets { - result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload) + result, err := DecryptGroupText(ctx, db, keys, p.PacketHash, p.RawPayload, true) if err != nil { slog.Error(fmt.Sprintf("ingest: backfill: decrypt failed for packet %s", hex.EncodeToString(p.PacketHash)), "component", "ingest", "error", err) continue diff --git a/internal/ingest/backfill_test.go b/internal/ingest/backfill_test.go index c50d7a6a..7088b2bb 100644 --- a/internal/ingest/backfill_test.go +++ b/internal/ingest/backfill_test.go @@ -39,7 +39,7 @@ func TestDecryptGroupText_Success(t *testing.T) { }} raw := encryptedGroupText(t, channelHash, psk, "ded", "hello") - result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x01}, raw) + result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x01}, raw, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -64,7 +64,7 @@ func TestDecryptGroupText_NoMatchingKey(t *testing.T) { raw := encryptedGroupText(t, channelHash, make([]byte, 16), "ded", "hello") keys := &mapKeys{entries: map[byte][]keystore.Entry{}} - result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x02}, raw) + result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x02}, raw, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -88,7 +88,7 @@ func TestDecryptGroupText_WrongKeyForHash(t *testing.T) { channelHash: {{Key: wrongKey, Fingerprint: []byte{0xBB}}}, }} - result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x03}, raw) + result, err := DecryptGroupText(context.Background(), db, keys, []byte{0x03}, raw, false) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -100,7 +100,7 @@ func TestDecryptGroupText_WrongKeyForHash(t *testing.T) { func TestDecryptGroupText_MalformedPayload(t *testing.T) { db := &stubDB{} keys := &mapKeys{} - _, err := DecryptGroupText(context.Background(), db, keys, []byte{0x04}, []byte{}) + _, err := DecryptGroupText(context.Background(), db, keys, []byte{0x04}, []byte{}, false) if err == nil { t.Fatal("expected an error for an empty/malformed payload") } @@ -163,3 +163,24 @@ func TestBackfillChannelMessages_NoUndecryptedPackets(t *testing.T) { t.Errorf("expected 0, got %d", n) } } + +func TestBackfillMarksMessagesHistorical(t *testing.T) { + psk := make([]byte, 16) + db := &stubDB{ + insertChannelMessageResult: true, + undecryptedPackets: []UndecryptedPacket{{PacketHash: []byte{0x01}, RawPayload: encryptedGroupText(t, 0x44, psk, "ded", "hi")}}, + } + keys := &mapKeys{entries: map[byte][]keystore.Entry{0x44: {{Key: psk, Fingerprint: []byte{0xAA}}}}} + if n, err := BackfillChannelMessages(context.Background(), db, keys); err != nil || n != 1 { + t.Fatalf("n=%d err=%v", n, err) + } + if len(db.channelMessages) != 1 || !db.channelMessages[0].Historical { + t.Errorf("backfilled messages %+v; want Historical so their rolled hours are re-rolled", db.channelMessages) + } + if _, err := DecryptGroupText(context.Background(), db, keys, []byte{0x02}, encryptedGroupText(t, 0x44, psk, "x", "live"), false); err != nil { + t.Fatal(err) + } + if db.channelMessages[1].Historical { + t.Error("live messages must not queue re-rolls") + } +} diff --git a/internal/ingest/endpoint_work_test.go b/internal/ingest/endpoint_work_test.go index 50302913..ae124335 100644 --- a/internal/ingest/endpoint_work_test.go +++ b/internal/ingest/endpoint_work_test.go @@ -25,7 +25,7 @@ func endpointTestPacket() *meshcore.Packet { } func TestEndpointLookupsOnlyForLiveHearings(t *testing.T) { r := newRepeatHarness(t, true) - d := &endpointLookupDB{stubDB: r.db.stubDB} + d := &endpointLookupDB{stubDB: r.db} r.w.db = d packet := endpointTestPacket() hear := func(inserted bool, path byte) { diff --git a/internal/ingest/frame_length_test.go b/internal/ingest/frame_length_test.go index d69601d8..c2afb511 100644 --- a/internal/ingest/frame_length_test.go +++ b/internal/ingest/frame_length_test.go @@ -26,9 +26,9 @@ func (s *frameCaptureDB) UpsertPacket(_ context.Context, p UpsertPacketParams) ( return true, nil } -func (s *frameCaptureDB) InsertObservation(_ context.Context, o InsertObservationParams) (bool, error) { +func (s *frameCaptureDB) InsertObservation(_ context.Context, o InsertObservationParams) (bool, int64, error) { s.observed = append(s.observed, o) - return true, nil + return true, 1, nil } func (s *frameCaptureDB) GetObserverRadio(context.Context, uuid.UUID) (RadioSettings, error) { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index c9d0dc7a..9ca35846 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -86,7 +86,7 @@ type Config struct { type DB interface { // UpsertObserver upserts the observers row keyed on pubkey, and returns // the Observer ID, Display Name and an error if any. - UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) + UpsertObserver(ctx context.Context, pubkey []byte, iata string) (uuid.UUID, string, error) // UpsertObserverBroker records that this observer was seen on brokerName. UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error @@ -100,9 +100,9 @@ type DB interface { // SetPacketDecrypted marks a packet as decrypted in the DB SetPacketDecrypted(ctx context.Context, hash []byte) error - // InsertObservation inserts a packet_observations row. - // Returns (inserted, error); inserted=false means ON CONFLICT DO NOTHING fired. - InsertObservation(ctx context.Context, o InsertObservationParams) (bool, error) + // InsertObservation inserts a packet_observations row and returns the packet's + // observation count; inserted=false means ON CONFLICT DO NOTHING fired. + InsertObservation(ctx context.Context, o InsertObservationParams) (inserted bool, observationCount int64, err error) // SetNodeCapability flips supports_multibyte_paths or supports_multibyte_traces // for a node, never downgrading an existing TRUE. @@ -138,7 +138,7 @@ type DB interface { // and any error. UpdateObserverStatus(ctx context.Context, p UpdateObserverStatusParams) (uuid.UUID, error) - // GetObserverLastIATA returns the IATA from the most recent observation for the given observer. + // GetObserverLastIATA returns the IATA of the observer's most recent packet, or "" if none. GetObserverLastIATA(ctx context.Context, observerID uuid.UUID) (string, error) // InsertObserverTelemetry stores a telemetry snapshot for an observer. @@ -180,11 +180,8 @@ type DB interface { // UpsertChannelIATA upserts a channel_iatas row. UpsertChannelIATA(ctx context.Context, channelHash []byte, iata string, heardAt time.Time) error - // UpsertTraceIATA upserts a trace_iatas row. - UpsertTraceIATA(ctx context.Context, traceTag []byte, iata string, heardAt time.Time) error - - // GetPacketObservationCount returns the number of rows for the packet observations - GetPacketObservationCount(ctx context.Context, packetHash []byte) (int64, error) + // RecordTrace records one hearing of a TRACE packet in trace_iatas and trace_tags. + RecordTrace(ctx context.Context, h TraceHearing) error // GetTransportScopeByName returns the ID of a transport scope by its normalized name. GetTransportScopeByName(ctx context.Context, name string) (int32, error) diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 14acd2a6..cffa4654 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -180,8 +180,11 @@ type stubDB struct { upsertChannelCalls int upsertChannelHashOnlyCalls int upsertChannelIATACalls int - upsertTraceIATACalls int + traceHearings []TraceHearing + packetIsNew bool observationInserted bool + channelMessages []InsertChannelMessageParams + observationCount int64 insertChannelMessageResult bool // configurable return for InsertChannelMessage; default false undecryptedPackets []UndecryptedPacket undecryptedByHash []UndecryptedPacket @@ -201,7 +204,7 @@ func (s *stubDB) SetNodeCapability(_ context.Context, nodeID uuid.UUID, paths, t } // no-op implementations for remaining DB interface methods -func (s *stubDB) UpsertObserver(_ context.Context, _ []byte) (uuid.UUID, string, error) { +func (s *stubDB) UpsertObserver(_ context.Context, _ []byte, _ string) (uuid.UUID, string, error) { return uuid.Nil, "", nil } func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, _ bool) error { @@ -209,11 +212,11 @@ func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, } func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } func (s *stubDB) UpsertPacket(_ context.Context, _ UpsertPacketParams) (bool, error) { - return false, nil + return s.packetIsNew, nil } func (s *stubDB) SetPacketDecrypted(_ context.Context, _ []byte) error { return nil } -func (s *stubDB) InsertObservation(_ context.Context, _ InsertObservationParams) (bool, error) { - return s.observationInserted, nil +func (s *stubDB) InsertObservation(_ context.Context, _ InsertObservationParams) (bool, int64, error) { + return s.observationInserted, s.observationCount, nil } func (s *stubDB) SetNodeDefaultScope(_ context.Context, _ uuid.UUID, _ int32) error { return nil } func (s *stubDB) UpsertNode(_ context.Context, params UpsertNodeParams, _ RadioSettings) (uuid.UUID, error) { @@ -234,7 +237,8 @@ func (s *stubDB) GetNodesByIDs(_ context.Context, _ []uuid.UUID) (map[uuid.UUID] return nil, nil } -func (s *stubDB) InsertChannelMessage(_ context.Context, _ InsertChannelMessageParams) (*InsertedChannelMessage, error) { +func (s *stubDB) InsertChannelMessage(_ context.Context, m InsertChannelMessageParams) (*InsertedChannelMessage, error) { + s.channelMessages = append(s.channelMessages, m) if !s.insertChannelMessageResult { return nil, nil } @@ -294,15 +298,11 @@ func (s *stubDB) UpsertChannelIATA(_ context.Context, _ []byte, _ string, _ time return nil } -func (s *stubDB) UpsertTraceIATA(_ context.Context, _ []byte, _ string, _ time.Time) error { - s.upsertTraceIATACalls++ +func (s *stubDB) RecordTrace(_ context.Context, h TraceHearing) error { + s.traceHearings = append(s.traceHearings, h) return nil } -func (s *stubDB) GetPacketObservationCount(_ context.Context, _ []byte) (int64, error) { - return 0, nil -} - func (s *stubDB) GetTransportScopeByName(_ context.Context, _ string) (int32, error) { return 0, nil } func (s *stubDB) UpsertObserverScope(_ context.Context, _ uuid.UUID, _ int32) error { return nil } func (s *stubDB) UpsertKnownRoute(_ context.Context, _ []uuid.UUID, _ [][]byte, _ string, _ int32) error { diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go index 3def0c9a..65795603 100644 --- a/internal/ingest/neighbors.go +++ b/internal/ingest/neighbors.go @@ -43,7 +43,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra return } - observerID, _, err := w.db.UpsertObserver(ctx, pubkey) + observerID, _, err := w.db.UpsertObserver(ctx, pubkey, "") if err != nil { w.log.Error(fmt.Sprintf("db: upsert observer failed in neighbors from %s", pubkeyHex), "error", err) return diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 7bc4ccf5..8f56c819 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -38,6 +38,14 @@ type UpsertPacketParams struct { TraceTag []byte } +// TraceHearing is one hearing of a TRACE packet. The packet's own contribution to its tag +// (count, type, best path) is recorded when the packet is stored. +type TraceHearing struct { + TraceTag []byte + IATA string + HeardAt time.Time +} + // InsertObservationParams mirrors the columns written on packet_observations insert. type InsertObservationParams struct { PacketHash []byte @@ -321,7 +329,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ return } - id, observerName, err := w.db.UpsertObserver(ctx, pubkeyBytes) + id, observerName, err := w.db.UpsertObserver(ctx, pubkeyBytes, iata) if err != nil { w.log.Error(fmt.Sprintf("db: upsert observer failed with packet from %s/%s", iata, pubkeyHex), "error", err) return @@ -775,7 +783,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ PayloadType: int16(packet.PayloadType()), AirtimeMs: airtimeMs, } - inserted, err := w.db.InsertObservation(ctx, oParams) + inserted, observationCount, err := w.db.InsertObservation(ctx, oParams) if err != nil { w.log.Error(fmt.Sprintf("db: insert observation failed from %s/%s", iata, pubkeyHex), "error", err) return @@ -799,10 +807,10 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } } - // Runs on duplicate observations too; the upsert only writes when the row is >1h stale. + // Runs on duplicate observations too, so last_heard_at tracks every hearing. if traceTag != nil { - if err := w.db.UpsertTraceIATA(ctx, traceTag, iata, heardAt); err != nil { - w.log.Error(fmt.Sprintf("db: upsert trace IATA failed from %s/%s", iata, pubkeyHex), "error", err) + if err := w.db.RecordTrace(ctx, TraceHearing{TraceTag: traceTag, IATA: iata, HeardAt: heardAt}); err != nil { + w.log.Error(fmt.Sprintf("db: record trace failed from %s/%s", iata, pubkeyHex), "error", err) } } @@ -895,12 +903,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ evt.Observation.PathLength.HopCount = packet.PathHashCount() evt.Observation.PropagationTimeMs = 0 // not yet calculated if !repeat { - count, err := w.db.GetPacketObservationCount(ctx, packetHash[:]) - if err != nil { - w.log.Error("failed to get observation count", "error", err) - count = 0 - } - evt.Packet.ObservationCount = count + evt.Packet.ObservationCount = observationCount } if matchedScope != nil { evt.Packet.Scope = matchedScope diff --git a/internal/ingest/packet_endpoints_test.go b/internal/ingest/packet_endpoints_test.go index 3611f6f2..ef37c9d0 100644 --- a/internal/ingest/packet_endpoints_test.go +++ b/internal/ingest/packet_endpoints_test.go @@ -37,9 +37,9 @@ func (s *endpointCaptureDB) ResolveEndpointHashes(_ context.Context, _ string, h } return nil, nil } -func (s *endpointCaptureDB) InsertObservation(context.Context, InsertObservationParams) (bool, error) { +func (s *endpointCaptureDB) InsertObservation(context.Context, InsertObservationParams) (bool, int64, error) { s.observed++ - return true, nil + return true, 1, nil } // Stored rows resolve endpoints at read time, but live events still carry them. diff --git a/internal/ingest/repeats_test.go b/internal/ingest/repeats_test.go index 0f9e73d9..178b8f01 100644 --- a/internal/ingest/repeats_test.go +++ b/internal/ingest/repeats_test.go @@ -14,28 +14,17 @@ import ( "github.com/google/uuid" ) -type countQueryDB struct { - *stubDB - countQueries int -} - -func (s *countQueryDB) GetPacketObservationCount(context.Context, []byte) (int64, error) { - s.countQueries++ - return 1, nil -} - type repeatHarness struct { t *testing.T ctx context.Context w *Worker - db *countQueryDB + db *stubDB client *hub.Client } func newRepeatHarness(t *testing.T, includeRepeats bool) *repeatHarness { - w, base := newTestWorker() - db := &countQueryDB{stubDB: base} - w.db = db + w, db := newTestWorker() + db.observationCount = 3 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) go w.hub.Run() @@ -96,8 +85,8 @@ func TestHandlePacket_Repeats(t *testing.T) { if len(first) != 1 || strings.Contains(string(first[0]), "isRepeat") { t.Fatalf("first hearing: %s", first) } - if r.db.countQueries != 1 { - t.Fatalf("first hearing ran %d count queries", r.db.countQueries) + if got := string(packetFields(t, first[0])["observationCount"]); got != "3" { + t.Fatalf("first hearing observationCount = %s, want the count InsertObservation returned", got) } if got := r.hear(false, 0x11); len(got) != 0 { t.Fatalf("broker copy of the first hearing sent: %s", got) @@ -111,9 +100,6 @@ func TestHandlePacket_Repeats(t *testing.T) { if string(fields["isRepeat"]) != "true" || string(fields["isFirstObservation"]) != "false" || string(fields["observationCount"]) != "0" { t.Fatalf("repeat packet fields: %v", fields) } - if r.db.countQueries != 1 { - t.Fatalf("repeat ran a count query (%d total)", r.db.countQueries) - } if got := r.hear(false, 0x22, 0x11); len(got) != 0 { t.Fatalf("same path sent twice: %s", got) } @@ -127,9 +113,6 @@ func TestHandlePacket_RepeatsOffByDefault(t *testing.T) { if got := r.hear(false, 0x22, 0x11); len(got) != 0 { t.Fatalf("repeat sent with nobody opted in: %s", got) } - if r.db.countQueries != 1 { - t.Fatalf("duplicate ran a count query (%d total)", r.db.countQueries) - } } // A broker copy racing through the inserted hearing's DB work must find its path already diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 46fa8fe8..c92095f3 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -37,6 +37,7 @@ type InsertChannelMessageParams struct { SenderName string Content string SentAt time.Time + Historical bool // backfilled: its hours may already be rolled, so queue them for a re-roll } // InsertedChannelMessage is returned only for a new message, using its stored packet evidence. @@ -207,7 +208,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc } channelHashBytes := []byte{grpTxt.ChannelHash} - result, err := DecryptGroupText(ctx, w.db, w.keys, packetHash, packet.Payload) + result, err := DecryptGroupText(ctx, w.db, w.keys, packetHash, packet.Payload, false) if err != nil { w.log.Error("decrypt group text failed", "error", err) return diff --git a/internal/ingest/side_effects_test.go b/internal/ingest/side_effects_test.go index 93c5a4e3..d6e6fd53 100644 --- a/internal/ingest/side_effects_test.go +++ b/internal/ingest/side_effects_test.go @@ -297,14 +297,14 @@ func TestHandlePacket_Advert_SkipsChannelIATA(t *testing.T) { if db.upsertChannelIATACalls != 0 { t.Errorf("expected UpsertChannelIATA NOT to be called for a non-channel packet, got %d calls", db.upsertChannelIATACalls) } - if db.upsertTraceIATACalls != 0 { - t.Errorf("expected UpsertTraceIATA NOT to be called for a non-trace packet, got %d calls", db.upsertTraceIATACalls) + if len(db.traceHearings) != 0 { + t.Errorf("expected RecordTrace NOT to be called for a non-trace packet, got %d calls", len(db.traceHearings)) } } func buildTracePacket(t *testing.T) *meshcore.Packet { t.Helper() - payload, err := (&meshcore.Trace{Tag: 0xdeadbeef, AuthCode: 1}).ToBytes() + payload, err := (&meshcore.Trace{Tag: 0xdeadbeef, AuthCode: 1, PathHashes: []byte{0xaa, 0xbb}}).ToBytes() if err != nil { t.Fatalf("trace to bytes: %v", err) } @@ -314,14 +314,22 @@ func buildTracePacket(t *testing.T) *meshcore.Packet { } } -func TestHandlePacket_Trace_UpsertsTraceIATA(t *testing.T) { +func TestHandlePacket_Trace_RecordsTrace(t *testing.T) { w, db := newTestWorker() db.observationInserted = true + db.packetIsNew = true envelope := packetEnvelope(t, buildTracePacket(t)) + w.handlePacket(context.Background(), "YOW", "0102", envelope) + db.packetIsNew, db.observationInserted = false, false w.handlePacket(context.Background(), "YOW", "0102", envelope) - if db.upsertTraceIATACalls != 1 { - t.Errorf("expected UpsertTraceIATA to be called once for a stored trace, got %d", db.upsertTraceIATACalls) + if len(db.traceHearings) != 2 { + t.Fatalf("expected RecordTrace for every hearing, got %d", len(db.traceHearings)) + } + for _, h := range db.traceHearings { + if h.IATA != "YOW" || h.HeardAt.IsZero() || hex.EncodeToString(h.TraceTag) != "efbeadde" { + t.Errorf("unexpected hearing %+v", h) + } } } diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 9fe6a82b..8dde6129 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -152,7 +152,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) if len(dirty) > 0 || rawDirty { w.log.Warn(fmt.Sprintf("stripped NUL bytes from status from %s", pubkeyHex), "fields", dirty, "metadata", rawDirty) } - id, _, err := w.db.UpsertObserver(ctx, pubkey) + id, _, err := w.db.UpsertObserver(ctx, pubkey, "") if err != nil { w.log.Error(fmt.Sprintf("db: upsert observer failed in status from %s", pubkeyHex), "error", err) return diff --git a/internal/presence/coalescer.go b/internal/presence/coalescer.go index 517f1217..221aac9e 100644 --- a/internal/presence/coalescer.go +++ b/internal/presence/coalescer.go @@ -25,9 +25,10 @@ import ( type Store interface { ingest.DB - // TouchObservers applies coalesced last_seen bumps and observation_count - // deltas for the given observer IDs in one statement. - TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error + // TouchObservers applies coalesced last_seen bumps, observation_count + // deltas and last_iata for the given observer IDs in one statement. An + // IATA only replaces one heard earlier (iataAts), so a late flush can't regress it. + TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, iatas []string, iataAts []time.Time) error // TouchObserverBrokers applies coalesced last_seen/last_packet_at bumps // for the given (observer, broker) pairs in one statement. @@ -47,8 +48,10 @@ type identity struct { } type observerBump struct { - seen time.Time - count int32 + seen time.Time + count int32 + iata string + iataAt time.Time } type brokerBump struct{ seen, packet time.Time } @@ -97,13 +100,16 @@ func New(store Store, flushInterval, packetTTL time.Duration) *Coalescer { // UpsertObserver serves repeat lookups from the identity cache and records a // last_seen/observation_count bump instead of writing through. -func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) { +func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte, iata string) (uuid.UUID, string, error) { key := string(pubkey) c.mu.Lock() if ident, ok := c.identities[key]; ok { bump := c.dirtyObservers[ident.id] bump.seen = c.now() bump.count++ + if iata != "" { + bump.iata, bump.iataAt = iata, bump.seen + } c.dirtyObservers[ident.id] = bump ident.seen = bump.seen c.identities[key] = ident @@ -112,7 +118,7 @@ func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUI } c.mu.Unlock() - id, name, err := c.Store.UpsertObserver(ctx, pubkey) + id, name, err := c.Store.UpsertObserver(ctx, pubkey, iata) if err != nil { return id, name, err } @@ -305,10 +311,14 @@ func (c *Coalescer) flushObservers(ctx context.Context, observers map[uuid.UUID] ids := make([]uuid.UUID, 0, len(observers)) seen := make([]time.Time, 0, len(observers)) counts := make([]int32, 0, len(observers)) + iatas := make([]string, 0, len(observers)) + iataAts := make([]time.Time, 0, len(observers)) for id, bump := range observers { ids = append(ids, id) seen = append(seen, bump.seen) counts = append(counts, bump.count) + iatas = append(iatas, bump.iata) + iataAts = append(iataAts, bump.iataAt) } - return c.Store.TouchObservers(ctx, ids, seen, counts) + return c.Store.TouchObservers(ctx, ids, seen, counts, iatas, iataAts) } diff --git a/internal/presence/coalescer_test.go b/internal/presence/coalescer_test.go index 0900f5f7..6dd22672 100644 --- a/internal/presence/coalescer_test.go +++ b/internal/presence/coalescer_test.go @@ -10,21 +10,22 @@ import ( "testing" "time" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/google/uuid" ) type retentionStore struct { Store id uuid.UUID - touch func(context.Context, []uuid.UUID, []time.Time, []int32) error + touch func(context.Context, []uuid.UUID, []time.Time, []int32, []string, []time.Time) error prune func(context.Context, time.Time) ([]uuid.UUID, error) } -func (s *retentionStore) UpsertObserver(context.Context, []byte) (uuid.UUID, string, error) { +func (s *retentionStore) UpsertObserver(context.Context, []byte, string) (uuid.UUID, string, error) { return s.id, "fixture", nil } -func (s *retentionStore) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error { - return s.touch(ctx, ids, seen, counts) +func (s *retentionStore) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, iatas []string, iataAts []time.Time) error { + return s.touch(ctx, ids, seen, counts, iatas, iataAts) } func (s *retentionStore) DeleteOldObservers(ctx context.Context, cutoff time.Time) ([]uuid.UUID, error) { return s.prune(ctx, cutoff) @@ -34,13 +35,13 @@ func TestObserverRetentionAbortOnFlushFailure(t *testing.T) { for _, cause := range []error{errors.New("flush unavailable"), context.Canceled} { t.Run(cause.Error(), func(t *testing.T) { store := &retentionStore{id: uuid.New()} - store.touch = func(context.Context, []uuid.UUID, []time.Time, []int32) error { return cause } + store.touch = func(context.Context, []uuid.UUID, []time.Time, []int32, []string, []time.Time) error { return cause } store.prune = func(context.Context, time.Time) ([]uuid.UUID, error) { t.Error("deletion attempted after failed preparation") return nil, nil } c := New(store, time.Second, time.Second) - if _, _, err := c.UpsertObserver(context.Background(), []byte("fixture")); err != nil { + if _, _, err := c.UpsertObserver(context.Background(), []byte("fixture"), "YVR"); err != nil { t.Fatal(err) } if _, err := c.DeleteOldObservers(context.Background(), time.Now()); !errors.Is(err, cause) { @@ -63,13 +64,13 @@ func TestObserverRetentionPreservesActivityAfterFlush(t *testing.T) { c := New(store, time.Second, time.Second) c.now = func() time.Time { return now } for range 2 { - if _, _, err := c.UpsertObserver(ctx, []byte("fixture")); err != nil { + if _, _, err := c.UpsertObserver(ctx, []byte("fixture"), "YVR"); err != nil { t.Fatal(err) } } started, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) var calls atomic.Int32 - store.touch = func(_ context.Context, ids []uuid.UUID, seen []time.Time, counts []int32) error { + store.touch = func(_ context.Context, ids []uuid.UUID, seen []time.Time, counts []int32, _ []string, _ []time.Time) error { if calls.Add(1) == 1 { close(started) <-release @@ -103,3 +104,38 @@ func TestObserverRetentionPreservesActivityAfterFlush(t *testing.T) { }) } } + +func (s *retentionStore) UpdateObserverStatus(context.Context, ingest.UpdateObserverStatusParams) (uuid.UUID, error) { + return s.id, nil +} + +func TestObserverLastIATAFlush(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + store := &retentionStore{id: uuid.New()} + var gotIATAs []string + var gotAts []time.Time + store.touch = func(_ context.Context, _ []uuid.UUID, _ []time.Time, _ []int32, iatas []string, ats []time.Time) error { + gotIATAs, gotAts = iatas, ats + return nil + } + c := New(store, time.Second, time.Second) + c.now = func() time.Time { return now } + pk := []byte("fixture") + upsert := func(iata string) { + t.Helper() + if _, _, err := c.UpsertObserver(ctx, pk, iata); err != nil { + t.Fatal(err) + } + now = now.Add(time.Second) + } + + upsert("YVR") // write-through + upsert("YYZ") + heard := now.Add(-time.Second) + upsert("") // status/neighbors path must not clear it or move its timestamp + c.Flush(ctx) + if len(gotIATAs) != 1 || gotIATAs[0] != "YYZ" || !gotAts[0].Equal(heard) { + t.Fatalf("flushed %v at %v, want YYZ at %s", gotIATAs, gotAts, heard) + } +} diff --git a/sqlc.yaml b/sqlc.yaml index 032c9911..177f5b3c 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -5,7 +5,7 @@ version: "2" sql: - engine: "postgresql" queries: "db/queries" - schema: "db/migrations" + schema: ["db/migrations", "db/sqlc_scratch"] gen: go: package: "db" @@ -22,11 +22,3 @@ sql: go_type: "encoding/json.RawMessage" - db_type: "bpchar" go_type: "string" - - # Coalesced archive labels can still be NULL; preserve the API pointers. - - column: "mv_top_observers_by_iata.display_name" - go_type: {type: "string", pointer: true} - - column: "mv_top_observers_by_iata.observer_type" - go_type: {type: "string", pointer: true} - - column: "mv_top_advertisers_by_iata.name" - go_type: {type: "string", pointer: true} From 3e0b6c074c13a77b5aa2c13169fb0414686dab96 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 13:51:04 -0400 Subject: [PATCH 125/141] feat(stats): report the longest path per hour in /stats/paths --- db/paths.go | 1 + db/paths_integration_test.go | 23 +++++++++++++++++++++++ db/paths_test.go | 27 +++++++++++++++++++++++++++ db/queries/paths.sql | 4 +++- db/sqlc/paths.sql.go | 6 +++++- docs/docs.go | 4 ++++ docs/swagger.json | 4 ++++ docs/swagger.yaml | 4 ++++ internal/api/paths.go | 1 + 9 files changed, 72 insertions(+), 2 deletions(-) diff --git a/db/paths.go b/db/paths.go index 19cffd8c..2189f926 100644 --- a/db/paths.go +++ b/db/paths.go @@ -31,6 +31,7 @@ func (s *Store) GetPathStats(ctx context.Context, since, until time.Time, iatas } point := &stats.Hourly[len(stats.Hourly)-1] point.Receptions += row.Receptions + point.MaxEntries = max(point.MaxEntries, row.MaxEntries) switch row.Category { case 0: switch row.HashBytes { diff --git a/db/paths_integration_test.go b/db/paths_integration_test.go index 656c48f3..fbec87f4 100644 --- a/db/paths_integration_test.go +++ b/db/paths_integration_test.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http/httptest" "os" + "slices" "testing" "time" @@ -112,6 +113,28 @@ SELECT int4send(n),'00000000-0000-0000-0000-000000000001',heard_at,iata,payload_ }) } } + // The longest routed path per hour; hours with only empty, TRACE or invalid paths report 0. + for _, tc := range []struct { + iatas []string + want []int32 + }{ + {nil, []int32{32, 63, 0}}, + {[]string{"YVR", "YYJ"}, []int32{32, 63, 0}}, + {[]string{"YYJ"}, []int32{2, 0}}, + {[]string{"YYZ"}, []int32{0}}, + } { + got, err := store.GetPathStats(ctx, since, until, tc.iatas) + if err != nil { + t.Fatal(err) + } + var max []int32 + for _, h := range got.Hourly { + max = append(max, h.MaxEntries) + } + if !slices.Equal(max, tc.want) { + t.Errorf("maxEntries for %v = %v, want %v", tc.iatas, max, tc.want) + } + } w := httptest.NewRecorder() handlers.StatsRouter(store, handlers.StatsOptions{}).ServeHTTP(w, httptest.NewRequest("GET", fmt.Sprintf("/paths?since=%d&until=%d&iatas=YVR", since.UnixMilli()+123, until.UnixMilli()+123), nil).WithContext(ctx)) var response api.PathStats diff --git a/db/paths_test.go b/db/paths_test.go index beb2f582..3b13606d 100644 --- a/db/paths_test.go +++ b/db/paths_test.go @@ -9,6 +9,7 @@ import ( "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5/pgtype" ) type pathBucketQuerier struct { @@ -31,3 +32,29 @@ func TestPathStatsRejectsInvalidDatabaseBucket(t *testing.T) { } } } + +type pathRowsQuerier struct { + sqlc.Querier + rows []sqlc.GetPathStatsRow +} + +func (q pathRowsQuerier) GetPathStats(context.Context, sqlc.GetPathStatsParams) ([]sqlc.GetPathStatsRow, error) { + return q.rows, nil +} + +// An hour's maxEntries is the largest across its category/width rows, not the last one seen. +func TestPathStatsHourlyMaxEntries(t *testing.T) { + hour := pgtype.Timestamptz{Time: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), Valid: true} + store := &Store{q: pathRowsQuerier{rows: []sqlc.GetPathStatsRow{ + {IsHourly: true, Hour: hour, Category: 0, HashBytes: 1, Receptions: 3, MaxEntries: 9}, + {IsHourly: true, Hour: hour, Category: 0, HashBytes: 2, Receptions: 1, MaxEntries: 5}, + {IsHourly: true, Hour: hour, Category: 1, HashBytes: 0, Receptions: 2, MaxEntries: 0}, + }}} + got, err := store.GetPathStats(context.Background(), time.Time{}, time.Time{}, nil) + if err != nil { + t.Fatal(err) + } + if len(got.Hourly) != 1 || got.Hourly[0].MaxEntries != 9 { + t.Errorf("hourly %+v, want one hour with maxEntries 9", got.Hourly) + } +} diff --git a/db/queries/paths.sql b/db/queries/paths.sql index 8cdaf51b..effe72f3 100644 --- a/db/queries/paths.sql +++ b/db/queries/paths.sql @@ -12,7 +12,9 @@ WITH readings AS ( SELECT (grouping(hour) = 0)::boolean AS is_hourly, COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, category, hash_bytes, COALESCE(entries, 0)::integer AS entries, - sum(receptions)::bigint AS receptions + sum(receptions)::bigint AS receptions, + -- Non-routed categories store entries = 0, so an hour of only empty paths gives 0. + COALESCE(max(entries) FILTER (WHERE receptions > 0), 0)::integer AS max_entries FROM readings GROUP BY GROUPING SETS ((category, hash_bytes, entries), (hour, category, hash_bytes)) ORDER BY is_hourly, hour, category, hash_bytes, entries; diff --git a/db/sqlc/paths.sql.go b/db/sqlc/paths.sql.go index f5c24f13..5c2f3422 100644 --- a/db/sqlc/paths.sql.go +++ b/db/sqlc/paths.sql.go @@ -24,7 +24,9 @@ WITH readings AS ( SELECT (grouping(hour) = 0)::boolean AS is_hourly, COALESCE(hour, 'epoch'::timestamptz)::timestamptz AS hour, category, hash_bytes, COALESCE(entries, 0)::integer AS entries, - sum(receptions)::bigint AS receptions + sum(receptions)::bigint AS receptions, + -- Non-routed categories store entries = 0, so an hour of only empty paths gives 0. + COALESCE(max(entries) FILTER (WHERE receptions > 0), 0)::integer AS max_entries FROM readings GROUP BY GROUPING SETS ((category, hash_bytes, entries), (hour, category, hash_bytes)) ORDER BY is_hourly, hour, category, hash_bytes, entries @@ -43,6 +45,7 @@ type GetPathStatsRow struct { HashBytes int32 `json:"hash_bytes"` Entries int32 `json:"entries"` Receptions int64 `json:"receptions"` + MaxEntries int32 `json:"max_entries"` } // All HTTP aggregates read the hourly rollup, never observations. @@ -62,6 +65,7 @@ func (q *Queries) GetPathStats(ctx context.Context, arg GetPathStatsParams) ([]G &i.HashBytes, &i.Entries, &i.Receptions, + &i.MaxEntries, ); err != nil { return nil, err } diff --git a/docs/docs.go b/docs/docs.go index 425a460f..7dd349dd 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4835,6 +4835,10 @@ const docTemplate = `{ "hour": { "type": "integer" }, + "maxEntries": { + "description": "longest path received that hour across the requested IATAs; 0 if only empty paths", + "type": "integer" + }, "oneByte": { "type": "integer" }, diff --git a/docs/swagger.json b/docs/swagger.json index 757caae0..18fd7a9a 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -4833,6 +4833,10 @@ "hour": { "type": "integer" }, + "maxEntries": { + "description": "longest path received that hour across the requested IATAs; 0 if only empty paths", + "type": "integer" + }, "oneByte": { "type": "integer" }, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index b1431feb..f63782d8 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1021,6 +1021,10 @@ definitions: type: integer hour: type: integer + maxEntries: + description: longest path received that hour across the requested IATAs; 0 + if only empty paths + type: integer oneByte: type: integer receptions: diff --git a/internal/api/paths.go b/internal/api/paths.go index 86c3a00d..dad76d10 100644 --- a/internal/api/paths.go +++ b/internal/api/paths.go @@ -43,4 +43,5 @@ type PathHour struct { Empty int64 `json:"empty"` Trace int64 `json:"trace"` Unclassified int64 `json:"unclassified"` + MaxEntries int32 `json:"maxEntries"` // longest path received that hour across the requested IATAs; 0 if only empty paths } From 53d9f86dd4b5f5efc7a5100b1e5ac1080a0fe0a5 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 13:52:58 -0400 Subject: [PATCH 126/141] feat(stats): split /stats/scopes packet counts by hour --- db/queries/queries.sql | 11 ++++++++ db/scope_stats_integration_test.go | 13 ++++++--- db/sqlc/mock/querier.go | 15 +++++++++++ db/sqlc/querier.go | 2 ++ db/sqlc/queries.sql.go | 43 ++++++++++++++++++++++++++++++ db/stats.go | 16 ++++++++++- db/stats_test.go | 18 +++++++++++-- docs/docs.go | 21 ++++++++++++++- docs/swagger.json | 21 ++++++++++++++- docs/swagger.yaml | 22 ++++++++++++--- internal/api/handlers/stats.go | 2 +- internal/api/stats.go | 15 ++++++++--- internal/cache/rollup_stats.go | 3 ++- 13 files changed, 184 insertions(+), 18 deletions(-) diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 349d621a..36f90cb9 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1242,6 +1242,17 @@ WHERE ($1::text = '' OR preset = $1::text) AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR iata = ANY($2::bpchar[])) ORDER BY preset, iata, source_type; +-- name: GetScopeStatsHourly :many +-- GetScopeStats packet counts split by hour (same window and IATA-set filter); zero hours omitted. +SELECT ts.name, s.hour, SUM(s.packets)::bigint AS packets +FROM analytics_hourly_scope_sets s +JOIN transport_scopes ts ON ts.id = s.scope_id +WHERE s.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR s.iatas && @iatas::bpchar[]) +GROUP BY ts.name, s.hour +HAVING SUM(s.packets) > 0 +ORDER BY ts.name, s.hour; + -- name: GetScopeStats :many -- Packets since the given hour come from the IATA-set rollup (counted once per hour heard). -- Observer and node counts are current memberships; observers filter by their latest IATA. diff --git a/db/scope_stats_integration_test.go b/db/scope_stats_integration_test.go index 207e09d7..9fd4115f 100644 --- a/db/scope_stats_integration_test.go +++ b/db/scope_stats_integration_test.go @@ -82,10 +82,17 @@ SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YY if err != nil { t.Fatal(err) } + // All fixture traffic is in hour h; hours with no packets are omitted. + hourly := func(packets int64) []api.ScopeHour { + if packets == 0 { + return []api.ScopeHour{} + } + return []api.ScopeHour{{Hour: h.UnixMilli(), Packets: packets}} + } want := []api.ScopeStats{ - {Name: "#a", PacketCount: tc.a[0], ObserverCount: tc.a[1], NodeCount: tc.a[2]}, - {Name: "#b", PacketCount: tc.b[0], ObserverCount: tc.b[1], NodeCount: tc.b[2]}, - {Name: "#unused"}, + {Name: "#a", PacketCount: tc.a[0], ObserverCount: tc.a[1], NodeCount: tc.a[2], Hourly: hourly(tc.a[0])}, + {Name: "#b", PacketCount: tc.b[0], ObserverCount: tc.b[1], NodeCount: tc.b[2], Hourly: hourly(tc.b[0])}, + {Name: "#unused", Hourly: []api.ScopeHour{}}, } if !reflect.DeepEqual(rows, want) { t.Fatalf("counts = %+v, want %+v", rows, want) diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 2b436d9e..5f089cfe 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -972,6 +972,21 @@ func (mr *MockQuerierMockRecorder) GetScopeStats(ctx, arg any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStats", reflect.TypeOf((*MockQuerier)(nil).GetScopeStats), ctx, arg) } +// GetScopeStatsHourly mocks base method. +func (m *MockQuerier) GetScopeStatsHourly(ctx context.Context, arg db.GetScopeStatsHourlyParams) ([]db.GetScopeStatsHourlyRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetScopeStatsHourly", ctx, arg) + ret0, _ := ret[0].([]db.GetScopeStatsHourlyRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetScopeStatsHourly indicates an expected call of GetScopeStatsHourly. +func (mr *MockQuerierMockRecorder) GetScopeStatsHourly(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeStatsHourly", reflect.TypeOf((*MockQuerier)(nil).GetScopeStatsHourly), ctx, arg) +} + // GetSignalStats mocks base method. func (m *MockQuerier) GetSignalStats(ctx context.Context, arg db.GetSignalStatsParams) ([]db.GetSignalStatsRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 1eaf656e..837bf2c5 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -122,6 +122,8 @@ type Querier interface { // Packets since the given hour come from the IATA-set rollup (counted once per hour heard). // Observer and node counts are current memberships; observers filter by their latest IATA. GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([]GetScopeStatsRow, error) + // GetScopeStats packet counts split by hour (same window and IATA-set filter); zero hours omitted. + GetScopeStatsHourly(ctx context.Context, arg GetScopeStatsHourlyParams) ([]GetScopeStatsHourlyRow, error) // Read the hourly rollup, never observations on an HTTP request. // Weight averages by sample counts instead of averaging regional/hourly means. GetSignalStats(ctx context.Context, arg GetSignalStatsParams) ([]GetSignalStatsRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 0a3b572f..da083763 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1725,6 +1725,49 @@ func (q *Queries) GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([ return items, nil } +const getScopeStatsHourly = `-- name: GetScopeStatsHourly :many +SELECT ts.name, s.hour, SUM(s.packets)::bigint AS packets +FROM analytics_hourly_scope_sets s +JOIN transport_scopes ts ON ts.id = s.scope_id +WHERE s.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR s.iatas && $2::bpchar[]) +GROUP BY ts.name, s.hour +HAVING SUM(s.packets) > 0 +ORDER BY ts.name, s.hour +` + +type GetScopeStatsHourlyParams struct { + Since pgtype.Timestamptz `json:"since"` + Iatas []string `json:"iatas"` +} + +type GetScopeStatsHourlyRow struct { + Name string `json:"name"` + Hour pgtype.Timestamptz `json:"hour"` + Packets int64 `json:"packets"` +} + +// GetScopeStats packet counts split by hour (same window and IATA-set filter); zero hours omitted. +func (q *Queries) GetScopeStatsHourly(ctx context.Context, arg GetScopeStatsHourlyParams) ([]GetScopeStatsHourlyRow, error) { + rows, err := q.db.Query(ctx, getScopeStatsHourly, arg.Since, arg.Iatas) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetScopeStatsHourlyRow{} + for rows.Next() { + var i GetScopeStatsHourlyRow + if err := rows.Scan(&i.Name, &i.Hour, &i.Packets); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const getStatsClockDrift = `-- name: GetStatsClockDrift :many WITH page AS ( SELECT diff --git a/db/stats.go b/db/stats.go index 5bda12c1..1a061139 100644 --- a/db/stats.go +++ b/db/stats.go @@ -225,17 +225,31 @@ func (s *Store) GetRadioPresets(ctx context.Context, preset string, iatas []stri } func (s *Store) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { - rows, err := s.q.GetScopeStats(ctx, sqlc.GetScopeStatsParams{Since: rollupSince(since, 7*24*time.Hour), Iatas: iatas}) + start := rollupSince(since, 7*24*time.Hour) + rows, err := s.q.GetScopeStats(ctx, sqlc.GetScopeStatsParams{Since: start, Iatas: iatas}) if err != nil { return nil, err } + hours, err := s.q.GetScopeStatsHourly(ctx, sqlc.GetScopeStatsHourlyParams{Since: start, Iatas: iatas}) + if err != nil { + return nil, err + } + byScope := make(map[string][]api.ScopeHour) + for _, h := range hours { + byScope[h.Name] = append(byScope[h.Name], api.ScopeHour{Hour: h.Hour.Time.UnixMilli(), Packets: h.Packets}) + } items := make([]api.ScopeStats, 0, len(rows)) for _, r := range rows { + hourly := byScope[r.Name] + if hourly == nil { + hourly = []api.ScopeHour{} + } items = append(items, api.ScopeStats{ Name: r.Name, PacketCount: r.PacketCount, ObserverCount: r.ObserverCount, NodeCount: r.NodeCount, + Hourly: hourly, }) } return items, nil diff --git a/db/stats_test.go b/db/stats_test.go index 88e978b2..d70a1f37 100644 --- a/db/stats_test.go +++ b/db/stats_test.go @@ -232,6 +232,14 @@ func TestGetScopeStats(t *testing.T) { GetScopeStats(gomock.Any(), gomock.Any()). Return([]sqlc.GetScopeStatsRow{ {Name: "default", PacketCount: 100, ObserverCount: 5, NodeCount: 20}, + {Name: "quiet"}, + }, nil) + hour := time.Date(2026, 1, 1, 3, 0, 0, 0, time.UTC) + mock.EXPECT(). + GetScopeStatsHourly(gomock.Any(), gomock.Any()). + Return([]sqlc.GetScopeStatsHourlyRow{ + {Name: "default", Hour: pgtype.Timestamptz{Time: hour, Valid: true}, Packets: 60}, + {Name: "default", Hour: pgtype.Timestamptz{Time: hour.Add(time.Hour), Valid: true}, Packets: 40}, }, nil) store := &Store{q: mock} @@ -239,8 +247,14 @@ func TestGetScopeStats(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if len(items) != 1 { - t.Fatalf("expected 1 item, got %d", len(items)) + if len(items) != 2 { + t.Fatalf("expected 2 items, got %d", len(items)) + } + if h := items[0].Hourly; len(h) != 2 || h[0].Hour != hour.UnixMilli() || h[0].Packets+h[1].Packets != items[0].PacketCount { + t.Errorf("hourly %+v doesn't split packetCount %d", h, items[0].PacketCount) + } + if items[1].Hourly == nil || len(items[1].Hourly) != 0 { + t.Errorf("a scope without packets must have hourly [], got %#v", items[1].Hourly) } if items[0].Name != "default" { t.Errorf("expected Name default, got %s", items[0].Name) diff --git a/docs/docs.go b/docs/docs.go index 7dd349dd..4f826f36 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2798,7 +2798,7 @@ const docTemplate = `{ }, "/stats/scopes": { "get": { - "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour (zero hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -5172,9 +5172,28 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the UTC hour", + "type": "integer" + }, + "packets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats": { "type": "object", "properties": { + "hourly": { + "description": "packetCount by hour, oldest first; hours with none omitted", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour" + } + }, "name": { "description": "normalized scope name e.g. \"#bc\"", "type": "string" diff --git a/docs/swagger.json b/docs/swagger.json index 18fd7a9a..4765d079 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2796,7 +2796,7 @@ }, "/stats/scopes": { "get": { - "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour (zero hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -5170,9 +5170,28 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour": { + "type": "object", + "properties": { + "hour": { + "description": "epoch ms, start of the UTC hour", + "type": "integer" + }, + "packets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats": { "type": "object", "properties": { + "hourly": { + "description": "packetCount by hour, oldest first; hours with none omitted", + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour" + } + }, "name": { "description": "normalized scope name e.g. \"#bc\"", "type": "string" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f63782d8..3053d3a7 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1246,8 +1246,21 @@ definitions: packetCount: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour: + properties: + hour: + description: epoch ms, start of the UTC hour + type: integer + packets: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeStats: properties: + hourly: + description: packetCount by hour, oldest first; hours with none omitted + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeHour' + type: array name: description: normalized scope name e.g. "#bc" type: string @@ -3510,10 +3523,11 @@ paths: description: 'Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, - nodes by their IATA memberships. With filters, lists only manual scopes configured - for a matching region and imported scopes whose current MeshMapper catalogue - includes a matching IATA; those remain listed with zero counts. An empty region - returns an empty array.' + nodes by their IATA memberships. hourly splits packetCount by UTC hour (zero + hours omitted; outage gaps come from /stats/series hour status). With filters, + lists only manual scopes configured for a matching region and imported scopes + whose current MeshMapper catalogue includes a matching IATA; those remain + listed with zero counts. An empty region returns an empty array.' parameters: - description: Comma-separated IATA codes in: query diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index dc6216f9..6a331b35 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -457,7 +457,7 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // getStatsScopes godoc // // @Summary Scope statistics -// @Description Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. +// @Description Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour (zero hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" diff --git a/internal/api/stats.go b/internal/api/stats.go index bb1e8eb3..8086f441 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -92,10 +92,17 @@ type PayloadBreakdownItem struct { // ScopeStats represents aggregate statistics for a single transport scope. type ScopeStats struct { - Name string `json:"name"` // normalized scope name e.g. "#bc" - PacketCount int64 `json:"packetCount"` // distinct packets matched to this scope - ObserverCount int64 `json:"observerCount"` // distinct observers that forwarded packets in this scope - NodeCount int64 `json:"nodeCount"` // distinct nodes with this as their default scope + Name string `json:"name"` // normalized scope name e.g. "#bc" + PacketCount int64 `json:"packetCount"` // distinct packets matched to this scope + ObserverCount int64 `json:"observerCount"` // distinct observers that forwarded packets in this scope + NodeCount int64 `json:"nodeCount"` // distinct nodes with this as their default scope + Hourly []ScopeHour `json:"hourly"` // packetCount by hour, oldest first; hours with none omitted +} + +// ScopeHour is one UTC hour of a scope's packets; a scope's hours sum to its packetCount. +type ScopeHour struct { + Hour int64 `json:"hour"` // epoch ms, start of the UTC hour + Packets int64 `json:"packets"` } // TopNode is a node ranked by how often its adverts were heard in the window. diff --git a/internal/cache/rollup_stats.go b/internal/cache/rollup_stats.go index 6272fb19..d5dbb4a8 100644 --- a/internal/cache/rollup_stats.go +++ b/internal/cache/rollup_stats.go @@ -158,7 +158,8 @@ func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, // GetScopeStats implements [api.Reader]. func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { - return cachedRollup(ctx, cr, keyScopeStats+":", iatas, func(ctx context.Context) ([]api.ScopeStats, error) { + // ":h" keeps entries cached before hourly existed from being served without it. + return cachedRollup(ctx, cr, keyScopeStats+":h:", iatas, func(ctx context.Context) ([]api.ScopeStats, error) { return cr.inner.GetScopeStats(ctx, iatas, since) }, hourKey(since)) } From 2042c1617fcaa5abba0da55e5f6902c9ed262174 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 15:22:17 -0400 Subject: [PATCH 127/141] feat(stats): roll hourly scope observers and advertising nodes --- db/migrations/001_baseline.sql | 18 +++++++++ db/queries/queries.sql | 43 +++++++++++++++++----- db/queries/rollup.sql | 22 +++++++++-- db/rollup.go | 2 + db/rollup_integration_test.go | 11 +++++- db/scope_stats_integration_test.go | 47 +++++++++++++----------- db/sqlc/mock/querier.go | 28 ++++++++++++++ db/sqlc/models.go | 14 +++++++ db/sqlc/querier.go | 5 ++- db/sqlc/queries.sql.go | 59 +++++++++++++++++++++++------- db/sqlc/rollup.sql.go | 34 +++++++++++++++-- db/stats.go | 2 +- docs/docs.go | 10 ++++- docs/historical-stats.md | 1 + docs/swagger.json | 10 ++++- docs/swagger.yaml | 17 ++++++--- internal/api/handlers/stats.go | 2 +- internal/api/stats.go | 8 ++-- internal/cache/rollup_stats.go | 4 +- 19 files changed, 271 insertions(+), 66 deletions(-) diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql index 00af4592..7dd67ed8 100644 --- a/db/migrations/001_baseline.sql +++ b/db/migrations/001_baseline.sql @@ -547,6 +547,24 @@ CREATE TABLE analytics_hourly_scope_sets ( PRIMARY KEY (hour, iatas, scope_id) ); +-- Observers that heard a scoped packet, per IATA; distinct counts span any IATA filter. +CREATE TABLE analytics_hourly_scope_observers ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + scope_id integer NOT NULL, + observer_id uuid NOT NULL, + PRIMARY KEY (hour, scope_id, iata, observer_id) +); + +-- Nodes whose ADVERTs were heard in a scope, per IATA. +CREATE TABLE analytics_hourly_scope_nodes ( + hour timestamp with time zone NOT NULL, + iata character(3) NOT NULL, + scope_id integer NOT NULL, + origin_pubkey bytea NOT NULL, + PRIMARY KEY (hour, scope_id, iata, origin_pubkey) +); + CREATE TABLE analytics_rollup_hours ( hour timestamp with time zone PRIMARY KEY, status text NOT NULL CHECK (status IN ('complete', 'partial', 'missing')), diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 36f90cb9..dacbb98e 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1243,15 +1243,40 @@ WHERE ($1::text = '' OR preset = $1::text) ORDER BY preset, iata, source_type; -- name: GetScopeStatsHourly :many --- GetScopeStats packet counts split by hour (same window and IATA-set filter); zero hours omitted. -SELECT ts.name, s.hour, SUM(s.packets)::bigint AS packets -FROM analytics_hourly_scope_sets s -JOIN transport_scopes ts ON ts.id = s.scope_id -WHERE s.hour >= @since::timestamptz - AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR s.iatas && @iatas::bpchar[]) -GROUP BY ts.name, s.hour -HAVING SUM(s.packets) > 0 -ORDER BY ts.name, s.hour; +-- GetScopeStats packet counts split by hour (same window and IATA-set filter), with the +-- distinct observers and advertising nodes active in each scope that hour. Empty hours omitted. +WITH pk AS ( + SELECT s.scope_id, s.hour, SUM(s.packets)::bigint AS n + FROM analytics_hourly_scope_sets s + WHERE s.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR s.iatas && @iatas::bpchar[]) + GROUP BY s.scope_id, s.hour +), ob AS ( + SELECT o.scope_id, o.hour, COUNT(DISTINCT o.observer_id)::bigint AS n + FROM analytics_hourly_scope_observers o + WHERE o.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR o.iata = ANY(@iatas::bpchar[])) + GROUP BY o.scope_id, o.hour +), nd AS ( + SELECT d.scope_id, d.hour, COUNT(DISTINCT d.origin_pubkey)::bigint AS n + FROM analytics_hourly_scope_nodes d + WHERE d.hour >= @since::timestamptz + AND (COALESCE(cardinality(@iatas::bpchar[]), 0) = 0 OR d.iata = ANY(@iatas::bpchar[])) + GROUP BY d.scope_id, d.hour +), k AS ( + SELECT scope_id, hour FROM pk UNION SELECT scope_id, hour FROM ob UNION SELECT scope_id, hour FROM nd +) +SELECT ts.name, k.hour, + COALESCE(pk.n, 0)::bigint AS packets, + COALESCE(ob.n, 0)::bigint AS observers, + COALESCE(nd.n, 0)::bigint AS nodes +FROM k +JOIN transport_scopes ts ON ts.id = k.scope_id +LEFT JOIN pk ON pk.scope_id = k.scope_id AND pk.hour = k.hour +LEFT JOIN ob ON ob.scope_id = k.scope_id AND ob.hour = k.hour +LEFT JOIN nd ON nd.scope_id = k.scope_id AND nd.hour = k.hour +WHERE COALESCE(pk.n, 0) > 0 OR COALESCE(ob.n, 0) > 0 OR COALESCE(nd.n, 0) > 0 +ORDER BY ts.name, k.hour; -- name: GetScopeStats :many -- Packets since the given hour come from the IATA-set rollup (counted once per hour heard). diff --git a/db/queries/rollup.sql b/db/queries/rollup.sql index 5b336763..889199c9 100644 --- a/db/queries/rollup.sql +++ b/db/queries/rollup.sql @@ -94,8 +94,10 @@ WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour = @ho g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour = @hour::timestamptz), h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour = @hour::timestamptz), i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour = @hour::timestamptz), - j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = @hour::timestamptz) -DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = @hour::timestamptz; + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = @hour::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = @hour::timestamptz), + l AS (DELETE FROM analytics_hourly_scope_observers tl WHERE tl.hour = @hour::timestamptz) +DELETE FROM analytics_hourly_scope_nodes tm WHERE tm.hour = @hour::timestamptz; -- name: RollIATAObservations :exec INSERT INTO analytics_hourly_iata_observations (hour, iata, observation_count) @@ -221,6 +223,16 @@ FROM (SELECT scope_id, array_agg(DISTINCT iata ORDER BY iata) AS iatas GROUP BY packet_hash, scope_id) p GROUP BY iatas, scope_id; +-- name: RollScopeObservers :exec +INSERT INTO analytics_hourly_scope_observers (hour, iata, scope_id, observer_id) +SELECT DISTINCT @hour::timestamptz, iata, scope_id, observer_id +FROM rollup_obs WHERE scope_id IS NOT NULL; + +-- name: RollScopeNodes :exec +INSERT INTO analytics_hourly_scope_nodes (hour, iata, scope_id, origin_pubkey) +SELECT DISTINCT @hour::timestamptz, iata, scope_id, origin_pubkey +FROM rollup_obs WHERE payload_type = 4 AND origin_pubkey IS NOT NULL AND scope_id IS NOT NULL; + -- name: RollupContentHash :one -- Order-independent fingerprint of every family row for the hour. SELECT md5(concat_ws('|', @@ -234,7 +246,9 @@ SELECT md5(concat_ws('|', (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_sets t WHERE t.hour = @hour::timestamptz), (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_talker_sets t WHERE t.hour = @hour::timestamptz), (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_packet_sets t WHERE t.hour = @hour::timestamptz), - (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = @hour::timestamptz) + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_observers t WHERE t.hour = @hour::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_nodes t WHERE t.hour = @hour::timestamptz) ))::text AS content_hash; -- name: FinishRollupHour :one @@ -275,5 +289,7 @@ WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour < @cu j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour < @cutoff::timestamptz), k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour < @cutoff::timestamptz), l AS (DELETE FROM analytics_dirty_hours tl WHERE tl.hour < @cutoff::timestamptz), + n AS (DELETE FROM analytics_hourly_scope_observers tn WHERE tn.hour < @cutoff::timestamptz), + o AS (DELETE FROM analytics_hourly_scope_nodes tob WHERE tob.hour < @cutoff::timestamptz), m AS (DELETE FROM analytics_rollup_hours tm WHERE tm.hour < @cutoff::timestamptz RETURNING 1) UPDATE analytics_raw_state SET coverage = coverage + 1 WHERE EXISTS (SELECT 1 FROM m); diff --git a/db/rollup.go b/db/rollup.go index 3c2cab32..fc8e3fd7 100644 --- a/db/rollup.go +++ b/db/rollup.go @@ -153,6 +153,8 @@ func rollHour(ctx context.Context, tx pgx.Tx, hour time.Time) (RollOutcome, bool "talker sets": q.RollTalkerSets, "packet sets": q.RollPacketSets, "scope sets": q.RollScopeSets, + "scope observers": q.RollScopeObservers, + "scope nodes": q.RollScopeNodes, } { if err := roll(ctx, h); err != nil { return RollSkipped, false, fmt.Errorf("roll %s: %w", name, err) diff --git a/db/rollup_integration_test.go b/db/rollup_integration_test.go index a5cace6f..d71d73bb 100644 --- a/db/rollup_integration_test.go +++ b/db/rollup_integration_test.go @@ -37,7 +37,7 @@ INSERT INTO channels (id, channel_hash) VALUES (1, '\x11'); INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, origin_pubkey, scope_id, raw_payload, raw_header, first_heard_at, last_heard_at) SELECT hash, pt, rt, 0, origin, scope, '\x00', '\x00', $1::timestamptz, $1::timestamptz FROM (VALUES ('\x01'::bytea, 4, 1, '\xee'::bytea, NULL::int), -- flood advert, two IATAs - ('\x02', 4, 2, '\xee', NULL), -- direct advert, one IATA + ('\x02', 4, 2, '\xee', 1), -- direct advert, scoped, one IATA ('\x03', 5, 1, NULL, 1), -- scoped channel message, two IATAs ('\x04', 5, 1, NULL, 1), -- scoped channel message, one IATA ('\x05', 2, 1, NULL, NULL), -- straddles h0 and h0+1h @@ -129,6 +129,15 @@ var rollupOracles = map[string][2]string{ `SELECT hour, iata, observer_id, observation_count, display_name, observer_type FROM analytics_hourly_observer_identity`, `SELECT date_trunc('hour', po.heard_at, 'UTC'), po.iata, po.observer_id, count(*), o.display_name, o.observer_type FROM packet_observations po JOIN observers o ON o.id = po.observer_id GROUP BY 1, 2, 3, 5, 6`}, + "scope observers": { + `SELECT hour, iata, scope_id, observer_id FROM analytics_hourly_scope_observers`, + `SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.iata, p.scope_id, po.observer_id + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash WHERE p.scope_id IS NOT NULL`}, + "scope nodes": { + `SELECT hour, iata, scope_id, origin_pubkey FROM analytics_hourly_scope_nodes`, + `SELECT DISTINCT date_trunc('hour', po.heard_at, 'UTC'), po.iata, p.scope_id, p.origin_pubkey + FROM packet_observations po JOIN packets p ON p.packet_hash = po.packet_hash + WHERE po.payload_type = 4 AND p.origin_pubkey IS NOT NULL AND p.scope_id IS NOT NULL`}, "advert hearings": { `SELECT hour, iata, origin_pubkey, observations, last_heard, name, node_type FROM analytics_hourly_advert_hearings`, `SELECT date_trunc('hour', po.heard_at, 'UTC'), po.iata, p.origin_pubkey, count(*), max(po.heard_at), n.name, n.node_type diff --git a/db/scope_stats_integration_test.go b/db/scope_stats_integration_test.go index 9fd4115f..8a07435d 100644 --- a/db/scope_stats_integration_test.go +++ b/db/scope_stats_integration_test.go @@ -31,15 +31,16 @@ FROM (VALUES (1,'#a'),(2,'#b'),(3,'#unused')) v(i,name); INSERT INTO observers (id,public_key,last_iata) SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),2,'0'),'hex'),iata FROM (VALUES (1,'YVR'),(2,'YYJ'),(3,'YVR'),(4,'YVR'),(5,'YYZ')) v(i,iata); -INSERT INTO packets (packet_hash,scope_id,payload_type,payload_version,route_type,raw_payload,raw_header,first_heard_at,last_heard_at) -SELECT decode(lpad(to_hex(i),2,'0'),'hex'),scope_id,4,0,1,'\x00','\x00',$1::timestamptz,$1::timestamptz +-- every packet is an advert from its own node, so per-scope nodes equal per-scope packets +INSERT INTO packets (packet_hash,scope_id,payload_type,payload_version,route_type,origin_pubkey,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),scope_id,4,0,1,decode(lpad(to_hex(100+i),64,'0'),'hex'),'\x00','\x00',$1::timestamptz,$1::timestamptz FROM (VALUES (1,1),(2,1),(3,1),(4,1),(5,2),(6,2),(7,2),(8,NULL)) v(i,scope_id); INSERT INTO observer_scopes (observer_id,scope_id) SELECT md5(i::text)::uuid,scope_id FROM (VALUES (1,1),(2,1),(4,1),(5,1),(1,2),(2,2),(3,2)) v(i,scope_id); -- packet 4 is never heard, so it never reaches the rollup -INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type) SELECT id,decode(lpad(to_hex(packet),2,'0'),'hex'),md5(observer::text)::uuid,iata, - $1::timestamptz+id*interval '1 second',0,1,0 + $1::timestamptz+id*interval '1 second',0,1,0,4 FROM (VALUES (1,1,1,'YVR'),(2,1,4,'YVR'),(3,1,2,'YYJ'),(4,2,2,'YVR'), (5,3,1,'YYZ'),(6,5,3,'YVR'),(7,6,1,'YYJ'),(8,7,2,'YYZ'),(9,8,1,'YVR')) v(id,packet,observer,iata); INSERT INTO nodes (id,public_key,node_type,default_scope_id) @@ -62,20 +63,21 @@ SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YY r.Close(ctx) for _, tc := range []struct { - name string - iatas []string - since time.Time - a, b [3]int64 // packets, observers, nodes + name string + iatas []string + since time.Time + a, b [3]int64 // packets, observers, nodes + ah, bh int64 // distinct observers heard carrying the scope in hour h }{ - {"global", nil, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}}, - {"empty filter", []string{}, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}}, - {"YVR", []string{"YVR"}, time.Time{}, [3]int64{2, 2, 2}, [3]int64{1, 2, 1}}, - {"YYJ", []string{"YYJ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 1, 2}}, - {"YYZ", []string{"YYZ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 0, 0}}, - {"overlapping IATAs", []string{"YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}}, - {"duplicate IATAs", []string{"YYJ", "YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}}, - {"unknown IATA", []string{"ZZZ"}, time.Time{}, [3]int64{}, [3]int64{}}, - {"window after the traffic", nil, h.Add(time.Hour), [3]int64{0, 4, 4}, [3]int64{0, 3, 2}}, + {"global", nil, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}, 3, 3}, + {"empty filter", []string{}, time.Time{}, [3]int64{3, 4, 4}, [3]int64{3, 3, 2}, 3, 3}, + {"YVR", []string{"YVR"}, time.Time{}, [3]int64{2, 2, 2}, [3]int64{1, 2, 1}, 3, 1}, + {"YYJ", []string{"YYJ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 1, 2}, 1, 1}, + {"YYZ", []string{"YYZ"}, time.Time{}, [3]int64{1, 1, 1}, [3]int64{1, 0, 0}, 1, 1}, + {"overlapping IATAs", []string{"YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}, 3, 2}, + {"duplicate IATAs", []string{"YYJ", "YVR", "YYJ"}, time.Time{}, [3]int64{2, 3, 2}, [3]int64{2, 3, 2}, 3, 2}, + {"unknown IATA", []string{"ZZZ"}, time.Time{}, [3]int64{}, [3]int64{}, 0, 0}, + {"window after the traffic", nil, h.Add(time.Hour), [3]int64{0, 4, 4}, [3]int64{0, 3, 2}, 0, 0}, } { t.Run(tc.name, func(t *testing.T) { rows, err := store.GetScopeStats(ctx, tc.iatas, tc.since) @@ -83,15 +85,16 @@ SELECT md5(i::text)::uuid,iata FROM (VALUES (1,'YVR'),(1,'YYJ'),(2,'YVR'),(3,'YY t.Fatal(err) } // All fixture traffic is in hour h; hours with no packets are omitted. - hourly := func(packets int64) []api.ScopeHour { - if packets == 0 { + // Each packet has its own origin, so active nodes per hour equal packets per hour. + hourly := func(packets, observers int64) []api.ScopeHour { + if packets == 0 && observers == 0 { return []api.ScopeHour{} } - return []api.ScopeHour{{Hour: h.UnixMilli(), Packets: packets}} + return []api.ScopeHour{{Hour: h.UnixMilli(), Packets: packets, Observers: observers, Nodes: packets}} } want := []api.ScopeStats{ - {Name: "#a", PacketCount: tc.a[0], ObserverCount: tc.a[1], NodeCount: tc.a[2], Hourly: hourly(tc.a[0])}, - {Name: "#b", PacketCount: tc.b[0], ObserverCount: tc.b[1], NodeCount: tc.b[2], Hourly: hourly(tc.b[0])}, + {Name: "#a", PacketCount: tc.a[0], ObserverCount: tc.a[1], NodeCount: tc.a[2], Hourly: hourly(tc.a[0], tc.ah)}, + {Name: "#b", PacketCount: tc.b[0], ObserverCount: tc.b[1], NodeCount: tc.b[2], Hourly: hourly(tc.b[0], tc.bh)}, {Name: "#unused", Hourly: []api.ScopeHour{}}, } if !reflect.DeepEqual(rows, want) { diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 5f089cfe..e61bc786 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -1978,6 +1978,34 @@ func (mr *MockQuerierMockRecorder) RollPayloadBreakdown(ctx, hour any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollPayloadBreakdown", reflect.TypeOf((*MockQuerier)(nil).RollPayloadBreakdown), ctx, hour) } +// RollScopeNodes mocks base method. +func (m *MockQuerier) RollScopeNodes(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollScopeNodes", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollScopeNodes indicates an expected call of RollScopeNodes. +func (mr *MockQuerierMockRecorder) RollScopeNodes(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollScopeNodes", reflect.TypeOf((*MockQuerier)(nil).RollScopeNodes), ctx, hour) +} + +// RollScopeObservers mocks base method. +func (m *MockQuerier) RollScopeObservers(ctx context.Context, hour pgtype.Timestamptz) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RollScopeObservers", ctx, hour) + ret0, _ := ret[0].(error) + return ret0 +} + +// RollScopeObservers indicates an expected call of RollScopeObservers. +func (mr *MockQuerierMockRecorder) RollScopeObservers(ctx, hour any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RollScopeObservers", reflect.TypeOf((*MockQuerier)(nil).RollScopeObservers), ctx, hour) +} + // RollScopeSets mocks base method. func (m *MockQuerier) RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 709db8f6..78e14301 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -91,6 +91,20 @@ type AnalyticsHourlyPayloadBreakdown struct { Count int64 `json:"count"` } +type AnalyticsHourlyScopeNode struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + ScopeID int32 `json:"scope_id"` + OriginPubkey []byte `json:"origin_pubkey"` +} + +type AnalyticsHourlyScopeObserver struct { + Hour pgtype.Timestamptz `json:"hour"` + Iata string `json:"iata"` + ScopeID int32 `json:"scope_id"` + ObserverID uuid.UUID `json:"observer_id"` +} + type AnalyticsHourlyScopeSet struct { Hour pgtype.Timestamptz `json:"hour"` Iatas []string `json:"iatas"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 837bf2c5..834628f1 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -122,7 +122,8 @@ type Querier interface { // Packets since the given hour come from the IATA-set rollup (counted once per hour heard). // Observer and node counts are current memberships; observers filter by their latest IATA. GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([]GetScopeStatsRow, error) - // GetScopeStats packet counts split by hour (same window and IATA-set filter); zero hours omitted. + // GetScopeStats packet counts split by hour (same window and IATA-set filter), with the + // distinct observers and advertising nodes active in each scope that hour. Empty hours omitted. GetScopeStatsHourly(ctx context.Context, arg GetScopeStatsHourlyParams) ([]GetScopeStatsHourlyRow, error) // Read the hourly rollup, never observations on an HTTP request. // Weight averages by sample counts instead of averaging regional/hourly means. @@ -317,6 +318,8 @@ type Querier interface { // category: 0 = routed, 1 = zero-hop, 2 = TRACE, 3 = invalid (meshcore-go IsValidPathLen). RollPaths(ctx context.Context, hour pgtype.Timestamptz) error RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestamptz) error + RollScopeNodes(ctx context.Context, hour pgtype.Timestamptz) error + RollScopeObservers(ctx context.Context, hour pgtype.Timestamptz) error RollScopeSets(ctx context.Context, hour pgtype.Timestamptz) error // kind = grouping(snr_bin, rssi_bin): 3 = per IATA totals, 1 = per SNR bin, 2 = per RSSI bin. // rssi=0 AND snr=0 means "not reported". diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index da083763..7c6d7dc3 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -1726,14 +1726,38 @@ func (q *Queries) GetScopeStats(ctx context.Context, arg GetScopeStatsParams) ([ } const getScopeStatsHourly = `-- name: GetScopeStatsHourly :many -SELECT ts.name, s.hour, SUM(s.packets)::bigint AS packets -FROM analytics_hourly_scope_sets s -JOIN transport_scopes ts ON ts.id = s.scope_id -WHERE s.hour >= $1::timestamptz - AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR s.iatas && $2::bpchar[]) -GROUP BY ts.name, s.hour -HAVING SUM(s.packets) > 0 -ORDER BY ts.name, s.hour +WITH pk AS ( + SELECT s.scope_id, s.hour, SUM(s.packets)::bigint AS n + FROM analytics_hourly_scope_sets s + WHERE s.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR s.iatas && $2::bpchar[]) + GROUP BY s.scope_id, s.hour +), ob AS ( + SELECT o.scope_id, o.hour, COUNT(DISTINCT o.observer_id)::bigint AS n + FROM analytics_hourly_scope_observers o + WHERE o.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR o.iata = ANY($2::bpchar[])) + GROUP BY o.scope_id, o.hour +), nd AS ( + SELECT d.scope_id, d.hour, COUNT(DISTINCT d.origin_pubkey)::bigint AS n + FROM analytics_hourly_scope_nodes d + WHERE d.hour >= $1::timestamptz + AND (COALESCE(cardinality($2::bpchar[]), 0) = 0 OR d.iata = ANY($2::bpchar[])) + GROUP BY d.scope_id, d.hour +), k AS ( + SELECT scope_id, hour FROM pk UNION SELECT scope_id, hour FROM ob UNION SELECT scope_id, hour FROM nd +) +SELECT ts.name, k.hour, + COALESCE(pk.n, 0)::bigint AS packets, + COALESCE(ob.n, 0)::bigint AS observers, + COALESCE(nd.n, 0)::bigint AS nodes +FROM k +JOIN transport_scopes ts ON ts.id = k.scope_id +LEFT JOIN pk ON pk.scope_id = k.scope_id AND pk.hour = k.hour +LEFT JOIN ob ON ob.scope_id = k.scope_id AND ob.hour = k.hour +LEFT JOIN nd ON nd.scope_id = k.scope_id AND nd.hour = k.hour +WHERE COALESCE(pk.n, 0) > 0 OR COALESCE(ob.n, 0) > 0 OR COALESCE(nd.n, 0) > 0 +ORDER BY ts.name, k.hour ` type GetScopeStatsHourlyParams struct { @@ -1742,12 +1766,15 @@ type GetScopeStatsHourlyParams struct { } type GetScopeStatsHourlyRow struct { - Name string `json:"name"` - Hour pgtype.Timestamptz `json:"hour"` - Packets int64 `json:"packets"` + Name string `json:"name"` + Hour pgtype.Timestamptz `json:"hour"` + Packets int64 `json:"packets"` + Observers int64 `json:"observers"` + Nodes int64 `json:"nodes"` } -// GetScopeStats packet counts split by hour (same window and IATA-set filter); zero hours omitted. +// GetScopeStats packet counts split by hour (same window and IATA-set filter), with the +// distinct observers and advertising nodes active in each scope that hour. Empty hours omitted. func (q *Queries) GetScopeStatsHourly(ctx context.Context, arg GetScopeStatsHourlyParams) ([]GetScopeStatsHourlyRow, error) { rows, err := q.db.Query(ctx, getScopeStatsHourly, arg.Since, arg.Iatas) if err != nil { @@ -1757,7 +1784,13 @@ func (q *Queries) GetScopeStatsHourly(ctx context.Context, arg GetScopeStatsHour items := []GetScopeStatsHourlyRow{} for rows.Next() { var i GetScopeStatsHourlyRow - if err := rows.Scan(&i.Name, &i.Hour, &i.Packets); err != nil { + if err := rows.Scan( + &i.Name, + &i.Hour, + &i.Packets, + &i.Observers, + &i.Nodes, + ); err != nil { return nil, err } items = append(items, i) diff --git a/db/sqlc/rollup.sql.go b/db/sqlc/rollup.sql.go index db8ee8f0..02f7fd08 100644 --- a/db/sqlc/rollup.sql.go +++ b/db/sqlc/rollup.sql.go @@ -24,6 +24,8 @@ WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour < $1: j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour < $1::timestamptz), k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour < $1::timestamptz), l AS (DELETE FROM analytics_dirty_hours tl WHERE tl.hour < $1::timestamptz), + n AS (DELETE FROM analytics_hourly_scope_observers tn WHERE tn.hour < $1::timestamptz), + o AS (DELETE FROM analytics_hourly_scope_nodes tob WHERE tob.hour < $1::timestamptz), m AS (DELETE FROM analytics_rollup_hours tm WHERE tm.hour < $1::timestamptz RETURNING 1) UPDATE analytics_raw_state SET coverage = coverage + 1 WHERE EXISTS (SELECT 1 FROM m) ` @@ -44,8 +46,10 @@ WITH a AS (DELETE FROM analytics_hourly_iata_observations ta WHERE ta.hour = $1: g AS (DELETE FROM analytics_hourly_advert_hearings tg WHERE tg.hour = $1::timestamptz), h AS (DELETE FROM analytics_hourly_advert_sets th WHERE th.hour = $1::timestamptz), i AS (DELETE FROM analytics_hourly_talker_sets ti WHERE ti.hour = $1::timestamptz), - j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = $1::timestamptz) -DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = $1::timestamptz + j AS (DELETE FROM analytics_hourly_packet_sets tj WHERE tj.hour = $1::timestamptz), + k AS (DELETE FROM analytics_hourly_scope_sets tk WHERE tk.hour = $1::timestamptz), + l AS (DELETE FROM analytics_hourly_scope_observers tl WHERE tl.hour = $1::timestamptz) +DELETE FROM analytics_hourly_scope_nodes tm WHERE tm.hour = $1::timestamptz ` func (q *Queries) DeleteRollupHour(ctx context.Context, hour pgtype.Timestamptz) error { @@ -379,6 +383,28 @@ func (q *Queries) RollPayloadBreakdown(ctx context.Context, hour pgtype.Timestam return err } +const rollScopeNodes = `-- name: RollScopeNodes :exec +INSERT INTO analytics_hourly_scope_nodes (hour, iata, scope_id, origin_pubkey) +SELECT DISTINCT $1::timestamptz, iata, scope_id, origin_pubkey +FROM rollup_obs WHERE payload_type = 4 AND origin_pubkey IS NOT NULL AND scope_id IS NOT NULL +` + +func (q *Queries) RollScopeNodes(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollScopeNodes, hour) + return err +} + +const rollScopeObservers = `-- name: RollScopeObservers :exec +INSERT INTO analytics_hourly_scope_observers (hour, iata, scope_id, observer_id) +SELECT DISTINCT $1::timestamptz, iata, scope_id, observer_id +FROM rollup_obs WHERE scope_id IS NOT NULL +` + +func (q *Queries) RollScopeObservers(ctx context.Context, hour pgtype.Timestamptz) error { + _, err := q.db.Exec(ctx, rollScopeObservers, hour) + return err +} + const rollScopeSets = `-- name: RollScopeSets :exec INSERT INTO analytics_hourly_scope_sets (hour, iatas, scope_id, packets) SELECT $1::timestamptz, iatas, scope_id, count(*) @@ -451,7 +477,9 @@ SELECT md5(concat_ws('|', (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_advert_sets t WHERE t.hour = $1::timestamptz), (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_talker_sets t WHERE t.hour = $1::timestamptz), (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_packet_sets t WHERE t.hour = $1::timestamptz), - (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = $1::timestamptz) + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_sets t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_observers t WHERE t.hour = $1::timestamptz), + (SELECT string_agg(t::text, ',' ORDER BY t::text) FROM analytics_hourly_scope_nodes t WHERE t.hour = $1::timestamptz) ))::text AS content_hash ` diff --git a/db/stats.go b/db/stats.go index 1a061139..8d614ca6 100644 --- a/db/stats.go +++ b/db/stats.go @@ -236,7 +236,7 @@ func (s *Store) GetScopeStats(ctx context.Context, iatas []string, since time.Ti } byScope := make(map[string][]api.ScopeHour) for _, h := range hours { - byScope[h.Name] = append(byScope[h.Name], api.ScopeHour{Hour: h.Hour.Time.UnixMilli(), Packets: h.Packets}) + byScope[h.Name] = append(byScope[h.Name], api.ScopeHour{Hour: h.Hour.Time.UnixMilli(), Packets: h.Packets, Observers: h.Observers, Nodes: h.Nodes}) } items := make([]api.ScopeStats, 0, len(rows)) for _, r := range rows { diff --git a/docs/docs.go b/docs/docs.go index 4f826f36..43664243 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2798,7 +2798,7 @@ const docTemplate = `{ }, "/stats/scopes": { "get": { - "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour (zero hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour and adds the distinct observers and advertising nodes active in the scope each hour (empty hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -5179,6 +5179,14 @@ const docTemplate = `{ "description": "epoch ms, start of the UTC hour", "type": "integer" }, + "nodes": { + "description": "distinct nodes whose adverts were heard in the scope that hour", + "type": "integer" + }, + "observers": { + "description": "distinct observers that heard the scope's packets that hour", + "type": "integer" + }, "packets": { "type": "integer" } diff --git a/docs/historical-stats.md b/docs/historical-stats.md index 72ad3ac8..8c8ebffb 100644 --- a/docs/historical-stats.md +++ b/docs/historical-stats.md @@ -48,6 +48,7 @@ sub-hour observer activity. | Observations, receptions, payload counts | Sum | | Unique packets, adverts, messages, scoped packets | Sum of hourly IATA-set counts: each is counted once per hour it was heard, so one heard across an hour boundary counts twice (measured +1.1%) | | Active observers, IATAs, scopes | Distinct across the window | +| Active scope observers and nodes (per scope, per hour) | Distinct within the hour, across the requested IATAs | | Max path entries | Max | | SNR / RSSI averages | Σ sum / Σ samples, never an average of averages | diff --git a/docs/swagger.json b/docs/swagger.json index 4765d079..0d7a5fc9 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2796,7 +2796,7 @@ }, "/stats/scopes": { "get": { - "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour (zero hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", + "description": "Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour and adds the distinct observers and advertising nodes active in the scope each hour (empty hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array.", "produces": [ "application/json" ], @@ -5177,6 +5177,14 @@ "description": "epoch ms, start of the UTC hour", "type": "integer" }, + "nodes": { + "description": "distinct nodes whose adverts were heard in the scope that hour", + "type": "integer" + }, + "observers": { + "description": "distinct observers that heard the scope's packets that hour", + "type": "integer" + }, "packets": { "type": "integer" } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 3053d3a7..a0963a4a 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1251,6 +1251,12 @@ definitions: hour: description: epoch ms, start of the UTC hour type: integer + nodes: + description: distinct nodes whose adverts were heard in the scope that hour + type: integer + observers: + description: distinct observers that heard the scope's packets that hour + type: integer packets: type: integer type: object @@ -3523,11 +3529,12 @@ paths: description: 'Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, - nodes by their IATA memberships. hourly splits packetCount by UTC hour (zero - hours omitted; outage gaps come from /stats/series hour status). With filters, - lists only manual scopes configured for a matching region and imported scopes - whose current MeshMapper catalogue includes a matching IATA; those remain - listed with zero counts. An empty region returns an empty array.' + nodes by their IATA memberships. hourly splits packetCount by UTC hour and + adds the distinct observers and advertising nodes active in the scope each + hour (empty hours omitted; outage gaps come from /stats/series hour status). + With filters, lists only manual scopes configured for a matching region and + imported scopes whose current MeshMapper catalogue includes a matching IATA; + those remain listed with zero counts. An empty region returns an empty array.' parameters: - description: Comma-separated IATA codes in: query diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index 6a331b35..43164d04 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -457,7 +457,7 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { // getStatsScopes godoc // // @Summary Scope statistics -// @Description Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour (zero hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. +// @Description Packets are those heard since the window start, each counted once per hour however many of the requested IATAs heard it. Observers and nodes are current memberships: observers filter by the IATA they last reported from, nodes by their IATA memberships. hourly splits packetCount by UTC hour and adds the distinct observers and advertising nodes active in the scope each hour (empty hours omitted; outage gaps come from /stats/series hour status). With filters, lists only manual scopes configured for a matching region and imported scopes whose current MeshMapper catalogue includes a matching IATA; those remain listed with zero counts. An empty region returns an empty array. // @Tags Stats // @Produce json // @Param iatas query string false "Comma-separated IATA codes" diff --git a/internal/api/stats.go b/internal/api/stats.go index 8086f441..fe3b6973 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -99,10 +99,12 @@ type ScopeStats struct { Hourly []ScopeHour `json:"hourly"` // packetCount by hour, oldest first; hours with none omitted } -// ScopeHour is one UTC hour of a scope's packets; a scope's hours sum to its packetCount. +// ScopeHour is one UTC hour of a scope's activity; a scope's hours' packets sum to its packetCount. type ScopeHour struct { - Hour int64 `json:"hour"` // epoch ms, start of the UTC hour - Packets int64 `json:"packets"` + Hour int64 `json:"hour"` // epoch ms, start of the UTC hour + Packets int64 `json:"packets"` + Observers int64 `json:"observers"` // distinct observers that heard the scope's packets that hour + Nodes int64 `json:"nodes"` // distinct nodes whose adverts were heard in the scope that hour } // TopNode is a node ranked by how often its adverts were heard in the window. diff --git a/internal/cache/rollup_stats.go b/internal/cache/rollup_stats.go index d5dbb4a8..6ab01488 100644 --- a/internal/cache/rollup_stats.go +++ b/internal/cache/rollup_stats.go @@ -158,8 +158,8 @@ func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, // GetScopeStats implements [api.Reader]. func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string, since time.Time) ([]api.ScopeStats, error) { - // ":h" keeps entries cached before hourly existed from being served without it. - return cachedRollup(ctx, cr, keyScopeStats+":h:", iatas, func(ctx context.Context) ([]api.ScopeStats, error) { + // The suffix changes with the hourly shape so older cached entries aren't served. + return cachedRollup(ctx, cr, keyScopeStats+":h2:", iatas, func(ctx context.Context) ([]api.ScopeStats, error) { return cr.inner.GetScopeStats(ctx, iatas, since) }, hourKey(since)) } From 5ccb1a2ce37680deb37086e7a3ff88a25424bf2a Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 16:48:50 -0400 Subject: [PATCH 128/141] fix(api): warn on untrusted proxies and tighten config validation Warn at startup when limits are on without trusted_proxies, and once when a request carries forwarding headers Beacon ignores. Reject short or whitespace admin keys and negative durations. Key default-window stats by hour, return 500 for scope lookup failures, and validate IATA path params. --- cmd/beacon/main.go | 19 ++++-- cmd/beacon/main_test.go | 16 +++++ internal/api/handlers/iatas.go | 28 +++++++- internal/api/handlers/iatas_test.go | 38 +++++++++++ internal/api/handlers/paths.go | 4 +- internal/api/handlers/regions.go | 4 +- internal/api/handlers/scopes.go | 8 ++- internal/api/handlers/scopes_test.go | 30 +++++++++ internal/api/handlers/signal.go | 4 +- internal/api/handlers/signal_test.go | 4 +- internal/api/middleware/trusted_proxy.go | 21 ++++++ internal/api/middleware/trusted_proxy_test.go | 49 ++++++++++++++ internal/api/reader.go | 2 +- internal/cache/reader.go | 2 + internal/cache/rollup_stats.go | 37 +++++++---- internal/cache/scope_stats_test.go | 28 ++++++++ internal/config/auth_test.go | 37 +++++++++-- internal/config/config.go | 66 ++++++++++++++++++- internal/config/config_test.go | 29 ++++++++ internal/ws/handler.go | 2 +- 20 files changed, 393 insertions(+), 35 deletions(-) diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 800c88aa..dc87e2bf 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -108,11 +108,10 @@ func main() { } slog.SetDefault(logger) slog.Info("beacon starting", "component", "startup", "version", version) - if len(cfg.Server.TrustedProxies) == 0 { - slog.Warn("warning: server.trusted_proxies is empty; client IP headers are ignored and proxied clients share a WebSocket connection limit", "component", "startup") - } - resolved := config.Resolve(cfg) + if msg := proxyLimitWarning(resolved, len(cfg.Server.TrustedProxies)); msg != "" { + slog.Warn(msg, "component", "startup") + } borderFiles, err := config.LoadLocalBorders(cfg) if err != nil { slog.Error("invalid local border configuration", "component", "startup", "error", err) @@ -466,6 +465,18 @@ func main() { coalescer.Flush(shutdownCtx) } +// proxyLimitWarning explains why per-IP limits go site-wide behind an unlisted reverse proxy. +func proxyLimitWarning(r config.ResolvedConfig, trustedProxies int) string { + if trustedProxies > 0 { + return "" + } + limits := "the WebSocket connect limit" + if r.RateLimit.Enabled { + limits = "the REST rate limit and the WebSocket connect limit" + } + return fmt.Sprintf("warning: server.trusted_proxies is empty, so %s key on the direct peer; behind a reverse proxy every visitor shares one budget. List the proxy in server.trusted_proxies and have it set X-Real-IP", limits) +} + // getEnv returns the value of an env var and logs a warning if it is unset. // Callers that require the value to be non-empty should fatal themselves; // ingest workers tolerate missing broker config and will fail on connect instead. diff --git a/cmd/beacon/main_test.go b/cmd/beacon/main_test.go index e62082c5..88f8ee40 100644 --- a/cmd/beacon/main_test.go +++ b/cmd/beacon/main_test.go @@ -11,8 +11,24 @@ import ( "path/filepath" "strings" "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" ) +func TestProxyLimitWarning(t *testing.T) { + limited := config.ResolvedConfig{RateLimit: config.ResolvedRateLimitConfig{Enabled: true}, MaxConnectsPerMinute: 10} + if got := proxyLimitWarning(limited, 0); !strings.Contains(got, "server.trusted_proxies") || !strings.Contains(got, "X-Real-IP") { + t.Errorf("rate limited without proxies: %q", got) + } + wsOnly := config.ResolvedConfig{MaxConnectsPerMinute: 10} + if got := proxyLimitWarning(wsOnly, 0); !strings.Contains(got, "WebSocket") || strings.Contains(got, "REST") { + t.Errorf("WebSocket limit only: %q", got) + } + if got := proxyLimitWarning(limited, 1); got != "" { + t.Errorf("proxies configured: %q", got) + } +} + func TestStartupLogging(t *testing.T) { if os.Getenv("BEACON_TEST_LOG_STARTUP") == "1" { main() diff --git a/internal/api/handlers/iatas.go b/internal/api/handlers/iatas.go index 8524611c..1294c029 100644 --- a/internal/api/handlers/iatas.go +++ b/internal/api/handlers/iatas.go @@ -5,6 +5,7 @@ package handlers import ( "net/http" + "strings" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" @@ -23,6 +24,21 @@ func IATAsRouter(reader api.Reader) http.Handler { return r } +// pathIATA uppercases the {iata} param and accepts only 3 ASCII letters, which also keeps it +// from reaching into other cache keys. +func pathIATA(r *http.Request) (string, bool) { + iata := strings.ToUpper(chi.URLParam(r, "iata")) + if len(iata) != 3 { + return "", false + } + for i := range len(iata) { + if iata[i] < 'A' || iata[i] > 'Z' { + return "", false + } + } + return iata, true +} + // listIATAs godoc // // @Summary List all IATA codes @@ -53,7 +69,11 @@ func listIATAs(reader api.Reader) http.HandlerFunc { // @Router /iatas/{iata} [get] func getIATA(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - iata := chi.URLParam(r, "iata") + iata, ok := pathIATA(r) + if !ok { + respondError(w, http.StatusNotFound, "IATA not found") + return + } result, err := reader.GetIATA(r.Context(), iata) if err != nil { respondError(w, http.StatusNotFound, "IATA not found") @@ -75,7 +95,11 @@ func getIATA(reader api.Reader) http.HandlerFunc { // @Router /iatas/{iata}/border [get] func getIATABorder(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - iata := chi.URLParam(r, "iata") + iata, ok := pathIATA(r) + if !ok { + respondError(w, http.StatusNotFound, "IATA not found") + return + } border, err := reader.GetIATABorder(r.Context(), iata) if err != nil { respondError(w, http.StatusNotFound, "IATA not found") diff --git a/internal/api/handlers/iatas_test.go b/internal/api/handlers/iatas_test.go index 5c7ff31d..b6717710 100644 --- a/internal/api/handlers/iatas_test.go +++ b/internal/api/handlers/iatas_test.go @@ -106,6 +106,44 @@ func TestGetIATABorder_NoContentOnCacheRoundTrippedNull(t *testing.T) { } } +func TestIATAPathValidation(t *testing.T) { + var got []string + reader := stubReader{ + getIATA: func(_ context.Context, iata string) (*api.IATA, error) { + got = append(got, iata) + return &api.IATA{IATA: iata}, nil + }, + getIATABorder: func(_ context.Context, iata string) (json.RawMessage, error) { + got = append(got, iata) + return json.RawMessage(`{"type":"Feature"}`), nil + }, + } + r := chi.NewRouter() + r.Mount("/iatas", IATAsRouter(reader)) + for _, tc := range []struct { + path string + want int + }{ + {"/iatas/border:YOW", http.StatusNotFound}, + {"/iatas/border:YOW/border", http.StatusNotFound}, + {"/iatas/YO", http.StatusNotFound}, + {"/iatas/YOWW", http.StatusNotFound}, + {"/iatas/Y%C3%96W", http.StatusNotFound}, + {"/iatas/Y1W/border", http.StatusNotFound}, + {"/iatas/yow", http.StatusOK}, + {"/iatas/yow/border", http.StatusOK}, + } { + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) + if w.Code != tc.want { + t.Errorf("%s: status = %d, want %d", tc.path, w.Code, tc.want) + } + } + if len(got) != 2 || got[0] != "YOW" || got[1] != "YOW" { + t.Errorf("reader saw %q, want only uppercased valid codes", got) + } +} + func TestGetIATABorder_NotFound(t *testing.T) { r := chi.NewRouter() r.Get("/iatas/{iata}/border", getIATABorder(stubReader{ diff --git a/internal/api/handlers/paths.go b/internal/api/handlers/paths.go index d427d68e..7dfcc2b0 100644 --- a/internal/api/handlers/paths.go +++ b/internal/api/handlers/paths.go @@ -16,7 +16,7 @@ import ( // getPathStats godoc // // @Summary Received path-entry and hash-width distributions -// @Description Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads use materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and missing hours are omitted. +// @Description Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly path rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted. // @Tags Stats // @Produce json // @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" @@ -32,7 +32,7 @@ import ( func getPathStats(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() - since, until, err := parseStatsWindow(r, rawStatsWindow) + since, until, err := parseStatsWindow(r, distributionWindow) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return diff --git a/internal/api/handlers/regions.go b/internal/api/handlers/regions.go index f80df798..f8366e82 100644 --- a/internal/api/handlers/regions.go +++ b/internal/api/handlers/regions.go @@ -75,8 +75,8 @@ func getRegion(reader api.Reader) http.HandlerFunc { } } -// rawStatsWindow is the longest window for endpoints that still aggregate raw-era snapshots. -const rawStatsWindow = 30 * 24 * time.Hour +// distributionWindow caps /stats/signal and /stats/paths, which sum every rolled hour per request. +const distributionWindow = 30 * 24 * time.Hour // parseStatsWindow rounds both endpoints down to UTC hours. Polls within an // hour share a cache key; the response reports the effective window. A valid diff --git a/internal/api/handlers/scopes.go b/internal/api/handlers/scopes.go index b6a2ba9f..9ca791e1 100644 --- a/internal/api/handlers/scopes.go +++ b/internal/api/handlers/scopes.go @@ -4,10 +4,12 @@ package handlers import ( + "errors" "net/http" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) // ScopeMembership lists scopes by configured region and imported catalogue membership. @@ -85,9 +87,13 @@ func getScope(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") scope, err := reader.GetScopeByName(r.Context(), name) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && scope == nil: respondError(w, http.StatusNotFound, "scope not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "failed to load scope") + return } respond(w, http.StatusOK, scope) } diff --git a/internal/api/handlers/scopes_test.go b/internal/api/handlers/scopes_test.go index f83262ee..e23bbc57 100644 --- a/internal/api/handlers/scopes_test.go +++ b/internal/api/handlers/scopes_test.go @@ -13,6 +13,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) func TestListScopes_NoIATAs_OK(t *testing.T) { @@ -45,6 +46,35 @@ func TestGetScope_OK(t *testing.T) { } } +func TestGetScope_Errors(t *testing.T) { + for _, tc := range []struct { + name string + scope *api.ScopeDetail + err error + want int + }{ + {"no rows", nil, pgx.ErrNoRows, http.StatusNotFound}, + {"nil scope", nil, nil, http.StatusNotFound}, + {"database failure", nil, context.DeadlineExceeded, http.StatusInternalServerError}, + } { + t.Run(tc.name, func(t *testing.T) { + r := chi.NewRouter() + r.Get("/scopes/{name}", getScope(stubReader{ + getScopeByName: func(context.Context, string) (*api.ScopeDetail, error) { return tc.scope, tc.err }, + })) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/scopes/%23bc", nil)) + if w.Code != tc.want { + t.Fatalf("status = %d, want %d", w.Code, tc.want) + } + var body map[string]APIError + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || body["error"].Message == "" { + t.Fatalf("not a JSON error: %s", w.Body.String()) + } + }) + } +} + func TestListScopes_FiltersByMembership(t *testing.T) { reader := stubReader{ getScopeNames: func(context.Context) ([]string, error) { diff --git a/internal/api/handlers/signal.go b/internal/api/handlers/signal.go index a5fd8648..34995f00 100644 --- a/internal/api/handlers/signal.go +++ b/internal/api/handlers/signal.go @@ -16,7 +16,7 @@ import ( // getSignalStats godoc // // @Summary Reception signal distributions and hourly trends -// @Description Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss. +// @Description Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly signal rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted. These last-hop readings do not measure end-to-end quality or packet loss. // @Tags Stats // @Produce json // @Param since query int true "Inclusive start, epoch milliseconds (0 through 253402300799999)" @@ -32,7 +32,7 @@ import ( func getSignalStats(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() - since, until, err := parseStatsWindow(r, rawStatsWindow) + since, until, err := parseStatsWindow(r, distributionWindow) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go index 578372b7..7d29d4bc 100644 --- a/internal/api/handlers/signal_test.go +++ b/internal/api/handlers/signal_test.go @@ -101,12 +101,12 @@ func TestSignalCancellationReachesReader(t *testing.T) { func TestStatsWindowSnapsPollingTimes(t *testing.T) { for _, query := range []string{"since=123&until=86400123", "since=3599999&until=89999999"} { - since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?"+query, nil), rawStatsWindow) + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?"+query, nil), distributionWindow) if err != nil || since.UnixMilli() != 0 || until.UnixMilli() != 86400000 { t.Fatalf("%v %v %v", since, until, err) } } - since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?since=1&until=2", nil), rawStatsWindow) + since, until, err := parseStatsWindow(httptest.NewRequest("GET", "/?since=1&until=2", nil), distributionWindow) if err != nil || !since.Equal(until) { t.Fatal("sub-hour normalization") } diff --git a/internal/api/middleware/trusted_proxy.go b/internal/api/middleware/trusted_proxy.go index 606fbc47..96b1942a 100644 --- a/internal/api/middleware/trusted_proxy.go +++ b/internal/api/middleware/trusted_proxy.go @@ -4,29 +4,46 @@ package middleware import ( + "log/slog" "net/http" "net/netip" + "sync" ) // TrustedProxyIP accepts X-Real-IP only from a configured direct proxy peer. // Run it before the access logger and handlers that use RemoteAddr as a key. // All other forwarding headers are ignored, as are invalid/ambiguous values. func TrustedProxyIP(proxies []netip.Prefix) func(http.Handler) http.Handler { + // Misconfigured proxies key every visitor to one IP; say so once, not per request. + var untrustedOnce, noRealIPOnce sync.Once return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { peer, err := netip.ParseAddrPort(r.RemoteAddr) if err == nil { + trusted := false for _, proxy := range proxies { if !proxy.Contains(peer.Addr().Unmap()) { continue } + trusted = true if values := r.Header.Values("X-Real-IP"); len(values) == 1 { if ip, err := netip.ParseAddr(values[0]); err == nil && ip.Zone() == "" { r.RemoteAddr = ip.Unmap().String() } + } else if len(values) == 0 && r.Header.Get("X-Forwarded-For") != "" { + noRealIPOnce.Do(func() { + slog.Warn("trusted proxy sent X-Forwarded-For without X-Real-IP; clients share its rate limits until the proxy sets X-Real-IP (see server.trusted_proxies)", + "component", "http", "peer", peer.Addr().Unmap().String()) + }) } break } + if !trusted && hasForwardingHeader(r.Header) { + untrustedOnce.Do(func() { + slog.Warn("forwarding headers from an untrusted peer were ignored; if it is your reverse proxy, add it to server.trusted_proxies and have it set X-Real-IP, or all clients share its rate limits", + "component", "http", "peer", peer.Addr().Unmap().String()) + }) + } } // Downstream client-IP keys must use the resolved RemoteAddr, not // reinterpret forwarding headers (including those from trusted peers). @@ -37,3 +54,7 @@ func TrustedProxyIP(proxies []netip.Prefix) func(http.Handler) http.Handler { }) } } + +func hasForwardingHeader(h http.Header) bool { + return len(h.Values("X-Forwarded-For")) > 0 || len(h.Values("X-Real-IP")) > 0 || len(h.Values("True-Client-IP")) > 0 +} diff --git a/internal/api/middleware/trusted_proxy_test.go b/internal/api/middleware/trusted_proxy_test.go index 264fe93a..6ac14ba4 100644 --- a/internal/api/middleware/trusted_proxy_test.go +++ b/internal/api/middleware/trusted_proxy_test.go @@ -4,9 +4,12 @@ package middleware import ( + "bytes" + "log/slog" "net/http" "net/http/httptest" "net/netip" + "strings" "testing" ) @@ -59,3 +62,49 @@ func TestTrustedProxyIP(t *testing.T) { }) } } + +func TestTrustedProxyIPWarnsOnce(t *testing.T) { + trusted := []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")} + for _, tc := range []struct { + name, peer, header string + proxies []netip.Prefix + want string // in the single warning; "" means none + }{ + {"untrusted XFF", "198.51.100.7:1234", "X-Forwarded-For", nil, "198.51.100.7"}, + {"untrusted X-Real-IP", "198.51.100.7:1234", "X-Real-IP", trusted, "198.51.100.7"}, + {"untrusted True-Client-IP", "198.51.100.7:1234", "True-Client-IP", nil, "198.51.100.7"}, + {"trusted XFF only", "192.0.2.10:1234", "X-Forwarded-For", trusted, "X-Real-IP"}, + {"trusted with X-Real-IP", "192.0.2.10:1234", "X-Real-IP", trusted, ""}, + {"direct client", "198.51.100.7:1234", "", nil, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&out, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + handler := TrustedProxyIP(tc.proxies)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + for range 3 { + r := httptest.NewRequest(http.MethodGet, "/", nil) + r.RemoteAddr = tc.peer + if tc.header != "" { + r.Header.Set(tc.header, "203.0.113.9") + } + handler.ServeHTTP(httptest.NewRecorder(), r) + } + logs := out.String() + warnings := strings.Count(logs, `"level":"WARN"`) + if tc.want == "" { + if warnings != 0 { + t.Fatalf("unexpected warning: %s", logs) + } + return + } + if warnings != 1 || !strings.Contains(logs, tc.want) || !strings.Contains(logs, "trusted_proxies") { + t.Fatalf("want one warning mentioning %q and trusted_proxies, got: %s", tc.want, logs) + } + if strings.Contains(logs, "203.0.113.9") { + t.Fatal("warning echoes a client-supplied header value") + } + }) + } +} diff --git a/internal/api/reader.go b/internal/api/reader.go index e74a2534..db267190 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -215,7 +215,7 @@ type Reader interface { // GetScopeByName returns full detail for a single scope by its normalized name (e.g. "#bc"), // including packet count, observer count, node count, and the list of IATAs it is active in. - // Returns nil if the scope is not found. + // Returns nil, pgx.ErrNoRows if the scope is not found. GetScopeByName(ctx context.Context, name string) (*ScopeDetail, error) // ListTraceTags returns a paginated list of trace tags with aggregate metadata. diff --git a/internal/cache/reader.go b/internal/cache/reader.go index cac83ed4..a78dc436 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -57,6 +57,7 @@ type CachedReader struct { c *Client ttl CacheTTLs rev revisionMemo + now func() time.Time } // CacheTTLs holds the resolved per-category TTLs for the cache layer. @@ -77,6 +78,7 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { inner: inner, c: c, ttl: ttl, + now: time.Now, } } diff --git a/internal/cache/rollup_stats.go b/internal/cache/rollup_stats.go index 6ab01488..0b271e63 100644 --- a/internal/cache/rollup_stats.go +++ b/internal/cache/rollup_stats.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "slices" + "strconv" "sync" "time" @@ -78,14 +79,26 @@ func cachedRollup[T any](ctx context.Context, cr *CachedReader, prefix string, i return getOrSet(ctx, cr.c, key, cr.ttl.Stats, fetch) } -// hourKey matches the store's hour snapping so polls within an hour share a key; 0 = default window. +// hourKey matches the store's hour snapping so polls within an hour share a key. func hourKey(t time.Time) int64 { - if t.IsZero() { - return 0 - } return t.UTC().Truncate(time.Hour).UnixMilli() } +// sinceKey keys a zero since on the current hour, since the store's default window slides with it. +func (cr *CachedReader) sinceKey(since time.Time) string { + if since.IsZero() { + return fmt.Sprintf("d%d", hourKey(cr.clock())) + } + return strconv.FormatInt(hourKey(since), 10) +} + +func (cr *CachedReader) clock() time.Time { + if cr.now == nil { + return time.Now() + } + return cr.now() +} + // GetStatsSeries implements [api.Reader]. func (cr *CachedReader) GetStatsSeries(ctx context.Context, since, until time.Time, iatas []string) (*api.StatsSeries, error) { return cachedRollup(ctx, cr, keyStatsSeriesPrefix, iatas, func(ctx context.Context) (*api.StatsSeries, error) { @@ -111,49 +124,49 @@ func (cr *CachedReader) GetPathStats(ctx context.Context, since, until time.Time func (cr *CachedReader) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { return cachedRollup(ctx, cr, keyStatsOverviewPrefix, iatas, func(ctx context.Context) (*api.StatsOverview, error) { return cr.inner.GetStatsOverview(ctx, iatas) - }, hourKey(time.Now())) + }, hourKey(cr.clock())) } // GetStatsObservations implements [api.Reader]. func (cr *CachedReader) GetStatsObservations(ctx context.Context, iatas []string, since time.Time) ([]api.ObservationPoint, error) { return cachedRollup(ctx, cr, keyStatsObservationsPrefix, iatas, func(ctx context.Context) ([]api.ObservationPoint, error) { return cr.inner.GetStatsObservations(ctx, iatas, since) - }, hourKey(since)) + }, cr.sinceKey(since)) } // GetStatsPayloadBreakdown implements [api.Reader]. func (cr *CachedReader) GetStatsPayloadBreakdown(ctx context.Context, iatas []string, since time.Time) ([]api.PayloadBreakdownItem, error) { return cachedRollup(ctx, cr, keyStatsBreakdownPrefix, iatas, func(ctx context.Context) ([]api.PayloadBreakdownItem, error) { return cr.inner.GetStatsPayloadBreakdown(ctx, iatas, since) - }, hourKey(since)) + }, cr.sinceKey(since)) } // GetStatsTopNodes implements [api.Reader]. func (cr *CachedReader) GetStatsTopNodes(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopNode, error) { return cachedRollup(ctx, cr, keyStatsTopNodesPrefix, iatas, func(ctx context.Context) ([]api.TopNode, error) { return cr.inner.GetStatsTopNodes(ctx, iatas, since, limit) - }, hourKey(since), limit) + }, cr.sinceKey(since), limit) } // GetStatsTopObservers implements [api.Reader]. func (cr *CachedReader) GetStatsTopObservers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopObserver, error) { return cachedRollup(ctx, cr, keyStatsTopObsPrefix, iatas, func(ctx context.Context) ([]api.TopObserver, error) { return cr.inner.GetStatsTopObservers(ctx, iatas, since, limit) - }, hourKey(since), limit) + }, cr.sinceKey(since), limit) } // GetStatsTopAdvertisers implements [api.Reader]. func (cr *CachedReader) GetStatsTopAdvertisers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopAdvertiser, error) { return cachedRollup(ctx, cr, keyStatsTopAdvPrefix, iatas, func(ctx context.Context) ([]api.TopAdvertiser, error) { return cr.inner.GetStatsTopAdvertisers(ctx, iatas, since, limit) - }, hourKey(since), limit) + }, cr.sinceKey(since), limit) } // GetStatsTopTalkers implements [api.Reader]. func (cr *CachedReader) GetStatsTopTalkers(ctx context.Context, iatas []string, since time.Time, limit int32) ([]api.TopTalker, error) { return cachedRollup(ctx, cr, keyStatsTopTalkersPrefix, iatas, func(ctx context.Context) ([]api.TopTalker, error) { return cr.inner.GetStatsTopTalkers(ctx, iatas, since, limit) - }, hourKey(since), limit) + }, cr.sinceKey(since), limit) } // GetScopeStats implements [api.Reader]. @@ -161,7 +174,7 @@ func (cr *CachedReader) GetScopeStats(ctx context.Context, iatas []string, since // The suffix changes with the hourly shape so older cached entries aren't served. return cachedRollup(ctx, cr, keyScopeStats+":h2:", iatas, func(ctx context.Context) ([]api.ScopeStats, error) { return cr.inner.GetScopeStats(ctx, iatas, since) - }, hourKey(since)) + }, cr.sinceKey(since)) } // GetScopeByName implements [api.Reader]. diff --git a/internal/cache/scope_stats_test.go b/internal/cache/scope_stats_test.go index 2975ad6b..6e07d386 100644 --- a/internal/cache/scope_stats_test.go +++ b/internal/cache/scope_stats_test.go @@ -59,3 +59,31 @@ func TestScopeStatsCacheSeparatesIATAs(t *testing.T) { t.Fatalf("underlying calls = %d, want 6", inner.calls) } } + +func TestDefaultWindowKeyFollowsTheHour(t *testing.T) { + c, _ := newTestClient(t) + inner := &scopeStatsReader{} + cr := NewCachedReader(inner, c, CacheTTLs{Stats: time.Hour}).(*CachedReader) + clock := time.Date(2026, 10, 2, 10, 30, 0, 0, time.UTC) + cr.now = func() time.Time { return clock } + get := func(since time.Time) int64 { + t.Helper() + rows, err := cr.GetScopeStats(context.Background(), nil, since) + if err != nil { + t.Fatal(err) + } + return rows[0].PacketCount + } + if get(time.Time{}) != 1 || get(time.Time{}) != 1 { + t.Fatal("default window not cached within the hour") + } + clock = clock.Add(35 * time.Minute) + if get(time.Time{}) != 2 { + t.Fatal("default window served from the previous hour") + } + // An explicit since at the current hour is a different window. + if get(clock.Truncate(time.Hour)) != 3 { + t.Fatal("explicit since collided with the default window") + } + +} diff --git a/internal/config/auth_test.go b/internal/config/auth_test.go index e3fccf58..fb7e02ba 100644 --- a/internal/config/auth_test.go +++ b/internal/config/auth_test.go @@ -17,10 +17,12 @@ func TestLoadAuth(t *testing.T) { file, set bool }{ {"no config", "", "", false, false}, - {"YAML", "", "yaml-test-key", true, false}, - {"environment overrides", "env-test-key", "env-test-key", true, true}, + {"YAML", "", "yaml-test-key-0123456789", true, false}, + {"environment overrides", "env-test-key-0123456789", "env-test-key-0123456789", true, true}, {"empty environment disables", "", "", true, true}, - {"environment without file", "env-test-key", "env-test-key", false, true}, + {"environment without file", "env-test-key-0123456789", "env-test-key-0123456789", false, true}, + {"surrounding whitespace trimmed", " env-test-key-0123456789\n", "env-test-key-0123456789", true, true}, + {"whitespace-only disables", " \t", "", true, true}, } { t.Run(tc.name, func(t *testing.T) { t.Setenv("BEACON_API_KEY", tc.env) @@ -31,7 +33,7 @@ func TestLoadAuth(t *testing.T) { } path := filepath.Join(t.TempDir(), "config.yaml") if tc.file { - if err := os.WriteFile(path, []byte("auth:\n api_key: yaml-test-key\n"), 0600); err != nil { + if err := os.WriteFile(path, []byte("auth:\n api_key: yaml-test-key-0123456789\n"), 0600); err != nil { t.Fatal(err) } } @@ -52,3 +54,30 @@ func TestLoadAuth(t *testing.T) { }) } } + +func TestLoadAuthRejectsUnusableKey(t *testing.T) { + for _, tc := range []struct{ name, key, want string }{ + {"inner space", "env-test-key 0123456789", "whitespace"}, + {"inner tab", "env-test-key\t0123456789", "whitespace"}, + {"too short", "short-test-key", "at least 16"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("BEACON_API_KEY", tc.key) + for _, file := range []bool{false, true} { + path := filepath.Join(t.TempDir(), "config.yaml") + if file { + if err := os.WriteFile(path, []byte("log:\n level: info\n"), 0600); err != nil { + t.Fatal(err) + } + } + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("file=%t: err = %v, want %q", file, err, tc.want) + } + if strings.Contains(err.Error(), "test-key") { + t.Fatal("key exposed in error") + } + } + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 1ad309e2..94b109e8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -14,6 +14,7 @@ import ( "slices" "strings" "time" + "unicode" "gopkg.in/yaml.v3" ) @@ -202,7 +203,7 @@ type CacheConfig struct { type CacheTTLsConfig struct { // Stats controls the TTL for aggregated network statistics endpoints // (overview, observations, payload breakdown, top nodes/observers, radio presets, scope stats). - // These are backed by materialized views refreshed hourly, so values under 1m are rarely useful. + // Rollup-backed keys also change with the rollup revision. Stats duration `yaml:"stats"` // Reference controls the TTL for mostly-static reference data @@ -230,7 +231,7 @@ type ScopeConfig struct { // TelemetryConfig controls observer telemetry storage behaviour. type TelemetryConfig struct { // Retention is how long telemetry rows are kept before the cleanup job removes them. - // Defaults to 672h (4 weeks) if not set. + // Defaults to 744h (31 days) if not set. Retention duration `yaml:"retention"` // Resolution is how frequently a telemetry snapshot is stored per observer. @@ -393,6 +394,9 @@ func Load(path string) (*Config, error) { if err != nil { if os.IsNotExist(err) { cfg.Auth.APIKey = os.Getenv("BEACON_API_KEY") + if err := cfg.validateAPIKey(); err != nil { + return nil, err + } return cfg, nil } return nil, err @@ -403,6 +407,12 @@ func Load(path string) (*Config, error) { if value, set := os.LookupEnv("BEACON_API_KEY"); set { cfg.Auth.APIKey = value } + if err := cfg.validateAPIKey(); err != nil { + return nil, err + } + if err := cfg.validateDurations(); err != nil { + return nil, err + } for i, prefix := range cfg.Server.TrustedProxies { if !prefix.IsValid() { return nil, fmt.Errorf("server.trusted_proxies[%d] must be a valid CIDR", i) @@ -447,6 +457,58 @@ func Load(path string) (*Config, error) { return cfg, nil } +// minAPIKeyLength keeps the admin key out of guessing range. +const minAPIKeyLength = 16 + +// validateAPIKey trims the key, since Bearer tokens can't carry whitespace; errors never echo it. +func (c *Config) validateAPIKey() error { + c.Auth.APIKey = strings.TrimSpace(c.Auth.APIKey) + switch key := c.Auth.APIKey; { + case key == "": + return nil + case strings.IndexFunc(key, unicode.IsSpace) >= 0: + return fmt.Errorf("auth.api_key / BEACON_API_KEY must not contain whitespace") + case len(key) < minAPIKeyLength: + return fmt.Errorf("auth.api_key / BEACON_API_KEY must be at least %d characters", minAPIKeyLength) + } + return nil +} + +// validateDurations rejects negative intervals, which would otherwise panic tickers or invert cutoffs. +// observers.delete_after is exempt: nonpositive disables it. +func (c *Config) validateDurations() error { + for _, d := range []struct { + key string + value duration + }{ + {"presence.flush_interval", c.Presence.FlushInterval}, + {"presence.packet_ttl", c.Presence.PacketTTL}, + {"background.view_refresh", c.Background.ViewRefresh}, + {"background.reconfirm", c.Background.Reconfirm}, + {"background.cleanup", c.Background.Cleanup}, + {"cache.ttl", c.Cache.TTL}, + {"cache.ttls.stats", c.Cache.TTLs.Stats}, + {"cache.ttls.reference", c.Cache.TTLs.Reference}, + {"cache.ttls.nodes", c.Cache.TTLs.Nodes}, + {"cache.ttls.observers", c.Cache.TTLs.Observers}, + {"telemetry.retention", c.Telemetry.Retention}, + {"telemetry.resolution", c.Telemetry.Resolution}, + {"routes.retention", c.Routes.Retention}, + {"routes.grace", c.Routes.Grace}, + {"nodes.clock_drift_threshold", c.Nodes.ClockDriftThreshold}, + {"nodes.stale_threshold", c.Nodes.StaleThreshold}, + {"nodes.delete_after", c.Nodes.DeleteAfter}, + {"meshmapper.scopes.refresh_interval", c.MeshMapper.Scopes.RefreshInterval}, + {"meshmapper.zones.refresh_interval", c.MeshMapper.Zones.RefreshInterval}, + {"meshmapper.channels.refresh_interval", c.MeshMapper.Channels.RefreshInterval}, + } { + if d.value.Duration < 0 { + return fmt.Errorf("%s must not be negative (omit it or use 0 for the default)", d.key) + } + } + return nil +} + // validateOrigin allows exact origins or a leading "*." subdomain wildcard; the WebSocket // library treats entries as path.Match patterns, so every other pattern character is rejected. func validateOrigin(origin string) error { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f13bbf61..f7716de2 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -272,3 +272,32 @@ func TestLoad_AnalyticsRollupRetentionTooShort(t *testing.T) { t.Errorf("expected error about analytics.rollup_retention, got: %v", err) } } + +func TestLoad_NegativeDurationsRejected(t *testing.T) { + for _, key := range []string{ + "background:\n cleanup: -1h", "background:\n view_refresh: -1h", "background:\n reconfirm: -1m", + "presence:\n flush_interval: -1s", "presence:\n packet_ttl: -1s", + "telemetry:\n resolution: -1h", "telemetry:\n retention: -1h", + "routes:\n retention: -1h", "routes:\n grace: -1h", + "nodes:\n clock_drift_threshold: -1m", "nodes:\n stale_threshold: -1h", "nodes:\n delete_after: -1h", + "cache:\n ttl: -1m", "cache:\n ttls:\n stats: -1m", + "meshmapper:\n scopes:\n refresh_interval: -1h", + } { + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte(key+"\n"), 0600); err != nil { + t.Fatal(err) + } + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), "must not be negative") { + t.Errorf("%q: err = %v, want a negative-duration error", key, err) + } + } + // Nonpositive still means "disabled" here. + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte("observers:\n delete_after: -1h\n"), 0600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err != nil { + t.Errorf("observers.delete_after: %v", err) + } +} diff --git a/internal/ws/handler.go b/internal/ws/handler.go index dd9c5453..4e93995d 100644 --- a/internal/ws/handler.go +++ b/internal/ws/handler.go @@ -24,7 +24,7 @@ // // Server → Client events (unsolicited): // packetObservation, observerStatus, nodeUpdate, channelMessage -// lagged { v, type, droppedCount, since, lastObservationId } +// lagged { v, type, droppedCount, since } // error { v, type, code, message } // // Idle connections (no ping) closed after 90s. From 52fc3e1e483ea481bb479e5816d996fc1e458945 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 16:48:56 -0400 Subject: [PATCH 129/141] fix(ingest): validate topic keys and harden against malformed input A short topic pubkey panicked a lane worker and killed the process; keys must now be 32 bytes and lane workers recover. Also reject non-finite radio values before airtime, strip NULs from advert names and neighbour scopes, guard the MQTT client pointer, and never stream a broker copy of a hearing as a repeat. --- internal/hub/hub.go | 4 +- internal/hub/hub_test.go | 19 +++--- internal/hub/sent.go | 21 ++++-- internal/ingest/ingest.go | 29 ++++++-- internal/ingest/mqtt_client_test.go | 30 ++++++++ internal/ingest/neighbors.go | 5 +- internal/ingest/packet.go | 4 +- internal/ingest/queue.go | 14 +++- internal/ingest/repeats_test.go | 20 +++++- internal/ingest/sanitize_test.go | 102 ++++++++++++++++++++++++++++ internal/ingest/side_effects.go | 9 ++- internal/ingest/status.go | 29 +++++++- internal/ingest/topic_test.go | 79 +++++++++++++++++++++ internal/lora/airtime.go | 6 +- internal/lora/airtime_test.go | 4 ++ 15 files changed, 338 insertions(+), 37 deletions(-) create mode 100644 internal/ingest/mqtt_client_test.go create mode 100644 internal/ingest/sanitize_test.go create mode 100644 internal/ingest/topic_test.go diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 28061264..7f67480b 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -243,8 +243,8 @@ func (h *Hub) RepeatsWanted() bool { return h.repeatClients.Load() > 0 } -// MarkSent records every hearing's path and reports whether it is new within the TTL, -// so broker copies and same-path duplicates are never streamed as repeats. +// MarkSent records every hearing's path and reports whether it is a later, new path for a +// packet the observer already heard, so broker copies and same-path duplicates never stream. func (h *Hub) MarkSent(packetHash, observerID, path []byte) bool { return h.sent.mark(packetHash, observerID, path, time.Now()) } diff --git a/internal/hub/hub_test.go b/internal/hub/hub_test.go index 94be9687..bb8fc7a9 100644 --- a/internal/hub/hub_test.go +++ b/internal/hub/hub_test.go @@ -516,20 +516,23 @@ func TestSentPaths(t *testing.T) { start := time.Now() s := newSentPaths(time.Minute, 3) hash, observer := []byte{1, 2}, []byte{3} - if !s.mark(hash, observer, []byte{0xaa}, start) { - t.Fatal("first hearing refused") + if s.mark(hash, observer, []byte{0xaa}, start) { + t.Fatal("first hearing counted as a repeat") } if s.mark(hash, observer, []byte{0xaa}, start.Add(time.Second)) { - t.Fatal("exact copy within the window allowed") + t.Fatal("broker copy of the first hearing counted as a repeat") } if !s.mark(hash, observer, []byte{0xaa, 0xbb}, start.Add(time.Second)) { t.Fatal("new path refused") } - if !s.mark(hash, []byte{4}, []byte{0xaa}, start.Add(time.Second)) { - t.Fatal("other observer refused") + if s.mark(hash, observer, []byte{0xaa, 0xbb}, start.Add(time.Second)) { + t.Fatal("exact copy within the window allowed") + } + if s.mark(hash, []byte{4}, []byte{0xaa}, start.Add(time.Second)) { + t.Fatal("other observer's first hearing counted as a repeat") } - if !s.mark(hash, observer, []byte{0xaa}, start.Add(time.Minute)) { - t.Fatal("copy after the window refused") + if s.mark(hash, observer, []byte{0xcc}, start.Add(time.Minute)) { + t.Fatal("hearing after the window not treated as a first hearing") } s = newSentPaths(time.Hour, 3) @@ -539,7 +542,7 @@ func TestSentPaths(t *testing.T) { if len(s.at) != 3 || len(s.order) != 3 { t.Fatalf("size bound: %d keys, %d queued", len(s.at), len(s.order)) } - if !s.mark(hash, observer, []byte{0}, start) { + if _, ok := s.at[s.key(hash, observer, []byte{0})]; ok { t.Fatal("oldest key not evicted at the size bound") } } diff --git a/internal/hub/sent.go b/internal/hub/sent.go index 45ac8015..f6ae3ed9 100644 --- a/internal/hub/sent.go +++ b/internal/hub/sent.go @@ -34,18 +34,29 @@ func newSentPaths(ttl time.Duration, max int) *sentPaths { return &sentPaths{seed: maphash.MakeSeed(), ttl: ttl, max: max, at: make(map[uint64]time.Time)} } -// mark records the hearing and reports whether it was not already recorded within the TTL. +// mark records the hearing and reports whether it is a new path for a (packet, observer) already +// heard over another one. Keying the first hearing by pair, not by which worker won the insert, +// means a broker copy of it is never a repeat however the two workers interleave. func (s *sentPaths) mark(packetHash, observerID, path []byte, now time.Time) bool { + pair, hearing := s.key(packetHash, observerID), s.key(packetHash, observerID, path) + s.mu.Lock() + defer s.mu.Unlock() + firstHearing := s.add(pair, now) + return s.add(hearing, now) && !firstHearing +} + +func (s *sentPaths) key(parts ...[]byte) uint64 { var h maphash.Hash h.SetSeed(s.seed) - for _, part := range [][]byte{packetHash, observerID, path} { + for _, part := range parts { h.WriteByte(byte(len(part))) h.Write(part) } - key := h.Sum64() + return h.Sum64() +} - s.mu.Lock() - defer s.mu.Unlock() +// add records key unless already present within the TTL and reports whether it was new. +func (s *sentPaths) add(key uint64, now time.Time) bool { for len(s.order) > 0 && (now.Sub(s.order[0].at) >= s.ttl || len(s.order) >= s.max) { delete(s.at, s.order[0].key) s.order = s.order[1:] diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 9ca35846..cf00b0c1 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -39,11 +39,13 @@ import ( "context" "crypto/rand" "encoding/binary" + "encoding/hex" "encoding/json" "fmt" "log/slog" "strconv" "strings" + "sync/atomic" "time" mqtt "github.com/eclipse/paho.mqtt.golang" @@ -226,7 +228,7 @@ type Worker struct { hub *hub.Hub keys ChannelKeyStore scopes ScopeStore - client mqtt.Client + client atomic.Pointer[mqtt.Client] // read by /brokers while Start sets it capabilities capabilityCache onNodeUpsert func(ctx context.Context, nodeID uuid.UUID) onObserverUpsert func(ctx context.Context, observerID uuid.UUID) @@ -289,9 +291,10 @@ func (w *Worker) Start(ctx context.Context) { w.log.Warn("connection lost, will reconnect", "error", err) }) - w.client = mqtt.NewClient(opts) + client := mqtt.NewClient(opts) + w.client.Store(&client) - token := w.client.Connect() + token := client.Connect() ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() connected := token.Done() @@ -305,7 +308,7 @@ func (w *Worker) Start(ctx context.Context) { case <-ticker.C: reportDrops() case <-ctx.Done(): - w.client.Disconnect(500) + client.Disconnect(500) return } } @@ -317,10 +320,11 @@ func (w *Worker) BrokerName() string { } func (w *Worker) IsConnected() bool { - if w.client == nil { + c := w.client.Load() + if c == nil { return false } - return w.client.IsConnected() + return (*c).IsConnected() } func (w *Worker) SetCacheInvalidators(onNode, onObserver func(ctx context.Context, id uuid.UUID)) { @@ -356,6 +360,15 @@ func isValidIATA(s string) bool { return true } +// isValidPubkeyHex reports whether s is a 32-byte public key in hex. +func isValidPubkeyHex(s string) bool { + if len(s) != 64 { + return false + } + _, err := hex.DecodeString(s) + return err == nil +} + // handleMessage dispatches incoming MQTT messages by subtopic. // Each message is processed with a 30s timeout to prevent slow DB calls // from holding a processing worker indefinitely. @@ -377,6 +390,10 @@ func (w *Worker) handleMessageContext(parent context.Context, msg mqtt.Message) w.log.Debug("dropped packet with malformed IATA", "iata", iata, "topic", msg.Topic()) return } + if !isValidPubkeyHex(pubkeyHex) { + w.log.Debug("dropped message with malformed observer pubkey", "pubkey", pubkeyHex, "topic", msg.Topic()) + return + } // Drop packets from IATAs outside the configured geographic filter. if w.cfg.AllowedIATAs != nil { diff --git a/internal/ingest/mqtt_client_test.go b/internal/ingest/mqtt_client_test.go new file mode 100644 index 00000000..096dd89e --- /dev/null +++ b/internal/ingest/mqtt_client_test.go @@ -0,0 +1,30 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/hub" +) + +// Run with -race: /brokers reads the client while Start creates it. +func TestIsConnectedWhileStarting(t *testing.T) { + w := New(Config{BrokerName: "test", URL: "tcp://127.0.0.1:1"}, &stubDB{}, hub.New(), &stubKeys{}, &stubScopes{}) + ctx, cancel := context.WithCancel(context.Background()) + stopped := make(chan struct{}) + go func() { w.Start(ctx); close(stopped) }() + deadline := time.Now().Add(100 * time.Millisecond) + for time.Now().Before(deadline) { + w.IsConnected() + } + cancel() + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("worker did not stop") + } +} diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go index 65795603..3972c652 100644 --- a/internal/ingest/neighbors.go +++ b/internal/ingest/neighbors.go @@ -55,7 +55,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra // self.scopes is always known (it's the observer's own config, not an OTA // query), so this is an unconditional write -- unlike the per-neighbor // scopes below, there's no "query failed" case to protect against here. - if err := w.db.UpdateObserverRegionScope(ctx, observerID, report.Self.Scopes); err != nil { + if err := w.db.UpdateObserverRegionScope(ctx, observerID, cleanText(report.Self.Scopes)); err != nil { w.log.Error(fmt.Sprintf("db: update observer region scope failed for %s", pubkeyHex), "error", err) } @@ -94,7 +94,8 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra // was last known. var regionScope *string if n.Status == "responded" { - regionScope = &n.Scopes + scope := cleanText(n.Scopes) + regionScope = &scope } if err := w.db.UpsertNodeNeighbor(ctx, observerNodeID, neighborNodeID, iata, &snr, regionScope); err != nil { diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 8f56c819..e8754925 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -427,7 +427,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ var name *string if hasName { - n := strings.ToValidUTF8(appData.Name, "\uFFFD") + n := cleanText(appData.Name) name = &n if n != "" { summary = name @@ -747,7 +747,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ now := time.Now().UTC() diff := heardAt.UTC().Sub(now) if diff > 30*time.Minute || diff < -30*time.Minute { - w.log.Debug(fmt.Sprintf("clamping suspicious timestamp %s (diff %v) for pubkey %s", envelope.Timestamp, diff, pubkeyHex[:8])) + w.log.Debug(fmt.Sprintf("clamping suspicious timestamp %s (diff %v) for pubkey %s", envelope.Timestamp, diff, pubkeyHex)) heardAt = now } } diff --git a/internal/ingest/queue.go b/internal/ingest/queue.go index ff7e6d5e..6a537c53 100644 --- a/internal/ingest/queue.go +++ b/internal/ingest/queue.go @@ -5,6 +5,8 @@ package ingest import ( "context" + "log/slog" + "runtime/debug" "strings" "sync" @@ -33,7 +35,7 @@ func newMessageQueue(workers, capacity, maxBytes int, handle func(context.Contex if ctx.Err() != nil { return } - handle(ctx, m) + handleSafely(ctx, m, handle) q.mu.Lock() q.bytes -= len(m.Topic()) + len(m.Payload()) q.mu.Unlock() @@ -44,6 +46,16 @@ func newMessageQueue(workers, capacity, maxBytes int, handle func(context.Contex return q } +// handleSafely keeps one bad message from killing the lane, and with it the process. +func handleSafely(ctx context.Context, m mqtt.Message, handle func(context.Context, mqtt.Message)) { + defer func() { + if r := recover(); r != nil { + slog.Error("ingest handler panicked", "component", "ingest", "topic", m.Topic(), "panic", r, "stack", string(debug.Stack())) + } + }() + handle(ctx, m) +} + func (q *messageQueue) enqueue(m mqtt.Message) bool { // Keep status and packets from each observer in arrival order. parts := strings.SplitN(m.Topic(), "/", 4) diff --git a/internal/ingest/repeats_test.go b/internal/ingest/repeats_test.go index 178b8f01..2ad85a98 100644 --- a/internal/ingest/repeats_test.go +++ b/internal/ingest/repeats_test.go @@ -123,18 +123,32 @@ func TestHandlePacket_PathMarkedRightAfterInsert(t *testing.T) { packet.Path = []byte{0x11} packet.PathLength = 1 hash := packet.PacketHash() - var fresh bool + var repeat bool r.db.dbHook = func() { // stubDB.UpsertObserver returns uuid.Nil, so that is the observer id ingest marks with - fresh = r.w.hub.MarkSent(hash[:], uuid.Nil[:], packet.Path) + repeat = r.w.hub.MarkSent(hash[:], uuid.Nil[:], []byte{0x22}) } r.db.observationInserted = true r.w.handlePacket(r.ctx, "YOW", "0102", packetEnvelope(r.t, packet)) - if fresh { + if !repeat { t.Fatal("path was not marked right after the insert") } } +// A broker copy can lose the insert race yet mark its path first; it is still the first hearing. +func TestHandlePacket_BrokerCopyMarkedFirstIsNotARepeat(t *testing.T) { + r := newRepeatHarness(t, true) + if got := r.hear(false, 0x11); len(got) != 0 { + t.Fatalf("broker copy streamed as a repeat: %s", got) + } + if got := r.hear(true, 0x11); len(got) != 1 || strings.Contains(string(got[0]), "isRepeat") { + t.Fatalf("inserted hearing: %s", got) + } + if got := r.hear(false, 0x22, 0x11); len(got) != 1 { + t.Fatalf("later path sent %d events", len(got)) + } +} + func TestHandlePacket_PathsRecordedWithoutSubscribers(t *testing.T) { r := newRepeatHarness(t, false) if got := r.hear(true, 0x11); len(got) != 1 { diff --git a/internal/ingest/sanitize_test.go b/internal/ingest/sanitize_test.go new file mode 100644 index 00000000..dc269866 --- /dev/null +++ b/internal/ingest/sanitize_test.go @@ -0,0 +1,102 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/meshcore-go/meshcore-go" +) + +type statusCaptureDB struct { + *stubDB + status []UpdateObserverStatusParams +} + +func (s *statusCaptureDB) UpdateObserverStatus(_ context.Context, p UpdateObserverStatusParams) (uuid.UUID, error) { + s.status = append(s.status, p) + return uuid.Nil, nil +} + +func TestHandleStatus_RejectsInsaneRadio(t *testing.T) { + for _, tc := range []struct { + radio string + freq, bw float32 + }{ + {"910.5,62.5,7,5", 910.5, 62.5}, + {"910.5,NaN,7,5", 910.5, 0}, + {"Inf,62.5,7,5", 0, 62.5}, + {"910.5,1e-40,7,5", 910.5, 0}, + {"-910.5,-62.5,7,5", 0, 0}, + {"1e9,1e9,7,5", 0, 0}, + } { + t.Run(tc.radio, func(t *testing.T) { + w, base := newTestWorker() + db := &statusCaptureDB{stubDB: base} + w.db = db + w.handleStatus(context.Background(), strings.Repeat("01", 32), []byte(`{"radio":"`+tc.radio+`"}`)) + if len(db.status) != 1 { + t.Fatal("status not stored") + } + if got := db.status[0]; got.RadioFreqMHz != tc.freq || got.RadioBWKHz != tc.bw { + t.Fatalf("radio stored as %v MHz / %v kHz, want %v / %v", got.RadioFreqMHz, got.RadioBWKHz, tc.freq, tc.bw) + } + }) + } +} + +func TestHandlePacket_AdvertNameNULsStripped(t *testing.T) { + w, base := newTestWorker() + db := &frameCaptureDB{stubDB: base} + w.db = db + data := append([]byte{meshcore.AdvertTypeRepeater | meshcore.AdvertNameMask}, "Base\x00Camp\x00\x00"...) + w.handlePacket(context.Background(), "YOW", "0102", packetEnvelope(t, buildAdvertPacketWithData(t, data, false))) + if len(db.packets) != 1 || bytes.Contains(db.packets[0].ParsedPayload, []byte(`\u0000`)) { + t.Fatalf("parsed payload kept NULs: %s", db.packets[0].ParsedPayload) + } + if base.upsertNodeCalls != 1 || base.upsertNodeParams.Name != "BaseCamp" { + t.Fatalf("node name stored as %q", base.upsertNodeParams.Name) + } +} + +type neighborCaptureDB struct { + *stubDB + selfScope string + scopes []string +} + +func (s *neighborCaptureDB) GetNodeByPubkey(_ context.Context, pubkey []byte) (uuid.UUID, error) { + if len(pubkey) == 0 { + return uuid.Nil, errors.New("not found") + } + return uuid.UUID{pubkey[0]}, nil +} + +func (s *neighborCaptureDB) UpdateObserverRegionScope(_ context.Context, _ uuid.UUID, scope string) error { + s.selfScope = scope + return nil +} + +func (s *neighborCaptureDB) UpsertNodeNeighbor(_ context.Context, _, _ uuid.UUID, _ string, _ *float32, scope *string) error { + if scope != nil { + s.scopes = append(s.scopes, *scope) + } + return nil +} + +func TestHandleNeighbors_ScopeNULsStripped(t *testing.T) { + w, base := newTestWorker() + db := &neighborCaptureDB{stubDB: base} + w.db = db + report := `{"self":{"scopes":"#ott\u0000"},"neighbors":[{"pubkey":"` + strings.Repeat("02", 32) + `","status":"responded","scopes":"#y\u0000ow"}]}` + w.handleNeighbors(context.Background(), "YOW", strings.Repeat("01", 32), []byte(report)) + if db.selfScope != "#ott" || len(db.scopes) != 1 || db.scopes[0] != "#yow" { + t.Fatalf("scopes stored as %q %q", db.selfScope, db.scopes) + } +} diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index c92095f3..3299027f 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -7,7 +7,6 @@ import ( "context" "encoding/hex" "fmt" - "strings" "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" @@ -106,7 +105,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc } params := UpsertNodeParams{ PublicKey: advert.PublicKey.PublicKeyBytes(), - Name: strings.ToValidUTF8(advert.AppData().Name, "\uFFFD"), + Name: cleanText(advert.AppData().Name), NodeType: advert.Type(), Latitude: lat, Longitude: lon, @@ -181,7 +180,7 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc evt := nodeUpdateEvent{ NodeID: nodeID.String(), PublicKey: pubkeyHex, - Name: advert.AppData().Name, + Name: cleanText(advert.AppData().Name), NodeType: advert.Type(), NodeTypeName: api.NodeTypeName(int16(advert.Type())), IATA: iata, @@ -230,8 +229,8 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc ChannelID: result.ChannelID, ChannelHash: hex.EncodeToString(channelHashBytes), PacketHash: hex.EncodeToString(packetHash), - SenderName: strings.ReplaceAll(strings.ToValidUTF8(result.Payload.Sender, "\uFFFD"), "\x00", ""), - Content: strings.ReplaceAll(strings.ToValidUTF8(result.Payload.Text, "\uFFFD"), "\x00", ""), + SenderName: cleanText(result.Payload.Sender), + Content: cleanText(result.Payload.Text), SentAt: time.Unix(int64(result.Payload.Timestamp), 0).UnixMilli(), } w.broadcast(hub.EventChannelMessage, iata, 0, fmt.Sprintf("%02x", grpTxt.ChannelHash), evt) diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 8dde6129..2e6d8cd0 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -15,6 +15,7 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/MeshCore-Beacon/beacon-server/internal/lora" ) // UpdateObserverStatusParams carries the fields parsed from a /status message. @@ -72,6 +73,25 @@ func (s statusStats) usable() bool { return s.NoiseFloor != 0 || s.TxAirSecs != 0 || s.RxAirSecs != 0 } +// LoRa radios span 137 MHz sub-GHz to 2.5 GHz SX128x. +const ( + minRadioFreqMHz = 100 + maxRadioFreqMHz = 3000 +) + +// parseRadioFloat rejects NaN, Inf and anything outside [lo, hi]; observers send free text and +// one bad value would poison every airtime the observer is later costed with. +func parseRadioFloat(s string, lo, hi float64) (float64, error) { + f, err := strconv.ParseFloat(s, 32) + if err != nil { + return 0, err + } + if !(f >= lo && f <= hi) { + return 0, fmt.Errorf("radio value %q outside %g..%g", s, lo, hi) + } + return f, nil +} + // stripNULs removes NUL bytes, which Postgres text columns reject, and returns the // names of the fields that had any. func stripNULs(fields map[string]*string) []string { @@ -86,6 +106,11 @@ func stripNULs(fields map[string]*string) []string { return dirty } +// cleanText makes over-the-air text storable: invalid UTF-8 replaced, NULs dropped. +func cleanText(s string) string { + return strings.ReplaceAll(strings.ToValidUTF8(s, "�"), "\x00", "") +} + // stripJSONNULs drops \u0000 from a JSON document's strings, which jsonb rejects. // Payloads without one are returned untouched. func stripJSONNULs(raw []byte) ([]byte, bool) { @@ -197,13 +222,13 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) if len(radio) != 4 { w.log.Warn(fmt.Sprintf("missing or malformed radio params in status from %s, skipping radio fields", pubkeyHex)) } else { - freq, err := strconv.ParseFloat(radio[0], 32) + freq, err := parseRadioFloat(radio[0], minRadioFreqMHz, maxRadioFreqMHz) if err != nil { w.log.Warn(fmt.Sprintf("error parsing radio freq in status from %s", pubkeyHex), "error", err) } else { params.RadioFreqMHz = float32(freq) } - bw, err := strconv.ParseFloat(radio[1], 32) + bw, err := parseRadioFloat(radio[1], lora.MinBWKHz, lora.MaxBWKHz) if err != nil { w.log.Warn(fmt.Sprintf("error parsing radio bw in status from %s", pubkeyHex), "error", err) } else { diff --git a/internal/ingest/topic_test.go b/internal/ingest/topic_test.go new file mode 100644 index 00000000..452c7fe4 --- /dev/null +++ b/internal/ingest/topic_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/google/uuid" +) + +type observerCountDB struct { + *frameCaptureDB + observers int +} + +func (s *observerCountDB) UpsertObserver(context.Context, []byte, string) (uuid.UUID, string, error) { + s.observers++ + return uuid.Nil, "", nil +} + +func TestTopicPubkeyMustBe32Bytes(t *testing.T) { + var envelope map[string]string + if err := json.Unmarshal(packetEnvelope(t, buildGrpTxtPacket(t, 0x1a, make([]byte, 16))), &envelope); err != nil { + t.Fatal(err) + } + envelope["timestamp"] = time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) // takes the clamp log path + body, err := json.Marshal(envelope) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"ab", strings.Repeat("ab", 31), strings.Repeat("ab", 33), strings.Repeat("zz", 32)} { + for _, sub := range []string{"packets", "status", "neighbors"} { + t.Run(sub+"/"+key, func(t *testing.T) { + w, base := newTestWorker() + db := &observerCountDB{frameCaptureDB: &frameCaptureDB{stubDB: base}} + w.db = db + w.handleMessage(queuedTestMessage{malformedTopicMessage{"meshcore/YOW/" + key + "/" + sub}, body}) + if db.observers != 0 || len(db.packets) != 0 { + t.Fatalf("malformed pubkey processed: %d observers, %d packets", db.observers, len(db.packets)) + } + }) + } + } + w, base := newTestWorker() + db := &observerCountDB{frameCaptureDB: &frameCaptureDB{stubDB: base}} + w.db = db + w.handleMessage(queuedTestMessage{malformedTopicMessage{"meshcore/YOW/" + strings.Repeat("AB", 32) + "/packets"}, body}) + if len(db.packets) != 1 { + t.Fatal("valid pubkey dropped") + } +} + +func TestQueueSurvivesHandlerPanic(t *testing.T) { + done := make(chan struct{}) + q := newMessageQueue(1, 8, 4096, func(_ context.Context, m mqtt.Message) { + if string(m.Payload()) == "0" { + panic("bad message") + } + close(done) + }) + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = q.close(ctx) + }) + q.enqueue(queueMessage("aa", "packets", 0)) + q.enqueue(queueMessage("aa", "packets", 1)) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("lane stopped after a panic") + } +} diff --git a/internal/lora/airtime.go b/internal/lora/airtime.go index 1bd8a159..5abd63c3 100644 --- a/internal/lora/airtime.go +++ b/internal/lora/airtime.go @@ -12,6 +12,10 @@ const ( maxSF = 12 minCR = 5 maxCR = 8 + + // MinBWKHz and MaxBWKHz bracket every LoRa bandwidth (7.8 kHz sub-GHz to 1625 kHz SX128x). + MinBWKHz = 7 + MaxBWKHz = 1625 ) // PreambleSymbols mirrors MeshCore's preambleLengthForSF: 32 symbols at SF<=8, 16 above. @@ -34,7 +38,7 @@ func FrameLength(transportCodes bool, pathLen, payloadLen int) int { // TimeOnAirMs is the Semtech LoRa time-on-air (RadioLib getTimeOnAir: CRC on, explicit header, // LDRO auto at t_sym >= 16 ms). ok is false when any parameter is outside the costable range. func TimeOnAirMs(frameLen, sf int, bwKHz float64, cr int) (ms float64, ok bool) { - if frameLen <= 0 || sf < minSF || sf > maxSF || bwKHz <= 0 || cr < minCR || cr > maxCR { + if frameLen <= 0 || sf < minSF || sf > maxSF || !(bwKHz >= MinBWKHz && bwKHz <= MaxBWKHz) || cr < minCR || cr > maxCR { return 0, false } tSym := math.Exp2(float64(sf)) / bwKHz diff --git a/internal/lora/airtime_test.go b/internal/lora/airtime_test.go index 795357bd..d67d898a 100644 --- a/internal/lora/airtime_test.go +++ b/internal/lora/airtime_test.go @@ -104,6 +104,10 @@ func TestTimeOnAirMsNotCostable(t *testing.T) { {"sf below range", 50, 6, 62.5, 5}, {"sf above range", 50, 13, 62.5, 5}, {"bw zero", 50, 7, 0, 5}, + {"bw NaN", 50, 7, math.NaN(), 5}, + {"bw infinite", 50, 7, math.Inf(1), 5}, + {"bw denormal", 50, 12, 1e-40, 5}, + {"bw above range", 50, 7, 1e9, 5}, {"cr below range", 50, 7, 62.5, 4}, {"cr above range", 50, 7, 62.5, 9}, {"empty frame", 0, 7, 62.5, 5}, From ec0037860c0332e479a8fbce57605e3035b8b134 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 16:48:56 -0400 Subject: [PATCH 130/141] fix(meshmapper): count cut-short requests and qualify groups by heard IATAs Record each attempt before sending so a request lost to a deadline still waits out MeshMapper's limit, and stop holding the directory lock across HTTP. Groups now qualify only through heard or configured IATAs, and one missing country list only skips pruning. --- db/meshmapper.go | 9 +++ internal/meshmapper/channels.go | 7 ++ internal/meshmapper/channels_test.go | 22 +++++++ internal/meshmapper/directory.go | 69 +++++++++++++------- internal/meshmapper/groups.go | 54 ++++++++++++---- internal/meshmapper/groups_test.go | 52 +++++++++++++++ internal/meshmapper/scopes.go | 7 ++ internal/meshmapper/scopes_test.go | 17 +++++ internal/meshmapper/zones.go | 23 +++++-- internal/meshmapper/zones_test.go | 95 +++++++++++++++++++++++++++- 10 files changed, 309 insertions(+), 46 deletions(-) diff --git a/db/meshmapper.go b/db/meshmapper.go index 8ce90d1b..82e2d59d 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -77,6 +77,15 @@ func (s *Store) ListKnownIATAs(ctx context.Context) ([]string, error) { return iatas, nil } +// ListHeardIATAs returns the IATAs observers last reported from. +func (s *Store) ListHeardIATAs(ctx context.Context) ([]string, error) { + rows, err := s.pool.Query(ctx, `SELECT DISTINCT last_iata::text FROM observers WHERE last_iata IS NOT NULL ORDER BY 1`) + if err != nil { + return nil, err + } + return pgx.CollectRows(rows, pgx.RowTo[string]) +} + func (s *Store) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { if keep == nil { keep = []string{} diff --git a/internal/meshmapper/channels.go b/internal/meshmapper/channels.go index 4ead380d..33a07e30 100644 --- a/internal/meshmapper/channels.go +++ b/internal/meshmapper/channels.go @@ -178,6 +178,13 @@ func (c *Channels) refresh(ctx context.Context, s *channelSource, now time.Time) if len(s.cache.Payload) > 0 { etag = s.cache.ETag } + // Recorded first: an abandoned request may still have used the region's call. + attempt := Cache{AttemptedAt: now, NextAttempt: now.Add(min(c.interval, channelRetry)), LastError: "no response"} + s.cache.NextAttempt = attempt.NextAttempt + if err := c.store.SaveChannelCatalogue(ctx, s.iata, s.url, attempt, nil); err != nil { + return fmt.Errorf("persist channel catalogue %s: %w", s.iata, err) + } + s.cache.AttemptedAt, s.cache.LastError = now, attempt.LastError status, body, header, err := get(ctx, c.client, "Beacon-MeshMapper-Channels/1", s.url, etag, MaxChannelBody) if err != nil { return err diff --git a/internal/meshmapper/channels_test.go b/internal/meshmapper/channels_test.go index 5a3def56..29773aa7 100644 --- a/internal/meshmapper/channels_test.go +++ b/internal/meshmapper/channels_test.go @@ -232,3 +232,25 @@ func TestChannelsDisabledClearsMembers(t *testing.T) { t.Fatal("disabled import kept membership or sources") } } + +func TestChannelsRequestCutShortStillCounts(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() })) + defer server.Close() + store := newChannelMemoryStore("YOW") + c, err := NewChannels(context.Background(), config.MeshMapperChannelsConfig{Enabled: true}, store, NewDirectory(newZoneListMemory()), keystore.NewMapKeyStore(nil)) + if err != nil { + t.Fatal(err) + } + s := &c.sources[0] + s.url = server.URL + now := time.Now().UTC() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _ = c.refresh(ctx, s, now) + if got := s.cache.NextAttempt.Sub(now); got != channelRetry { + t.Fatal("abandoned request retried inside the rate limit", got) + } + if row := store.rows[sourceKey{"YOW", server.URL}]; row.NextAttempt.Sub(now) != channelRetry { + t.Fatal("abandoned request not persisted", row) + } +} diff --git a/internal/meshmapper/directory.go b/internal/meshmapper/directory.go index 20f7503f..5113f02b 100644 --- a/internal/meshmapper/directory.go +++ b/internal/meshmapper/directory.go @@ -36,6 +36,7 @@ type zoneList struct { groups map[string]zoneGroup etag string fetchedAt, nextAttempt time.Time + fetching bool } // ZoneList is one country's saved get_zones.php response and fetch state. @@ -89,31 +90,45 @@ func (d *Directory) Restore(ctx context.Context) error { } // List returns the country's fresh zone list. fetched reports that this call -// spent the caller's one request; nil zones without fetched means back off. +// spent the caller's one request; nil zones without fetched means back off, +// including while another task's request for the country is in flight. func (d *Directory) List(ctx context.Context, country string, now time.Time) (zones map[string]zoneEntry, fetched bool, err error) { d.mu.Lock() - defer d.mu.Unlock() list := d.lists[country] if list != nil && list.zones != nil && now.Sub(list.fetchedAt) < zoneListFresh { + d.mu.Unlock() return list.zones, false, nil } - if list != nil && now.Before(list.nextAttempt) { + if list != nil && (list.fetching || now.Before(list.nextAttempt)) { + d.mu.Unlock() return nil, false, nil } - return nil, true, d.fetch(ctx, country, now) -} - -func (d *Directory) fetch(ctx context.Context, country string, now time.Time) error { - list := d.lists[country] if list == nil { list = &zoneList{} d.lists[country] = list } - endpoint := d.listURL + "?country=" + url.QueryEscape(country) - etag := "" - if list.zones != nil { + // Every request counts against the country's 23.5h allowance, failed, abandoned or not. + list.fetching, list.nextAttempt = true, now.Add(zoneListFresh) + cached, etag := list.zones != nil, "" + if cached { etag = list.etag } + d.mu.Unlock() + defer func() { + d.mu.Lock() + list.fetching = false + d.mu.Unlock() + }() + return nil, true, d.fetch(ctx, list, country, cached, etag, now) +} + +// fetch runs without d.mu so other tasks never wait out this request. +func (d *Directory) fetch(ctx context.Context, list *zoneList, country string, cached bool, etag string, now time.Time) error { + attempt := ZoneList{Country: country, AttemptedAt: now, NextAttempt: now.Add(zoneListFresh), LastError: "no response"} + if err := d.store.SaveZoneList(ctx, attempt); err != nil { + return fmt.Errorf("persist MeshMapper zone list %s: %w", country, err) + } + endpoint := d.listURL + "?country=" + url.QueryEscape(country) status, body, header, err := get(ctx, d.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxZoneList) if err != nil { return err @@ -121,25 +136,28 @@ func (d *Directory) fetch(ctx context.Context, country string, now time.Time) er problem := "" var payload json.RawMessage var retryAt time.Time + var zones map[string]zoneEntry + var groups map[string]zoneGroup switch status { case http.StatusOK: - zones, groups, decodeErr := decodeZones(body, country) - if decodeErr != nil { + var decodeErr error + if zones, groups, decodeErr = decodeZones(body, country); decodeErr != nil { problem = "invalid response" } else { - list.zones, list.groups, list.etag = zones, groups, header.Get("ETag") payload = body - d.version++ } case http.StatusNotModified: - if list.zones == nil { + if !cached { problem = "304 without cached list" } default: problem = statusProblem(status, header, now, &retryAt) } - // Every request counts against the country's 23.5h allowance, failed or not. - list.nextAttempt = now.Add(zoneListFresh) + d.mu.Lock() + if payload != nil { + list.zones, list.groups, list.etag = zones, groups, header.Get("ETag") + d.version++ + } if retryAt.After(list.nextAttempt) { list.nextAttempt = retryAt } @@ -148,28 +166,31 @@ func (d *Directory) fetch(ctx context.Context, country string, now time.Time) er list.fetchedAt = now saved.FetchedAt, saved.ETag = now, list.etag } + nZones, nGroups := len(list.zones), len(list.groups) + d.mu.Unlock() level := slog.LevelInfo if problem != "" { level = slog.LevelWarn } slog.Log(ctx, level, "MeshMapper zone list checked", "component", "meshmapper.zones", "country", country, - "zones", len(list.zones), "groups", len(list.groups), "next_attempt", list.nextAttempt, "last_error", problem) + "zones", nZones, "groups", nGroups, "next_attempt", saved.NextAttempt, "last_error", problem) if err := d.store.SaveZoneList(ctx, saved); err != nil { return fmt.Errorf("persist MeshMapper zone list %s: %w", country, err) } return nil } -// snapshot merges every loaded list's zones and groups by code. complete is false until -// each country has a list, so a cold start can't look like every group disappearing. +// snapshot merges every loaded list's zones and groups by code. complete is false while +// a country has no list, so a cold start or failed list can't look like groups disappearing. func (d *Directory) snapshot(countries []string) (merged map[string]zoneGroup, zones map[string]zoneEntry, version int, complete bool) { d.mu.Lock() defer d.mu.Unlock() - merged, zones = map[string]zoneGroup{}, map[string]zoneEntry{} + merged, zones, complete = map[string]zoneGroup{}, map[string]zoneEntry{}, true for _, country := range countries { list := d.lists[country] if list == nil || list.zones == nil { - return nil, nil, d.version, false + complete = false + continue } for code, z := range list.zones { zones[code] = z @@ -184,7 +205,7 @@ func (d *Directory) snapshot(countries []string) (merged map[string]zoneGroup, z merged[code] = g } } - return merged, zones, d.version, true + return merged, zones, d.version, complete } var ( diff --git a/internal/meshmapper/groups.go b/internal/meshmapper/groups.go index 588e1ee6..9a18154a 100644 --- a/internal/meshmapper/groups.go +++ b/internal/meshmapper/groups.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "log/slog" + "maps" "slices" "strings" ) @@ -55,9 +56,15 @@ func (z *Zones) pruneGroups(ctx context.Context) error { return nil } +// syncInputs is what syncDirectory last applied; it reruns when any of them changes. +type syncInputs struct { + version, known int + complete bool + heard string +} + // syncDirectory applies the loaded zone lists to IATA details and, when enabled, -// imported regions. It only runs once every tracked country has a list, and again -// when a list or the IATA set changes. +// imported regions. Groups are only pruned once every tracked country has a list. func (z *Zones) syncDirectory(ctx context.Context, iatas []string) error { var countries []string for _, r := range z.regions { @@ -65,18 +72,27 @@ func (z *Zones) syncDirectory(ctx context.Context, iatas []string) error { } slices.Sort(countries) groups, zones, version, complete := z.dir.snapshot(slices.Compact(countries)) - if !complete || (z.groupsSynced && version == z.groupsVersion && len(iatas) == z.groupsKnown) { + var heard []string + if z.importGroups { + var err error + if heard, err = z.store.ListHeardIATAs(ctx); err != nil { + return fmt.Errorf("list heard IATAs for MeshMapper groups: %w", err) + } + slices.Sort(heard) + } + inputs := syncInputs{version: version, known: len(iatas), complete: complete, heard: strings.Join(heard, ",")} + if z.groupsSynced && inputs == z.synced { return nil } if err := z.syncIATADetails(ctx, zones); err != nil { return err } if z.importGroups { - if err := z.syncGroups(ctx, iatas, groups, zones); err != nil { + if err := z.syncGroups(ctx, heard, groups, zones, complete); err != nil { return err } } - z.groupsSynced, z.groupsVersion, z.groupsKnown = true, version, len(iatas) + z.groupsSynced, z.synced = true, inputs return nil } @@ -111,10 +127,15 @@ func (z *Zones) syncIATADetails(ctx context.Context, zones map[string]zoneEntry) return nil } -func (z *Zones) syncGroups(ctx context.Context, iatas []string, groups map[string]zoneGroup, zones map[string]zoneEntry) error { - known := make(map[string]bool, len(iatas)) - for _, iata := range iatas { - known[iata] = true +// syncGroups imports groups with a member that is heard or configured, never one only +// its own import created. Without every country's list it adds and updates but never prunes. +func (z *Zones) syncGroups(ctx context.Context, heard []string, groups map[string]zoneGroup, zones map[string]zoneEntry, complete bool) error { + inUse := maps.Clone(z.configured) + if inUse == nil { + inUse = map[string]bool{} + } + for _, iata := range heard { + inUse[iata] = true } state, err := z.store.ListRegionState(ctx) if err != nil { @@ -125,11 +146,14 @@ func (z *Zones) syncGroups(ctx context.Context, iatas []string, groups map[strin stored[r.Slug] = r if !r.Imported { handOrder = max(handOrder, r.DisplayOrder) + for _, iata := range r.IATAs { + inUse[iata] = true + } } } var want []RegionState for code, g := range groups { - if slices.ContainsFunc(g.members, func(m string) bool { return known[m] }) { + if slices.ContainsFunc(g.members, func(m string) bool { return inUse[m] }) { lat, lng := center(g.members, zones) want = append(want, RegionState{Slug: strings.ToLower(code), Name: g.name, Imported: true, IATAs: g.members, CenterLat: lat, CenterLng: lng}) } @@ -157,11 +181,13 @@ func (z *Zones) syncGroups(ctx context.Context, iatas []string, groups map[strin keep, changed = append(keep, r.Slug), true slog.Info("MeshMapper region imported", "component", "meshmapper.zones", "slug", r.Slug, "name", r.Name, "iatas", r.IATAs) } - pruned, err := z.store.PruneImportedRegions(ctx, keep) - if err != nil { - return fmt.Errorf("prune MeshMapper regions: %w", err) + var pruned []string + if complete { + if pruned, err = z.store.PruneImportedRegions(ctx, keep); err != nil { + return fmt.Errorf("prune MeshMapper regions: %w", err) + } + z.regionsRemoved(pruned) } - z.regionsRemoved(pruned) if changed || len(pruned) > 0 { z.regionsChanged(ctx) } diff --git a/internal/meshmapper/groups_test.go b/internal/meshmapper/groups_test.go index 4caefa3c..fa1d853c 100644 --- a/internal/meshmapper/groups_test.go +++ b/internal/meshmapper/groups_test.go @@ -32,6 +32,9 @@ func newGroupHarness(t *testing.T, f *fakeMeshMapper, store *zoneMemoryStore, im if store.iatas == nil { store.iatas = []string{"YOW"} } + if store.heard == nil { + store.heard = slices.Clone(store.iatas) + } if store.rows == nil { store.rows = map[string]Boundary{} } @@ -182,3 +185,52 @@ func TestZoneListNamesAndLocatesIATAs(t *testing.T) { t.Fatal("unchanged details rewritten", h.store.writes, changed) } } + +func TestGroupStopsQualifyingOnImportedMembers(t *testing.T) { + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{}, true) + h.refetch(t) + if _, ok := h.store.regions["onqc"]; !ok { + t.Fatal("group not imported") + } + without := strings.Replace(groupedList, `"members":["YUL","YOW","YQB"]`, `"members":["YUL","YQB"]`, 1) + h.f.list = strings.ReplaceAll(without, "URL", h.f.URL) + h.refetch(t) + if _, ok := h.store.regions["onqc"]; ok { + t.Fatal("group kept only by members its own import created") + } +} + +func TestGroupQualifiesThroughConfiguredRegion(t *testing.T) { + hand := RegionState{Slug: "east", Name: "East", IATAs: []string{"YYZ"}} + h := newGroupHarness(t, groupedFake(t), &zoneMemoryStore{regions: map[string]RegionState{"east": hand}}, true) + h.refetch(t) + if _, ok := h.store.regions["golm"]; !ok { + t.Fatal("group with a configured member not imported") + } +} + +func TestGroupImportsFromListsItHas(t *testing.T) { + old := RegionState{Slug: "old", Name: "Old", Imported: true, IATAs: []string{"YOW"}} + store := &zoneMemoryStore{iatas: []string{"YOW", "SEA"}, regions: map[string]RegionState{"old": old}} + f := groupedFake(t) + f.otherStatus = 500 + h := newGroupHarness(t, f, store, true) + h.refetch(t) + for range 2 { + if err := h.z.Refresh(context.Background()); err != nil { + t.Fatal(err) + } + } + if f.otherStatus != 500 || h.z.dir.lists["US"] == nil || h.z.dir.lists["US"].zones != nil { + t.Fatal("US list should have failed") + } + if _, ok := store.regions["onqc"]; !ok { + t.Fatal("one failed country blocked group import") + } + if store.details["YOW"].Name != "Ottawa" { + t.Fatal("one failed country blocked IATA details", store.details) + } + if _, ok := store.regions["old"]; !ok { + t.Fatal("pruned with a country list missing") + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go index 3ecdcbe1..4099aae1 100644 --- a/internal/meshmapper/scopes.go +++ b/internal/meshmapper/scopes.go @@ -195,6 +195,13 @@ func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error if len(s.cache.Payload) > 0 && s.cache.ETag != "" { request.Header.Set("If-None-Match", s.cache.ETag) } + // Recorded first: an abandoned request may still have used the region's call. + attempt := Cache{AttemptedAt: now, NextAttempt: now.Add(min(i.interval, failureRetry)), LastError: "no response"} + s.cache.NextAttempt = attempt.NextAttempt + if err := i.store.SaveScopeCatalogue(ctx, s.iata, s.url, attempt, nil); err != nil { + return fmt.Errorf("persist scope catalogue %s: %w", s.iata, err) + } + s.cache.AttemptedAt, s.cache.LastError = now, attempt.LastError response, err := i.client.Do(request) var entries []scopestore.Entry var generated time.Time diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go index 48104f5a..cd645508 100644 --- a/internal/meshmapper/scopes_test.go +++ b/internal/meshmapper/scopes_test.go @@ -408,3 +408,20 @@ func TestScopesSkipUnlistedIATAs(t *testing.T) { t.Fatalf("unlisted IATA requested or retried early: hits=%d %+v", f.scopeHits, s.cache) } } + +func TestScopesRequestCutShortStillCounts(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() })) + defer server.Close() + store := newMemoryStore("YOW") + imp := newImporter(t, config.MeshMapperScopesConfig{Enabled: true}, store, server.URL, scopestore.New()) + now := time.Now().UTC() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _ = imp.refresh(ctx, &imp.sources[0], now) + if got := imp.sources[0].cache.NextAttempt.Sub(now); got != failureRetry { + t.Fatal("abandoned request retried inside the rate limit", got) + } + if row := store.rows[sourceKey{"YOW", server.URL}]; row.NextAttempt.Sub(now) != failureRetry { + t.Fatal("abandoned request not persisted", row) + } +} diff --git a/internal/meshmapper/zones.go b/internal/meshmapper/zones.go index 1310b712..4bec3897 100644 --- a/internal/meshmapper/zones.go +++ b/internal/meshmapper/zones.go @@ -28,7 +28,7 @@ const ( // MeshMapper asks clients to allow 60-120s; a request given up on may still use the day's call. requestTimeout = 60 * time.Second - refreshTimeout = requestTimeout + 30*time.Second // room for the DB write after a slow request + refreshTimeout = requestTimeout + 30*time.Second // room for the DB writes around a slow request zoneListFresh = config.MinZonesRefresh // get_zones.php allows one request per country per 23.5h zoneFailureRetry = config.MinZonesRefresh // failed requests count too @@ -47,6 +47,8 @@ type Boundary struct { type ZoneStore interface { ListKnownIATAs(ctx context.Context) ([]string, error) + // ListHeardIATAs returns IATAs observers report from; group imports create IATAs too. + ListHeardIATAs(ctx context.Context) ([]string, error) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) ListZoneBoundaries(ctx context.Context) ([]Boundary, error) SaveZoneBoundary(ctx context.Context, b Boundary) error @@ -75,12 +77,12 @@ type Zones struct { onChange func(ctx context.Context, iata string) onUpdate func(imported map[string]json.RawMessage) - importGroups bool - groupsSynced bool - groupsVersion, groupsKnown int - onRegions func(ctx context.Context) - configured map[string]bool - onIATAs func(ctx context.Context) + importGroups bool + groupsSynced bool + synced syncInputs + onRegions func(ctx context.Context) + configured map[string]bool + onIATAs func(ctx context.Context) } func NewZones(cfg config.MeshMapperZonesConfig, store ZoneStore, dir *Directory) *Zones { @@ -218,6 +220,13 @@ func (z *Zones) refresh(ctx context.Context, r *zoneRegion, zones map[string]zon if r.b.Feature != nil { etag = r.b.ETag } + // Recorded first: an abandoned request may still have used the region's call. + attempt := Boundary{IATA: r.b.IATA, URL: update.URL, AttemptedAt: now, NextAttempt: now.Add(min(z.interval, zoneFailureRetry)), LastError: "no response"} + r.b.NextAttempt = attempt.NextAttempt + if err := z.store.SaveZoneBoundary(ctx, attempt); err != nil { + return fmt.Errorf("persist MeshMapper boundary %s: %w", r.b.IATA, err) + } + r.b.URL, r.b.AttemptedAt, r.b.LastError = attempt.URL, now, attempt.LastError status, body, header, err := get(ctx, z.client, "Beacon-MeshMapper-Zones/1", endpoint, etag, MaxBoundaryBody) if err != nil { return err diff --git a/internal/meshmapper/zones_test.go b/internal/meshmapper/zones_test.go index bdf4589e..b4bb35fb 100644 --- a/internal/meshmapper/zones_test.go +++ b/internal/meshmapper/zones_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "slices" "strings" + "sync/atomic" "testing" "time" @@ -28,6 +29,7 @@ func boundaryBody(code, geometry string) string { type zoneMemoryStore struct { iatas []string + heard []string rows map[string]Boundary pruned []string fail bool @@ -96,6 +98,10 @@ func (s *zoneMemoryStore) ListKnownIATAs(context.Context) ([]string, error) { return slices.Clone(s.iatas), nil } +func (s *zoneMemoryStore) ListHeardIATAs(context.Context) ([]string, error) { + return slices.Clone(s.heard), nil +} + func (s *zoneMemoryStore) PruneZoneBoundaries(_ context.Context, keep []string) ([]string, error) { var removed []string for iata := range s.rows { @@ -162,16 +168,28 @@ type fakeMeshMapper struct { etag, retryAfter string listCalls, calls int ifNoneMatch []string + otherStatus int // get_zones.php status for countries other than CA; 0 is unexpected + hang atomic.Bool // requests wait until the client gives up + hung atomic.Int32 } func newFakeMeshMapper(t *testing.T) *fakeMeshMapper { f := &fakeMeshMapper{listStatus: 200, status: 200, etag: `"v1"`, boundary: boundaryBody("YOW", square)} f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if f.hang.Load() { + f.hung.Add(1) + <-r.Context().Done() + return + } switch r.URL.Path { case "/get_zones.php": f.listCalls++ if r.URL.Query().Get("country") != "CA" { - t.Errorf("country query: %s", r.URL.RawQuery) + if f.otherStatus == 0 { + t.Errorf("country query: %s", r.URL.RawQuery) + } + w.WriteHeader(f.otherStatus) + return } w.WriteHeader(f.listStatus) _, _ = w.Write([]byte(f.list)) @@ -492,3 +510,78 @@ func TestClientsAllowMeshMapperTimeout(t *testing.T) { t.Fatal("a refresh must outlast its request") } } + +func TestZoneListCutShortStillCounts(t *testing.T) { + f := newFakeMeshMapper(t) + f.hang.Store(true) + lists := newZoneListMemory() + dir := NewDirectory(lists) + dir.listURL = f.URL + "/get_zones.php" + now := time.Now().UTC() + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if _, fetched, _ := dir.List(ctx, "CA", now); !fetched { + t.Fatal("first call should spend the request") + } + f.hang.Store(false) + if zones, fetched, err := dir.List(context.Background(), "CA", now.Add(15*time.Second)); zones != nil || fetched || err != nil || f.listCalls != 0 || f.hung.Load() != 1 { + t.Fatal("abandoned request retried inside the rate limit", f.listCalls, fetched, err) + } + if got := lists.rows["CA"].NextAttempt.Sub(now); got != zoneListFresh { + t.Fatal("abandoned request not persisted", lists.rows["CA"]) + } +} + +func TestZoneListFetchDoesNotBlockOtherCallers(t *testing.T) { + release := make(chan struct{}) + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + <-release + _, _ = w.Write([]byte(`{"country":"CA","zones":[]}`)) + })) + defer server.Close() + defer close(release) + dir := NewDirectory(newZoneListMemory()) + dir.listURL = server.URL + now := time.Now().UTC() + go func() { _, _, _ = dir.List(context.Background(), "CA", now) }() + for calls.Load() == 0 { + time.Sleep(time.Millisecond) + } + done := make(chan struct{}) + go func() { + defer close(done) + if zones, fetched, err := dir.List(context.Background(), "CA", now); zones != nil || fetched || err != nil { + t.Error("second caller should back off while the list is in flight", fetched, err) + } + _, _, _, _ = dir.snapshot([]string{"CA"}) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("caller waited on another task's request") + } + if calls.Load() != 1 { + t.Fatal("duplicate list request", calls.Load()) + } +} + +func TestBoundaryCutShortStillCounts(t *testing.T) { + f := newFakeMeshMapper(t) + h := newZoneHarness(t, f, &zoneMemoryStore{rows: map[string]Boundary{}}, true) + if err := h.z.Refresh(context.Background()); err != nil || f.listCalls != 1 { + t.Fatal("list not fetched", err) + } + f.hang.Store(true) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _ = h.z.Refresh(ctx) + f.hang.Store(false) + if err := h.z.Refresh(context.Background()); err != nil || f.calls != 0 || f.hung.Load() != 1 { + t.Fatal("abandoned boundary request retried inside the rate limit", err, f.calls) + } + if b := h.store.rows["YOW"]; b.NextAttempt.Sub(b.AttemptedAt) != 24*time.Hour { + t.Fatal("abandoned boundary request not persisted", b) + } +} From 2f4f92e31ee11620c444299af56fbabf4998b33b Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 16:49:02 -0400 Subject: [PATCH 131/141] fix(db): tie-safe trace paging, atomic migrations and honest stats windows GET /traces takes an optional cursorTag so tags sharing a millisecond survive paging. Migrations run under an advisory lock and record each file in the same implicit transaction, so a crash can't strand a fresh database. Overview windowHours counts the hours it covers, hourly observer activity fills unrolled hours from raw rows, and integration fixtures are UTC. --- db/clock_drift_integration_test.go | 10 ++-- db/migrate.go | 48 ++++++++++++--- db/migrate_integration_test.go | 48 +++++++++++++++ db/migrate_test.go | 11 ++++ db/migrations/001_baseline.sql | 2 + db/nodes_integration_test.go | 6 +- db/observer_comparison_integration_test.go | 2 +- db/observer_metrics_integration_test.go | 56 +++++++++++++++++ db/observers.go | 11 +++- db/queries/queries.sql | 46 +++++++++++--- db/routes_integration_test.go | 4 +- db/sqlc/querier.go | 2 + db/sqlc/queries.sql.go | 58 +++++++++++++++--- db/stats.go | 5 +- db/stats_test.go | 13 +++- db/trace_pagination_integration_test.go | 70 ++++++++++++++++++++-- db/traces.go | 12 +++- db/traces_integration_test.go | 8 +-- db/traces_test.go | 4 +- docs/historical-stats.md | 5 +- internal/api/handlers/limits_test.go | 2 +- internal/api/handlers/stub_reader_test.go | 6 +- internal/api/handlers/traces.go | 18 +++++- internal/api/handlers/traces_test.go | 35 ++++++++++- internal/api/reader.go | 4 +- internal/api/stats.go | 2 +- internal/cache/cache_test.go | 2 +- internal/cache/reader.go | 4 +- 28 files changed, 426 insertions(+), 68 deletions(-) diff --git a/db/clock_drift_integration_test.go b/db/clock_drift_integration_test.go index fbe0ddc7..4145a80b 100644 --- a/db/clock_drift_integration_test.go +++ b/db/clock_drift_integration_test.go @@ -48,12 +48,12 @@ func TestClockDriftPagePostgres(t *testing.T) { _, err = tx.Exec(ctx, ` INSERT INTO nodes (id,public_key,node_type,name,device_clock_drift_seconds,last_advert_at) SELECT ('00000000-0000-0000-0000-'||lpad(i::text,12,'0'))::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),kind, - CASE WHEN i=2 THEN NULL ELSE 'node-'||i END,drift,CASE WHEN i=2 THEN NULL ELSE '2026-01-01'::timestamptz END + CASE WHEN i=2 THEN NULL ELSE 'node-'||i END,drift,CASE WHEN i=2 THEN NULL ELSE '2026-01-01 00:00:00+00'::timestamptz END FROM (VALUES (1,2,120),(2,3,-240),(3,1,99999),(4,4,-99999),(5,2,NULL),(6,2,60),(7,3,-60),(8,2,180)) v(i,kind,drift); INSERT INTO node_iatas (node_id,iata,last_heard) VALUES - ('00000000-0000-0000-0000-000000000001','YVR','2026-01-01'), + ('00000000-0000-0000-0000-000000000001','YVR','2026-01-01 00:00:00+00'), ('00000000-0000-0000-0000-000000000001','YYJ','2026-01-01 01:00:00+00'), - ('00000000-0000-0000-0000-000000000002','YYJ','2026-01-01');`) + ('00000000-0000-0000-0000-000000000002','YYJ','2026-01-01 00:00:00+00');`) if err != nil { t.Fatal(err) } @@ -141,10 +141,10 @@ INSERT INTO node_iatas (node_id,iata,last_heard) VALUES _, err = tx.Exec(ctx, ` TRUNCATE pg_temp.node_iatas,pg_temp.nodes; INSERT INTO nodes (id,public_key,node_type,name,device_clock_drift_seconds,last_advert_at) -SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),2,'clock-'||i,3600+i,'2026-01-01' +SELECT md5(i::text)::uuid,decode(lpad(to_hex(i),64,'0'),'hex'),2,'clock-'||i,3600+i,'2026-01-01 00:00:00+00' FROM generate_series(1,20000) i; INSERT INTO node_iatas (node_id,iata,last_heard) -SELECT md5(i::text)::uuid,iata,'2026-01-01'::timestamptz-i*interval '1 second' +SELECT md5(i::text)::uuid,iata,'2026-01-01 00:00:00+00'::timestamptz-i*interval '1 second' FROM generate_series(1,20000) i CROSS JOIN (VALUES ('YVR'),('YYJ')) regions(iata); ANALYZE nodes; ANALYZE node_iatas;`) if err != nil { diff --git a/db/migrate.go b/db/migrate.go index 560c02e6..06f2a638 100644 --- a/db/migrate.go +++ b/db/migrate.go @@ -32,8 +32,13 @@ type execQuerier interface { var ( lineCommentRe = regexp.MustCompile(`(?m)--[^\n]*`) concurrentIndexRe = regexp.MustCompile(`(?is)\bCREATE\s+(?:UNIQUE\s+)?INDEX\s+CONCURRENTLY\s+(?:IF\s+NOT\s+EXISTS\s+)?("?[\w.]+"?)`) + // Statements that refuse to run inside a transaction block. + nonTransactionalRe = regexp.MustCompile(`(?i)\b(?:CONCURRENTLY|VACUUM)\b`) ) +// migrationLockKey serialises RunMigrations across instances sharing a database. +const migrationLockKey int64 = 0x6265_6163_6f6e // "beacon" + // concurrentIndexName returns the index a CREATE INDEX CONCURRENTLY migration builds. func concurrentIndexName(sql string) (string, bool) { m := concurrentIndexRe.FindStringSubmatch(lineCommentRe.ReplaceAllString(sql, "")) @@ -82,6 +87,21 @@ func applyMigration(ctx context.Context, db execQuerier, sql string) error { return err } +// applyAndRecord applies a migration and its ledger row. Without CONCURRENTLY both go in +// one simple-protocol Exec, an implicit transaction, so a crash can't apply one without the other. +func applyAndRecord(ctx context.Context, db execQuerier, name, sql string) error { + record := "INSERT INTO schema_migrations (filename) VALUES ('" + strings.ReplaceAll(name, "'", "''") + "')" + if !nonTransactionalRe.MatchString(lineCommentRe.ReplaceAllString(sql, "")) { + _, err := db.Exec(ctx, sql+"\n;\n"+record) + return err + } + if err := applyMigration(ctx, db, sql); err != nil { + return err + } + _, err := db.Exec(ctx, record) + return err +} + const baselineMigration = "001_baseline.sql" var errPreBaseline = errors.New("database schema predates Beacon 2.0.0; 2.0.0 needs a fresh database") @@ -105,6 +125,24 @@ func checkBaseline(ctx context.Context, db execQuerier) error { } func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { + conn, err := pool.Acquire(ctx) + if err != nil { + return fmt.Errorf("failed to acquire migration connection: %w", err) + } + defer conn.Release() + if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", migrationLockKey); err != nil { + return fmt.Errorf("failed to take migration lock: %w", err) + } + defer func() { + // A session lock outlives Release; drop the connection if unlock fails. + if _, err := conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", migrationLockKey); err != nil { + _ = conn.Conn().Close(context.Background()) + } + }() + return runMigrations(ctx, conn) +} + +func runMigrations(ctx context.Context, pool execQuerier) error { _, err := pool.Exec(ctx, ` CREATE TABLE IF NOT EXISTS schema_migrations ( filename TEXT PRIMARY KEY, @@ -150,18 +188,10 @@ func RunMigrations(ctx context.Context, pool *pgxpool.Pool) error { return fmt.Errorf("failed to read migration %s: %w", entry.Name(), err) } - if err := applyMigration(ctx, pool, string(sql)); err != nil { + if err := applyAndRecord(ctx, pool, entry.Name(), string(sql)); err != nil { return fmt.Errorf("failed to apply migration %s: %w", entry.Name(), err) } - if _, err := pool.Exec( - ctx, - "INSERT INTO schema_migrations (filename) VALUES ($1)", - entry.Name(), - ); err != nil { - return fmt.Errorf("failed to record migration %s: %w", entry.Name(), err) - } - slog.Info(fmt.Sprintf("applied migration: %s", entry.Name()), "component", "db") } diff --git a/db/migrate_integration_test.go b/db/migrate_integration_test.go index 1239f5d8..cc01ebf8 100644 --- a/db/migrate_integration_test.go +++ b/db/migrate_integration_test.go @@ -13,6 +13,54 @@ import ( "github.com/jackc/pgx/v5" ) +// Two instances starting on an empty database must not race the baseline. +func TestRunMigrationsConcurrentStartPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + errs := make(chan error, 3) + for range 3 { + go func() { errs <- RunMigrations(ctx, pool) }() + } + for range 3 { + if err := <-errs; err != nil { + t.Error(err) + } + } + var n int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM schema_migrations").Scan(&n); err != nil || n != 1 { + t.Errorf("ledger rows = %d, %v; want 1", n, err) + } +} + +// A failed ledger insert must roll back the migration it records, or a crash between +// the two leaves a schema the baseline check refuses forever. +func TestApplyAndRecordIsAtomicPostgres(t *testing.T) { + ctx, pool := schemaPool(t) + if _, err := pool.Exec(ctx, `CREATE TABLE schema_migrations (filename TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()); +INSERT INTO schema_migrations (filename) VALUES ('900_dup.sql')`); err != nil { + t.Fatal(err) + } + if err := applyAndRecord(ctx, pool, "900_dup.sql", "CREATE TABLE atomic_probe (v INT); -- trailing comment"); err == nil { + t.Fatal("duplicate ledger row should fail") + } + var exists bool + if err := pool.QueryRow(ctx, "SELECT to_regclass('atomic_probe') IS NOT NULL").Scan(&exists); err != nil || exists { + t.Errorf("migration survived its failed ledger insert: %v %v", exists, err) + } + if err := applyAndRecord(ctx, pool, "901_it's.sql", "CREATE TABLE atomic_probe (v INT)"); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE filename = '901_it''s.sql') AND to_regclass('atomic_probe') IS NOT NULL").Scan(&exists); err != nil || !exists { + t.Errorf("migration and ledger row not both recorded: %v %v", exists, err) + } + // CONCURRENTLY can't share a transaction; it still applies and records. + if err := applyAndRecord(ctx, pool, "902_idx.sql", "CREATE INDEX CONCURRENTLY atomic_probe_idx ON atomic_probe (v)"); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE filename = '902_idx.sql') AND to_regclass('atomic_probe_idx') IS NOT NULL").Scan(&exists); err != nil || !exists { + t.Errorf("concurrent migration not recorded: %v %v", exists, err) + } +} + // CONCURRENTLY is illegal inside a transaction, so this uses a plain connection // and a real table rather than the usual tx + TEMP table pattern. func TestApplyMigrationRecoversInvalidIndexPostgres(t *testing.T) { diff --git a/db/migrate_test.go b/db/migrate_test.go index f93f4442..0c6d3e65 100644 --- a/db/migrate_test.go +++ b/db/migrate_test.go @@ -43,6 +43,17 @@ ON known_routes (iata, last_seen DESC);`, "idx_known_routes_iata_last_seen", tru } } +// The baseline must apply in the same transaction as its ledger row. +func TestBaselineIsTransactional(t *testing.T) { + sql, err := migrationFiles.ReadFile("migrations/" + baselineMigration) + if err != nil { + t.Fatal(err) + } + if nonTransactionalRe.MatchString(lineCommentRe.ReplaceAllString(string(sql), "")) { + t.Fatal("baseline contains a statement that can't run in a transaction") + } +} + func TestIsDuplicateRelation(t *testing.T) { dup := &pgconn.PgError{Code: "42P07"} cases := []struct { diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql index 7dd67ed8..71e42f26 100644 --- a/db/migrations/001_baseline.sql +++ b/db/migrations/001_baseline.sql @@ -843,6 +843,8 @@ CREATE INDEX idx_trace_iatas_iata ON trace_iatas USING btree (iata); CREATE INDEX idx_trace_tags_last_heard ON trace_tags USING btree (last_heard_at DESC); +CREATE INDEX idx_trace_tags_keyset ON trace_tags USING btree (date_trunc('milliseconds', last_heard_at, 'UTC') DESC, trace_tag DESC); + CREATE INDEX idx_trace_tags_first_heard ON trace_tags USING btree (first_heard_at); CREATE INDEX idx_analytics_observer_activity_observer ON analytics_hourly_observer_activity USING btree (observer_id, hour); diff --git a/db/nodes_integration_test.go b/db/nodes_integration_test.go index 03ddcd07..4a45f8a0 100644 --- a/db/nodes_integration_test.go +++ b/db/nodes_integration_test.go @@ -50,14 +50,14 @@ func TestListNodesPostgres(t *testing.T) { INSERT INTO transport_scopes (id, name, transport_key, key_fingerprint) VALUES (1, '#test', decode(repeat('00', 16), 'hex'), decode(repeat('00', 8), 'hex')); INSERT INTO nodes (id, public_key, node_type, name, last_seen, default_scope_id, supports_multibyte_paths, supports_multibyte_traces) SELECT md5(i::text)::uuid, decode(lpad(to_hex(i), 64, '0'), 'hex'), (i % 4 + 1)::smallint, - 'node-' || i, '2026-01-01'::timestamptz - i * interval '1 second', + 'node-' || i, '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second', CASE WHEN i % 2 = 0 THEN 1 END, i % 2 = 0, i % 3 = 0 FROM generate_series(1, 20000) i; INSERT INTO node_iatas (node_id, iata, last_heard) -SELECT md5(i::text)::uuid, 'YVR', '2026-01-01'::timestamptz - i * interval '1 second' +SELECT md5(i::text)::uuid, 'YVR', '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second' FROM generate_series(1, 19999) i; INSERT INTO node_iatas (node_id, iata, last_heard) -SELECT md5(i::text)::uuid, 'YYJ', '2026-01-01'::timestamptz - i * interval '1 second' - interval '1 hour' +SELECT md5(i::text)::uuid, 'YYJ', '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second' - interval '1 hour' FROM generate_series(2, 19998, 2) i; INSERT INTO node_neighbors (node_id, neighbor_id, iata) SELECT md5(i::text)::uuid, md5((i+1)::text)::uuid, iata diff --git a/db/observer_comparison_integration_test.go b/db/observer_comparison_integration_test.go index 103f029b..8abc290c 100644 --- a/db/observer_comparison_integration_test.go +++ b/db/observer_comparison_integration_test.go @@ -49,7 +49,7 @@ FROM (VALUES (1,1),(2,0),(3,1),(4,1),(5,0),(6,1),(7,2),(8,3),(9,1),(10,1)) v(i,r INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,source_broker) SELECT id,decode(lpad(to_hex(packet),2,'0'),'hex'), ('00000000-0000-0000-0000-' || lpad(observer::text,12,'0'))::uuid, - iata,'2026-01-01'::timestamptz+second*interval '1 second',0,1,0,broker + iata,'2026-01-01 00:00:00+00'::timestamptz+second*interval '1 second',0,1,0,broker FROM (VALUES (1,1,1,'YVR',1,'a'),(2,1,1,'YVR',2,'b'), (3,2,2,'YVR',2,'a'),(4,3,1,'YVR',3,'a'),(5,3,2,'YYJ',4,'a'), diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 5c0b66bc..0ca707e3 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -11,6 +11,62 @@ import ( "time" ) +// Hourly activity reads the rollup, but the last ~2h aren't rolled yet; they come from raw rows. +func TestObserverActivityUnrolledTailPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + now := time.Now().UTC() + rolled := now.Truncate(time.Hour).Add(-5 * time.Hour) + if _, err := tx.Exec(ctx, ` + INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); + INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,last_heard_at,first_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(1,4) i;`); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, ` + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) VALUES + (int4send(1),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz+interval '10 minutes',0,1,0,4,5,-90), + (int4send(2),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz+interval '20 minutes',0,1,0,4,7,-80);`, rolled); err != nil { + t.Fatal(err) + } + rollTxHours(t, ctx, tx) + // Heard since the newest rolled hour; the rollup hasn't reached them. + if _, err := tx.Exec(ctx, ` + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) VALUES + (int4send(3),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz-interval '100 minutes',0,1,0,4,9,-70), + (int4send(4),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz-interval '5 minutes',0,1,0,NULL,0,0);`, now); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + activity, err := store.GetObserverActivity(ctx, id, 24*time.Hour, time.Hour, time.Time{}) + if err != nil { + t.Fatal(err) + } + if activity.Summary.RecordedPackets != 4 { + t.Errorf("recorded packets = %d, want 4 (2 rolled + 2 unrolled)", activity.Summary.RecordedPackets) + } + byHour := map[int64]int64{} + for _, p := range activity.Points { + byHour[p.T] = p.Observations + } + recent := now.Add(-100 * time.Minute).Truncate(time.Hour).UnixMilli() + if byHour[rolled.UnixMilli()] != 2 || byHour[recent] != 1 || byHour[now.Add(-5*time.Minute).Truncate(time.Hour).UnixMilli()] != 1 { + t.Errorf("points %+v", activity.Points) + } + for _, p := range activity.Points { + if p.T == recent && (p.SNRAvg == nil || *p.SNRAvg != 9 || p.RSSIAvg == nil || *p.RSSIAvg != -70) { + t.Errorf("tail signal: %+v", p) + } + } + types := map[int16]int64{} + for _, v := range activity.PayloadTypes { + types[v.PayloadType] = v.Count + } + if types[4] != 3 || types[-1] != 1 { + t.Errorf("payload types %+v", activity.PayloadTypes) + } +} + func TestObserverMetricsPostgres(t *testing.T) { ctx, tx := retentionTx(t) applyBaseline(t, ctx, tx) diff --git a/db/observers.go b/db/observers.go index 395e8871..888410f9 100644 --- a/db/observers.go +++ b/db/observers.go @@ -209,8 +209,13 @@ func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uui return api.BucketTelemetry(raw.Points, since, time.Duration(bucketHours)*time.Hour), nil } +// activityRawTail caps how far back hourly activity reads raw rows for hours not yet rolled +// (normally under rollupDelay + 1h). +const activityRawTail = 3 * time.Hour + // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. -// Buckets of an hour or coarser come from the hourly rollup; anything finer reads observations directly. +// Buckets of an hour or coarser come from the hourly rollup plus raw rows for the unrolled tail; +// anything finer reads observations directly. // Range and Interval are left empty for the handler to fill. func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { obs, err := s.q.GetObserverByID(ctx, observerID) @@ -259,10 +264,13 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w binWidth := pgtype.Interval{Microseconds: interval.Microseconds(), Valid: true} if interval >= time.Hour { + // Unrolled hours read raw rows, bounded to the normal rollup lag. + tailFloor := pgtype.Timestamptz{Time: until.Add(-activityRawTail), Valid: true} rows, err := s.q.GetObserverActivityHourly(ctx, sqlc.GetObserverActivityHourlyParams{ ObserverID: observerID, Column2: sinceTS, Column3: binWidth, + TailFloor: tailFloor, Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { @@ -295,6 +303,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityHourlyPayloadTypes(ctx, sqlc.GetObserverActivityHourlyPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, + TailFloor: tailFloor, Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { diff --git a/db/queries/queries.sql b/db/queries/queries.sql index dacbb98e..8ca8b3de 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -263,8 +263,24 @@ ORDER BY count DESC; -- name: GetObserverActivityHourly :many -- Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. +-- Hours after the newest complete one aren't rolled yet; that tail, from no earlier than @tail_floor, reads raw rows. +WITH tail AS ( + SELECT GREATEST(max(hour) + INTERVAL '1 hour', @tail_floor::timestamptz) AS tail_start + FROM analytics_rollup_hours WHERE status = 'complete' +), src AS ( + SELECT a.hour AS t, a.observations, a.airtime_ms, a.airtime_n, a.snr_sum, a.snr_n, a.snr_min, a.rssi_sum, a.rssi_n + FROM analytics_hourly_observer_activity a + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST(@until::timestamptz, (SELECT tail_start FROM tail)) + UNION ALL + SELECT o.heard_at, 1::bigint, o.airtime_ms, (o.airtime_ms IS NOT NULL)::int::bigint, + s.snr, (s.snr IS NOT NULL)::int::bigint, s.snr, s.rssi::bigint, (s.rssi IS NOT NULL)::int::bigint + FROM packet_observations o CROSS JOIN + LATERAL (SELECT CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.snr END AS snr, + CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.rssi END AS rssi) s + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, (SELECT tail_start FROM tail)) AND o.heard_at < @until::timestamptz +) SELECT - date_bin($3::interval, hour, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + date_bin($3::interval, t, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, SUM(observations)::bigint AS observations, COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, @@ -273,15 +289,26 @@ SELECT COALESCE(MIN(snr_min), 0)::real AS snr_min, COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM analytics_hourly_observer_activity -WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < @until::timestamptz +FROM src GROUP BY 1 ORDER BY 1; -- name: GetObserverActivityHourlyPayloadTypes :many -SELECT payload_type, SUM(observations)::bigint AS count -FROM analytics_hourly_observer_activity -WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < @until::timestamptz +-- Same rollup/raw-tail split as GetObserverActivityHourly. +WITH tail AS ( + SELECT GREATEST(max(hour) + INTERVAL '1 hour', @tail_floor::timestamptz) AS tail_start + FROM analytics_rollup_hours WHERE status = 'complete' +), src AS ( + SELECT a.payload_type, a.observations AS n + FROM analytics_hourly_observer_activity a + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST(@until::timestamptz, (SELECT tail_start FROM tail)) + UNION ALL + SELECT COALESCE(o.payload_type, -1)::smallint, 1::bigint + FROM packet_observations o + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, (SELECT tail_start FROM tail)) AND o.heard_at < @until::timestamptz +) +SELECT payload_type, SUM(n)::bigint AS count +FROM src GROUP BY payload_type ORDER BY count DESC; @@ -1405,9 +1432,12 @@ WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( AND ($2::text = '' OR t.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) AND ($3::timestamptz IS NULL OR t.first_heard_at >= $3) AND ($4::timestamptz IS NULL OR t.first_heard_at <= $4) - AND ($5::timestamptz IS NULL OR t.last_heard_at < $5) + -- Keyset on the millisecond clients see, tie-broken by tag; matches idx_trace_tags_keyset. + AND ($5::timestamptz IS NULL + OR date_trunc('milliseconds', t.last_heard_at, 'UTC') < $5 + OR ($8::bytea IS NOT NULL AND date_trunc('milliseconds', t.last_heard_at, 'UTC') = $5 AND t.trace_tag < $8)) AND ($7::text = '' OR t.trace_type = $7) -ORDER BY t.last_heard_at DESC +ORDER BY date_trunc('milliseconds', t.last_heard_at, 'UTC') DESC, t.trace_tag DESC LIMIT $6; -- ============================================================ diff --git a/db/routes_integration_test.go b/db/routes_integration_test.go index f37d2235..6d6addf2 100644 --- a/db/routes_integration_test.go +++ b/db/routes_integration_test.go @@ -45,8 +45,8 @@ INSERT INTO iata_codes (iata) VALUES ('GPT'); INSERT INTO known_routes (id,path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) SELECT i, decode(md5(i::text),'hex'), ARRAY[md5(i::text)::uuid], ARRAY['\x01'::bytea], CASE WHEN i > 100000 THEN 'GPT' ELSE 'YYZ' END, i % 4 + 2, - '2026-01-01'::timestamptz - i * interval '1 second', - '2026-01-01'::timestamptz - i * interval '1 second' + '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second', + '2026-01-01 00:00:00+00'::timestamptz - i * interval '1 second' FROM generate_series(1,100100) i; ANALYZE known_routes;`) if err != nil { diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 834628f1..8731ece4 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -84,7 +84,9 @@ type Querier interface { GetNodesByIDs(ctx context.Context, dollar_1 []uuid.UUID) ([]GetNodesByIDsRow, error) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]GetNodesByPubkeysRow, error) // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. + // Hours after the newest complete one aren't rolled yet; that tail, from no earlier than @tail_floor, reads raw rows. GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) + // Same rollup/raw-tail split as GetObserverActivityHourly. GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) // Two indexed ranges, bounded to one observer; no legacy presence counters. GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 7c6d7dc3..dfa697d0 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -813,8 +813,23 @@ func (q *Queries) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]Ge } const getObserverActivityHourly = `-- name: GetObserverActivityHourly :many +WITH tail AS ( + SELECT GREATEST(max(hour) + INTERVAL '1 hour', $4::timestamptz) AS tail_start + FROM analytics_rollup_hours WHERE status = 'complete' +), src AS ( + SELECT a.hour AS t, a.observations, a.airtime_ms, a.airtime_n, a.snr_sum, a.snr_n, a.snr_min, a.rssi_sum, a.rssi_n + FROM analytics_hourly_observer_activity a + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST($5::timestamptz, (SELECT tail_start FROM tail)) + UNION ALL + SELECT o.heard_at, 1::bigint, o.airtime_ms, (o.airtime_ms IS NOT NULL)::int::bigint, + s.snr, (s.snr IS NOT NULL)::int::bigint, s.snr, s.rssi::bigint, (s.rssi IS NOT NULL)::int::bigint + FROM packet_observations o CROSS JOIN + LATERAL (SELECT CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.snr END AS snr, + CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.rssi END AS rssi) s + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, (SELECT tail_start FROM tail)) AND o.heard_at < $5::timestamptz +) SELECT - date_bin($3::interval, hour, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, + date_bin($3::interval, t, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, SUM(observations)::bigint AS observations, COALESCE(SUM(airtime_ms), 0)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, @@ -823,8 +838,7 @@ SELECT COALESCE(MIN(snr_min), 0)::real AS snr_min, COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM analytics_hourly_observer_activity -WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < $4::timestamptz +FROM src GROUP BY 1 ORDER BY 1 ` @@ -833,6 +847,7 @@ type GetObserverActivityHourlyParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` + TailFloor pgtype.Timestamptz `json:"tail_floor"` Until pgtype.Timestamptz `json:"until"` } @@ -849,11 +864,13 @@ type GetObserverActivityHourlyRow struct { } // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. +// Hours after the newest complete one aren't rolled yet; that tail, from no earlier than @tail_floor, reads raw rows. func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) { rows, err := q.db.Query(ctx, getObserverActivityHourly, arg.ObserverID, arg.Column2, arg.Column3, + arg.TailFloor, arg.Until, ) if err != nil { @@ -885,9 +902,20 @@ func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserver } const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many -SELECT payload_type, SUM(observations)::bigint AS count -FROM analytics_hourly_observer_activity -WHERE observer_id = $1 AND hour >= $2::timestamptz AND hour < $3::timestamptz +WITH tail AS ( + SELECT GREATEST(max(hour) + INTERVAL '1 hour', $3::timestamptz) AS tail_start + FROM analytics_rollup_hours WHERE status = 'complete' +), src AS ( + SELECT a.payload_type, a.observations AS n + FROM analytics_hourly_observer_activity a + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST($4::timestamptz, (SELECT tail_start FROM tail)) + UNION ALL + SELECT COALESCE(o.payload_type, -1)::smallint, 1::bigint + FROM packet_observations o + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, (SELECT tail_start FROM tail)) AND o.heard_at < $4::timestamptz +) +SELECT payload_type, SUM(n)::bigint AS count +FROM src GROUP BY payload_type ORDER BY count DESC ` @@ -895,6 +923,7 @@ ORDER BY count DESC type GetObserverActivityHourlyPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` + TailFloor pgtype.Timestamptz `json:"tail_floor"` Until pgtype.Timestamptz `json:"until"` } @@ -903,8 +932,14 @@ type GetObserverActivityHourlyPayloadTypesRow struct { Count int64 `json:"count"` } +// Same rollup/raw-tail split as GetObserverActivityHourly. func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) + rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, + arg.ObserverID, + arg.Column2, + arg.TailFloor, + arg.Until, + ) if err != nil { return nil, err } @@ -4082,9 +4117,12 @@ WHERE (COALESCE(cardinality($1::bpchar[]), 0) = 0 OR EXISTS ( AND ($2::text = '' OR t.scope_id = (SELECT id FROM transport_scopes WHERE name = $2)) AND ($3::timestamptz IS NULL OR t.first_heard_at >= $3) AND ($4::timestamptz IS NULL OR t.first_heard_at <= $4) - AND ($5::timestamptz IS NULL OR t.last_heard_at < $5) + -- Keyset on the millisecond clients see, tie-broken by tag; matches idx_trace_tags_keyset. + AND ($5::timestamptz IS NULL + OR date_trunc('milliseconds', t.last_heard_at, 'UTC') < $5 + OR ($8::bytea IS NOT NULL AND date_trunc('milliseconds', t.last_heard_at, 'UTC') = $5 AND t.trace_tag < $8)) AND ($7::text = '' OR t.trace_type = $7) -ORDER BY t.last_heard_at DESC +ORDER BY date_trunc('milliseconds', t.last_heard_at, 'UTC') DESC, t.trace_tag DESC LIMIT $6 ` @@ -4096,6 +4134,7 @@ type ListTraceTagsParams struct { Column5 pgtype.Timestamptz `json:"column_5"` Limit int32 `json:"limit"` Column7 string `json:"column_7"` + Column8 []byte `json:"column_8"` } type ListTraceTagsRow struct { @@ -4121,6 +4160,7 @@ func (q *Queries) ListTraceTags(ctx context.Context, arg ListTraceTagsParams) ([ arg.Column5, arg.Limit, arg.Column7, + arg.Column8, ) if err != nil { return nil, err diff --git a/db/stats.go b/db/stats.go index 8d614ca6..a0907b63 100644 --- a/db/stats.go +++ b/db/stats.go @@ -39,7 +39,8 @@ func pgUUID(id pgtype.UUID) *uuid.UUID { return &u } -// GetStatsOverview summarises the 24 most recent hours that can have been rolled. +// GetStatsOverview summarises the 24 most recent hours that can have been rolled; +// WindowHours counts the complete ones the totals actually cover. func (s *Store) GetStatsOverview(ctx context.Context, iatas []string) (*api.StatsOverview, error) { until := time.Now().UTC().Add(-rollupDelay).Truncate(time.Hour).Add(time.Hour) since := until.Add(-24 * time.Hour) @@ -52,7 +53,7 @@ func (s *Store) GetStatsOverview(ctx context.Context, iatas []string) (*api.Stat TotalObservations: series.Summary.Observations, ActiveObservers: series.Summary.ActiveObservers, ActiveIATAs: series.Summary.ActiveIATAs, - WindowHours: 24, + WindowHours: series.CompleteHours, Since: since.UnixMilli(), Until: until.UnixMilli(), }, nil diff --git a/db/stats_test.go b/db/stats_test.go index d70a1f37..dad24c83 100644 --- a/db/stats_test.go +++ b/db/stats_test.go @@ -26,7 +26,16 @@ func TestGetStatsOverview(t *testing.T) { GetStatsSeries(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, p sqlc.GetStatsSeriesParams) ([]sqlc.GetStatsSeriesRow, error) { params = p - return []sqlc.GetStatsSeriesRow{{Status: "summary", Observations: 500, UniquePackets: 100, ActiveObservers: 10, ActiveIatas: 3}}, nil + rows := []sqlc.GetStatsSeriesRow{{Status: "summary", Observations: 500, UniquePackets: 100, ActiveObservers: 10, ActiveIatas: 3}} + // The newest hour isn't rolled yet; the summary covers the other 23. + for h := p.Since.Time; h.Before(p.Until.Time); h = h.Add(time.Hour) { + status := "complete" + if h.Add(time.Hour).Equal(p.Until.Time) { + status = "missing" + } + rows = append(rows, sqlc.GetStatsSeriesRow{Hour: pgtype.Timestamptz{Time: h, Valid: true}, Status: status}) + } + return rows, nil }) store := &Store{q: mock} @@ -37,7 +46,7 @@ func TestGetStatsOverview(t *testing.T) { if result.TotalPackets != 100 || result.TotalObservations != 500 || result.ActiveObservers != 10 || result.ActiveIATAs != 3 { t.Errorf("unexpected summary mapping: %+v", result) } - if result.WindowHours != 24 || result.Until-result.Since != (24*time.Hour).Milliseconds() { + if result.WindowHours != 23 || result.Until-result.Since != (24*time.Hour).Milliseconds() { t.Errorf("window %d..%d (%d hours)", result.Since, result.Until, result.WindowHours) } // The window ends at the last hour that can have been rolled. diff --git a/db/trace_pagination_integration_test.go b/db/trace_pagination_integration_test.go index 991d74d4..90ed19d5 100644 --- a/db/trace_pagination_integration_test.go +++ b/db/trace_pagination_integration_test.go @@ -114,7 +114,7 @@ func TestTraceTagPaginationPostgres(t *testing.T) { {"time window on first heard", nil, "", "", 3, 6, 26, []api.TraceTagSummary{c}}, } { t.Run(tc.name, func(t *testing.T) { - got, err := store.ListTraceTags(ctx, tc.iatas, tc.scope, tc.kind, at(tc.since), at(tc.until), at(tc.cursor), 20) + got, err := store.ListTraceTags(ctx, tc.iatas, tc.scope, tc.kind, at(tc.since), at(tc.until), at(tc.cursor), "", 20) if err != nil { t.Fatal(err) } @@ -129,7 +129,7 @@ func TestTraceTagPaginationPostgres(t *testing.T) { var cursor time.Time total := 0 for page := 0; page < 10; page++ { - rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, cursor, 2) + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, cursor, "", 2) if err != nil { t.Fatal(err) } @@ -156,7 +156,7 @@ func TestTraceTagPaginationPostgres(t *testing.T) { if err := store.DeleteOldTraceTags(ctx, at(20)); err != nil { t.Fatal(err) } - if got, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, 20); err != nil || !reflect.DeepEqual(got, []api.TraceTagSummary{a, b, c}) { + if got, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, "", 20); err != nil || !reflect.DeepEqual(got, []api.TraceTagSummary{a, b, c}) { t.Errorf("after cleanup got %+v, %v; want a, b, c", got, err) } } @@ -183,7 +183,7 @@ func TestTraceSummarySurvivesMissedHearingPostgres(t *testing.T) { } upsert(1) // the RecordTrace that would follow never runs upsert(1) // a broker duplicate must not count twice - rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, "", 10) if err != nil || len(rows) != 1 || rows[0].PacketCount != 1 || len(rows[0].PathHashes) != 2 || rows[0].TraceType != "TRACE" { t.Fatalf("summary after a missed hearing: %+v %v", rows, err) } @@ -191,8 +191,68 @@ func TestTraceSummarySurvivesMissedHearingPostgres(t *testing.T) { if err := store.RecordTrace(ctx, ingest.TraceHearing{TraceTag: tag, IATA: "YVR", HeardAt: heard}); err != nil { t.Fatal(err) } - rows, err = store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + rows, err = store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Time{}, "", 10) if err != nil || len(rows) != 1 || rows[0].FirstHeardAt != heard.UnixMilli() || rows[0].LastHeardAt != heard.UnixMilli() || rows[0].IATACount != 1 { t.Errorf("first hearing should replace provisional times: %+v %v", rows, err) } } + +// Tags sharing a millisecond (second-resolution observer clocks, sub-ms times) must +// all survive paging with the (cursor, cursorTag) keyset. +func TestTraceTagPaginationTiesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + if _, err := tx.Exec(ctx, `INSERT INTO trace_tags (trace_tag, first_heard_at, last_heard_at, heard) VALUES + ('\x00000001', '2026-01-01 00:00:00+00', '2026-01-01 00:00:10+00', true), + ('\x00000002', '2026-01-01 00:00:00+00', '2026-01-01 00:00:10+00', true), + ('\x00000003', '2026-01-01 00:00:00+00', '2026-01-01 00:00:10+00', true), + ('\x00000004', '2026-01-01 00:00:00+00', '2026-01-01 00:00:10+00', true), + ('\x00000005', '2026-01-01 00:00:00+00', '2026-01-01 00:00:05.000300+00', true), + ('\x00000006', '2026-01-01 00:00:00+00', '2026-01-01 00:00:05.000700+00', true), + ('\x00000007', '2026-01-01 00:00:00+00', '2026-01-01 00:00:04+00', true)`); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + want := []string{"00000004", "00000003", "00000002", "00000001", "00000006", "00000005", "00000007"} + var got []string + var cursor time.Time + var cursorTag string + for page := 0; page < 10; page++ { + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, cursor, cursorTag, 2) + if err != nil { + t.Fatal(err) + } + if len(rows) == 0 { + break + } + for _, r := range rows { + got = append(got, r.TraceTag) + } + last := rows[len(rows)-1] + cursor, cursorTag = time.UnixMilli(last.LastHeardAt), last.TraceTag + } + if !reflect.DeepEqual(got, want) { + t.Errorf("keyset pages = %v, want %v", got, want) + } + + // Without cursorTag the cursor stays exclusive on the millisecond. + rows, err := store.ListTraceTags(ctx, nil, "", "", time.Time{}, time.Time{}, time.Date(2026, 1, 1, 0, 0, 5, 0, time.UTC), "", 10) + if err != nil || len(rows) != 1 || rows[0].TraceTag != "00000007" { + t.Errorf("ms-only cursor: %+v %v", rows, err) + } +} diff --git a/db/traces.go b/db/traces.go index b33ebe3f..93ed18cd 100644 --- a/db/traces.go +++ b/db/traces.go @@ -7,6 +7,7 @@ import ( "context" "encoding/hex" "encoding/json" + "fmt" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -37,7 +38,7 @@ func (s *Store) DeleteOldTraceIATAs(ctx context.Context, cutoff time.Time) error return s.q.DeleteOldTraceIATAs(ctx, pgtype.Timestamptz{Time: cutoff, Valid: true}) } -func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) { +func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { var sinceTS, untilTS, cursorTS pgtype.Timestamptz if !since.IsZero() { sinceTS = pgtype.Timestamptz{Time: since, Valid: true} @@ -48,6 +49,14 @@ func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceT if !cursor.IsZero() { cursorTS = pgtype.Timestamptz{Time: cursor, Valid: true} } + var tag []byte + if cursorTag != "" { + b, err := hex.DecodeString(cursorTag) + if err != nil { + return nil, fmt.Errorf("cursor tag: %w", err) + } + tag = b + } rows, err := s.q.ListTraceTags(ctx, sqlc.ListTraceTagsParams{ Column1: iatas, Column2: scope, @@ -56,6 +65,7 @@ func (s *Store) ListTraceTags(ctx context.Context, iatas []string, scope, traceT Column5: cursorTS, Limit: limit, Column7: traceType, + Column8: tag, }) if err != nil { return nil, err diff --git a/db/traces_integration_test.go b/db/traces_integration_test.go index 7850988c..51305443 100644 --- a/db/traces_integration_test.go +++ b/db/traces_integration_test.go @@ -61,16 +61,16 @@ INSERT INTO node_short_ids (node_id,iata,prefix_4) VALUES INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,trace_tag,parsed_payload,first_heard_at,last_heard_at) SELECT decode(lpad(to_hex(i),64,'0'),'hex'),9,0,1,'\x00','\x00','\x01020304', CASE WHEN i=100 THEN '{}'::jsonb ELSE '{"flags":0,"pathHashes":["aa"],"snrValues":[12.5]}'::jsonb END, - '2026-01-01'::timestamptz+i*interval '1 second', '2026-01-01'::timestamptz+i*interval '1 second'+interval '1 second' + '2026-01-01 00:00:00+00'::timestamptz+i*interval '1 second', '2026-01-01 00:00:00+00'::timestamptz+i*interval '1 second'+interval '1 second' FROM generate_series(1,100) i; INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) -SELECT i,decode(lpad(to_hex(i),64,'0'),'hex'),md5(i::text)::uuid,'YVR','2026-01-01'::timestamptz+interval '2 seconds',1,1,1 +SELECT i,decode(lpad(to_hex(i),64,'0'),'hex'),md5(i::text)::uuid,'YVR','2026-01-01 00:00:00+00'::timestamptz+interval '2 seconds',1,1,1 FROM generate_series(1,100) i WHERE i<>99; -- The first packet was heard in YYJ before YVR. A repeat YYJ observation must -- not trigger another region lookup or change equal-confidence precedence. INSERT INTO packet_observations (id,packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) VALUES - (101,decode(lpad('1',64,'0'),'hex'),md5('101')::uuid,'YYJ','2026-01-01'::timestamptz,1,1,1), - (102,decode(lpad('1',64,'0'),'hex'),md5('102')::uuid,'YYJ','2026-01-01'::timestamptz+interval '3 seconds',1,1,1); + (101,decode(lpad('1',64,'0'),'hex'),md5('101')::uuid,'YYJ','2026-01-01 00:00:00+00'::timestamptz,1,1,1), + (102,decode(lpad('1',64,'0'),'hex'),md5('102')::uuid,'YYJ','2026-01-01 00:00:00+00'::timestamptz+interval '3 seconds',1,1,1); ANALYZE packets; ANALYZE packet_observations; ANALYZE nodes; ANALYZE node_short_ids;`) if err != nil { t.Fatal(err) diff --git a/db/traces_test.go b/db/traces_test.go index e7566979..0e3d4b48 100644 --- a/db/traces_test.go +++ b/db/traces_test.go @@ -24,7 +24,7 @@ func TestListTraceTags_Empty(t *testing.T) { Return([]sqlc.ListTraceTagsRow{}, nil) store := &Store{q: mock} - items, err := store.ListTraceTags(context.Background(), []string{"YVR"}, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + items, err := store.ListTraceTags(context.Background(), []string{"YVR"}, "", "", time.Time{}, time.Time{}, time.Time{}, "", 10) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -56,7 +56,7 @@ func TestListTraceTags_WithPayload(t *testing.T) { }, nil) store := &Store{q: mock} - items, err := store.ListTraceTags(context.Background(), []string{"YVR"}, "", "", time.Time{}, time.Time{}, time.Time{}, 10) + items, err := store.ListTraceTags(context.Background(), []string{"YVR"}, "", "", time.Time{}, time.Time{}, time.Time{}, "", 10) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/docs/historical-stats.md b/docs/historical-stats.md index 8c8ebffb..99812aa1 100644 --- a/docs/historical-stats.md +++ b/docs/historical-stats.md @@ -53,6 +53,8 @@ sub-hour observer activity. | SNR / RSSI averages | Σ sum / Σ samples, never an average of averages | Windows snap to UTC hours on the server, and responses report the effective window. +`/stats/overview` spans the 24 newest eligible hours; its `windowHours` counts the complete +ones its totals cover. ## IATA sets @@ -66,7 +68,8 @@ region or all IATAs. ## Raw or current, by design - Packet, trace and route detail, the packet list, observer comparison, and observer - activity below one hour. + activity below one hour. Hourly observer activity reads raw rows for the hours after the + newest complete one (at most the last 3 hours), so the unrolled tail isn't shown as zero. - Current memberships: scope observer and node counts, node types, clock drift, radio presets. - `/traces` reads `trace_tags`, a per-tag summary kept at ingest, not a rollup. A new packet adds its count, type and payload in the statement that stores it; hearings set the times. diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go index f59c42c3..b5267694 100644 --- a/internal/api/handlers/limits_test.go +++ b/internal/api/handlers/limits_test.go @@ -94,7 +94,7 @@ func TestListLimits(t *testing.T) { got = limit return nil, nil }, - listTraceTags: func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) { + listTraceTags: func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { calls++ got = limit return nil, nil diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index d4e148b6..6ae0ae4e 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -57,7 +57,7 @@ type stubReader struct { getStatsNodeTypes func(ctx context.Context, iatas []string) ([]api.NodeTypeCount, error) getScopeNames func(ctx context.Context) ([]string, error) getScopeByName func(ctx context.Context, name string) (*api.ScopeDetail, error) - listTraceTags func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) + listTraceTags func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) getTraceByTag func(ctx context.Context, tag string) (*api.TraceDetail, error) listKnownRoutes func(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) searchKnownRoutes func(ctx context.Context, iata, fromHash, toHash string) ([]api.KnownRoute, error) @@ -355,9 +355,9 @@ func (s stubReader) GetScopeByName(ctx context.Context, name string) (*api.Scope return nil, nil } -func (s stubReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) { +func (s stubReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { if s.listTraceTags != nil { - return s.listTraceTags(ctx, iatas, scope, traceType, since, until, cursor, limit) + return s.listTraceTags(ctx, iatas, scope, traceType, since, until, cursor, cursorTag, limit) } return nil, nil } diff --git a/internal/api/handlers/traces.go b/internal/api/handlers/traces.go index 54fd96dd..9f131b5e 100644 --- a/internal/api/handlers/traces.go +++ b/internal/api/handlers/traces.go @@ -4,6 +4,7 @@ package handlers import ( + "encoding/hex" "net/http" "strconv" "strings" @@ -36,7 +37,8 @@ func TracesRouter(reader api.Reader) http.Handler { // @Param type query string false "Filter by type: TRACE (any multi-hop packet) or PING (default: all)" // @Param since query int false "Filter by first_heard_at >= since (epoch ms)" // @Param until query int false "Filter by first_heard_at <= until (epoch ms)" -// @Param cursor query int false "last_heard_at epoch ms of last item for pagination" +// @Param cursor query int false "lastHeardAt (epoch ms) of the last item; returns older tags" +// @Param cursorTag query string false "traceTag (hex) of the last item; with cursor, also returns later tags sharing that millisecond" // @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Failure 400 {object} handlers.APIError // @Success 200 {object} []api.TraceTagSummary @@ -65,6 +67,18 @@ func listTraceTags(reader api.Reader) http.HandlerFunc { cursor = time.UnixMilli(ms) } } + // cursorTag breaks ties within the cursor's millisecond. + cursorTag := strings.ToLower(r.URL.Query().Get("cursorTag")) + if cursorTag != "" { + if b, err := hex.DecodeString(cursorTag); err != nil || len(b) == 0 { + respondError(w, http.StatusBadRequest, "cursorTag must be hex") + return + } + if cursor.IsZero() { + respondError(w, http.StatusBadRequest, "cursorTag requires cursor") + return + } + } iatas := parseIATAs(r) if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) @@ -76,7 +90,7 @@ func listTraceTags(reader api.Reader) http.HandlerFunc { } scope := r.URL.Query().Get("scope") traceType := strings.ToUpper(r.URL.Query().Get("type")) - tags, err := reader.ListTraceTags(r.Context(), iatas, scope, traceType, since, until, cursor, limit) + tags, err := reader.ListTraceTags(r.Context(), iatas, scope, traceType, since, until, cursor, cursorTag, limit) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/traces_test.go b/internal/api/handlers/traces_test.go index 722d8d3d..c69c169f 100644 --- a/internal/api/handlers/traces_test.go +++ b/internal/api/handlers/traces_test.go @@ -17,7 +17,7 @@ import ( func TestListTraceTags_OK(t *testing.T) { r := chi.NewRouter() r.Get("/traces", listTraceTags(stubReader{ - listTraceTags: func(_ context.Context, _ []string, _, _ string, _, _ time.Time, _ time.Time, _ int32) ([]api.TraceTagSummary, error) { + listTraceTags: func(_ context.Context, _ []string, _, _ string, _, _ time.Time, _ time.Time, _ string, _ int32) ([]api.TraceTagSummary, error) { return []api.TraceTagSummary{{TraceTag: "trace-001"}}, nil }, })) @@ -29,6 +29,39 @@ func TestListTraceTags_OK(t *testing.T) { } } +func TestListTraceTags_CursorTag(t *testing.T) { + var gotCursor time.Time + var gotTag string + r := chi.NewRouter() + r.Get("/traces", listTraceTags(stubReader{ + listTraceTags: func(_ context.Context, _ []string, _, _ string, _, _ time.Time, cursor time.Time, cursorTag string, _ int32) ([]api.TraceTagSummary, error) { + gotCursor, gotTag = cursor, cursorTag + return nil, nil + }, + })) + for _, tc := range []struct { + query string + code int + tag string + }{ + {"cursor=1700000000000&cursorTag=A1B2C3D4", http.StatusOK, "a1b2c3d4"}, + {"cursor=1700000000000", http.StatusOK, ""}, + {"cursor=1700000000000&cursorTag=zz", http.StatusBadRequest, ""}, + {"cursorTag=a1b2c3d4", http.StatusBadRequest, ""}, + } { + gotTag = "" + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/traces?"+tc.query, nil)) + if w.Code != tc.code { + t.Errorf("%s: code %d, want %d", tc.query, w.Code, tc.code) + continue + } + if tc.code == http.StatusOK && (gotTag != tc.tag || gotCursor.UnixMilli() != 1700000000000) { + t.Errorf("%s: cursor %v tag %q", tc.query, gotCursor, gotTag) + } + } +} + func TestGetTrace_OK(t *testing.T) { r := chi.NewRouter() r.Get("/traces/{tag}", getTrace(stubReader{ diff --git a/internal/api/reader.go b/internal/api/reader.go index db267190..36c67e07 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -218,8 +218,8 @@ type Reader interface { // Returns nil, pgx.ErrNoRows if the scope is not found. GetScopeByName(ctx context.Context, name string) (*ScopeDetail, error) - // ListTraceTags returns a paginated list of trace tags with aggregate metadata. - ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]TraceTagSummary, error) + // ListTraceTags returns trace tags newest first, keyset-paged on (cursor ms, cursorTag hex). + ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]TraceTagSummary, error) // GetTraceByTag returns all packets for a given trace tag with resolved routes. GetTraceByTag(ctx context.Context, tag string) (*TraceDetail, error) diff --git a/internal/api/stats.go b/internal/api/stats.go index fe3b6973..7e1e7596 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -35,7 +35,7 @@ type StatsOverview struct { TotalObservations int64 `json:"totalObservations"` ActiveObservers int64 `json:"activeObservers"` ActiveIATAs int64 `json:"activeIatas"` - WindowHours int `json:"windowHours"` // always 24 for now + WindowHours int `json:"windowHours"` // complete rolled hours the totals cover, at most 24 Since int64 `json:"since"` // epoch ms, start of the window Until int64 `json:"until"` // epoch ms, exclusive end of the window } diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 2f912f07..3af439d5 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -220,7 +220,7 @@ func (s *stubReader) SearchCrossIATARoutes(_ context.Context, _, _, _, _ string) return nil, nil } -func (s *stubReader) ListTraceTags(_ context.Context, _ []string, _, _ string, _, _ time.Time, _ time.Time, _ int32) ([]api.TraceTagSummary, error) { +func (s *stubReader) ListTraceTags(_ context.Context, _ []string, _, _ string, _, _ time.Time, _ time.Time, _ string, _ int32) ([]api.TraceTagSummary, error) { return nil, nil } diff --git a/internal/cache/reader.go b/internal/cache/reader.go index a78dc436..372cae62 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -375,6 +375,6 @@ func (cr *CachedReader) SearchCrossIATARoutes(ctx context.Context, fromHash, fro } // ListTraceTags implements [api.Reader]. -func (cr *CachedReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, limit int32) ([]api.TraceTagSummary, error) { - return cr.inner.ListTraceTags(ctx, iatas, scope, traceType, since, until, cursor, limit) +func (cr *CachedReader) ListTraceTags(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) { + return cr.inner.ListTraceTags(ctx, iatas, scope, traceType, since, until, cursor, cursorTag, limit) } From b30668bac4dac168861d4ffc6c1f93bd9628cda5 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 16:49:09 -0400 Subject: [PATCH 132/141] fix(ws): keep the newest event on overflow and apply route/observer filters A full send buffer dropped the oldest event and then the new one too while reporting one drop. Subscribe accepted routeTypes and observerIds but never applied them. --- internal/hub/hub.go | 17 +++++++++++- internal/hub/hub_test.go | 56 +++++++++++++++++++++++++++++++++++++++ internal/ingest/ingest.go | 2 ++ internal/ingest/status.go | 2 +- internal/ws/handler.go | 5 ++-- 5 files changed, 78 insertions(+), 4 deletions(-) diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 7f67480b..87a9ca06 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -54,17 +54,20 @@ type Event struct { IATA string PayloadType uint8 ChannelHash string // hex string, non-empty only for channelMessage events + RouteType uint8 // packetObservation only + ObserverID string // packetObservation and observerStatus only // Repeat marks a later hearing of a stored observation; only IncludeRepeats clients get it. Repeat bool } // Scope mirrors the client-side subscribe message. All fields are optional: // nil/empty means "no filter on this dimension" (match everything). -// An empty non-nil slice means "match nothing on this dimension". type Scope struct { IATAs []string PayloadTypes []uint8 + RouteTypes []uint8 ChannelHashes []string + ObserverIDs []string Events []EventType } @@ -146,6 +149,12 @@ func scopeMatches(s Scope, e Event) bool { if e.Type == EventChannelMessage && len(s.ChannelHashes) > 0 && !slices.Contains(s.ChannelHashes, e.ChannelHash) { return false } + if e.Type == EventPacketObservation && len(s.RouteTypes) > 0 && !slices.Contains(s.RouteTypes, e.RouteType) { + return false + } + if (e.Type == EventPacketObservation || e.Type == EventObserverStatus) && len(s.ObserverIDs) > 0 && !slices.Contains(s.ObserverIDs, e.ObserverID) { + return false + } return true } @@ -364,12 +373,18 @@ func (h *Hub) Run() { select { case c.Send <- outEvt: default: + // Evict the oldest so the newest still goes out. dropped := 1 select { case <-c.Send: default: } select { + case c.Send <- outEvt: + default: + dropped++ + } + select { case c.laggedCH <- LaggedNotification{DroppedCount: dropped}: default: // laggedCh itself full; write pump will catch up on next drain diff --git a/internal/hub/hub_test.go b/internal/hub/hub_test.go index bb8fc7a9..dbaaf0ef 100644 --- a/internal/hub/hub_test.go +++ b/internal/hub/hub_test.go @@ -240,6 +240,62 @@ func TestHub_Broadcast_FullBuffer_SendsLaggedNotification(t *testing.T) { } } +func TestScopeMatches_RouteTypeFilter(t *testing.T) { + s := Scope{RouteTypes: []uint8{1}} + if !scopeMatches(s, Event{Type: EventPacketObservation, RouteType: 1}) { + t.Error("expected flood observation to match") + } + if scopeMatches(s, Event{Type: EventPacketObservation, RouteType: 2}) { + t.Error("expected direct observation not to match") + } + if !scopeMatches(s, Event{Type: EventNodeUpdate}) { + t.Error("routeTypes should only filter packetObservation") + } +} + +func TestScopeMatches_ObserverFilter(t *testing.T) { + s := Scope{ObserverIDs: []string{"obs-a"}} + for _, typ := range []EventType{EventPacketObservation, EventObserverStatus} { + if !scopeMatches(s, Event{Type: typ, ObserverID: "obs-a"}) { + t.Errorf("%s from obs-a should match", typ) + } + if scopeMatches(s, Event{Type: typ, ObserverID: "obs-b"}) { + t.Errorf("%s from obs-b should not match", typ) + } + } + if !scopeMatches(s, Event{Type: EventChannelMessage}) { + t.Error("observerIds should not filter channelMessage") + } +} + +func TestHub_Broadcast_FullBuffer_KeepsNewestEvent(t *testing.T) { + h := runHub(t) + c := h.NewClient() + h.AddScope(c, "sub1", Scope{Events: []EventType{EventPacketObservation}}) + time.Sleep(10 * time.Millisecond) + + for i := 0; i < cap(c.Send); i++ { + h.Broadcast(Event{Type: EventPacketObservation, IATA: "YVR", Payload: json.RawMessage(`"old"`)}) + } + h.Broadcast(Event{Type: EventPacketObservation, IATA: "YVR", Payload: json.RawMessage(`"new"`)}) + + select { + case notif := <-c.LaggedCH(): + if notif.DroppedCount != 1 { + t.Errorf("DroppedCount = %d, want 1", notif.DroppedCount) + } + case <-time.After(200 * time.Millisecond): + t.Fatal("expected lagged notification, timed out") + } + var last Event + for len(c.Send) > 0 { + last = <-c.Send + } + if string(last.Payload) != `"new"` { + t.Errorf("newest queued event = %s, want the event that overflowed the buffer", last.Payload) + } +} + func TestClientMatches_NoSubscriptions(t *testing.T) { c := &Client{subscriptions: make(map[string]Scope)} if c.matches(Event{Type: EventPacketObservation}) { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index cf00b0c1..6c3abb72 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -453,6 +453,8 @@ func (w *Worker) broadcastPacketObservation(iata string, payloadType uint8, evt Payload: base, IATA: iata, PayloadType: payloadType, + RouteType: evt.Packet.RouteType, + ObserverID: evt.Observation.ObserverID, } evt.Observation.ResolvedPath = resolvedPath if out.PayloadResolved, err = json.Marshal(evt); err != nil { diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 2e6d8cd0..9d3222eb 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -315,5 +315,5 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) w.log.Error(fmt.Sprintf("failed to marshal status event payload for %s", pubkeyHex), "error", err) return } - w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus, Payload: payload, IATA: iata}) + w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus, Payload: payload, IATA: iata, ObserverID: evt.ObserverID}) } diff --git a/internal/ws/handler.go b/internal/ws/handler.go index 4e93995d..9cd27bfe 100644 --- a/internal/ws/handler.go +++ b/internal/ws/handler.go @@ -9,7 +9,7 @@ // // Client → Server: // subscribe { v, type, id, scope } → server replies subscribed { v, type, id, subscriptionId } -// unsubscribe { v, type, id, subscriptionId } +// unsubscribe { v, type, id, subscriptionId } → server replies unsubscribed { v, type, id, subscriptionId } // configure { v, type, id, resolvePath, includeObserverKey, includeRepeats } // → server replies configured { v, type, id, resolvePath, includeObserverKey, includeRepeats } // ping { v, type, id } → server replies pong { v, type, id } @@ -25,7 +25,6 @@ // Server → Client events (unsolicited): // packetObservation, observerStatus, nodeUpdate, channelMessage // lagged { v, type, droppedCount, since } -// error { v, type, code, message } // // Idle connections (no ping) closed after 90s. // Client should ping every 30s. @@ -235,7 +234,9 @@ func handleClientMessage(ctx context.Context, client *hub.Client, reader api.Rea scope := hub.Scope{ IATAs: iatas, PayloadTypes: msg.Scope.PayloadTypes, + RouteTypes: msg.Scope.RouteTypes, ChannelHashes: msg.Scope.ChannelHashes, + ObserverIDs: msg.Scope.ObserverIDs, Events: msg.Scope.Events, } subID := uuid.NewString() From ec0ecd321bbc807cbed2dcbc82b7cef891c3ca8e Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 16:49:09 -0400 Subject: [PATCH 133/141] refactor(db): move the heard-IATA lookup into sqlc --- db/meshmapper.go | 6 +----- db/queries/queries.sql | 3 +++ db/sqlc/mock/querier.go | 15 +++++++++++++++ db/sqlc/querier.go | 1 + db/sqlc/queries.sql.go | 24 ++++++++++++++++++++++++ 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/db/meshmapper.go b/db/meshmapper.go index 82e2d59d..fa2120de 100644 --- a/db/meshmapper.go +++ b/db/meshmapper.go @@ -79,11 +79,7 @@ func (s *Store) ListKnownIATAs(ctx context.Context) ([]string, error) { // ListHeardIATAs returns the IATAs observers last reported from. func (s *Store) ListHeardIATAs(ctx context.Context) ([]string, error) { - rows, err := s.pool.Query(ctx, `SELECT DISTINCT last_iata::text FROM observers WHERE last_iata IS NOT NULL ORDER BY 1`) - if err != nil { - return nil, err - } - return pgx.CollectRows(rows, pgx.RowTo[string]) + return s.q.ListHeardIATAs(ctx) } func (s *Store) PruneZoneBoundaries(ctx context.Context, keep []string) ([]string, error) { diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 8ca8b3de..ecdcffe1 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -21,6 +21,9 @@ SELECT * FROM iata_codes WHERE iata = $1; -- name: ListIATAs :many SELECT * FROM iata_codes ORDER BY iata; +-- name: ListHeardIATAs :many +SELECT DISTINCT last_iata::text AS iata FROM observers WHERE last_iata IS NOT NULL ORDER BY 1; + -- name: UpsertIATADetails :exec INSERT INTO iata_codes (iata, display_name, approx_lat, approx_lng) VALUES ($1, $2, $3, $4) diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index e61bc786..71191e67 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -1316,6 +1316,21 @@ func (mr *MockQuerierMockRecorder) ListDirtyRollupHours(ctx, limit any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDirtyRollupHours", reflect.TypeOf((*MockQuerier)(nil).ListDirtyRollupHours), ctx, limit) } +// ListHeardIATAs mocks base method. +func (m *MockQuerier) ListHeardIATAs(ctx context.Context) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListHeardIATAs", ctx) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListHeardIATAs indicates an expected call of ListHeardIATAs. +func (mr *MockQuerierMockRecorder) ListHeardIATAs(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListHeardIATAs", reflect.TypeOf((*MockQuerier)(nil).ListHeardIATAs), ctx) +} + // ListIATAs mocks base method. func (m *MockQuerier) ListIATAs(ctx context.Context) ([]db.IataCode, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 8731ece4..4f56b0f7 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -200,6 +200,7 @@ type Querier interface { // Only complete hours whose raw rows are still intact can be re-rolled. Queued missing hours // wait: their first roll consumes the entry only if the entry predates its snapshot. ListDirtyRollupHours(ctx context.Context, limit int32) ([]pgtype.Timestamptz, error) + ListHeardIATAs(ctx context.Context) ([]string, error) ListIATAs(ctx context.Context) ([]IataCode, error) // Only one branch runs. Keep the IATA range ordered by the composite index: // generic plans can otherwise prefer scanning the global timestamp index. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index dfa697d0..435bb941 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2912,6 +2912,30 @@ func (q *Queries) ListChannelsAfter(ctx context.Context, arg ListChannelsAfterPa return items, nil } +const listHeardIATAs = `-- name: ListHeardIATAs :many +SELECT DISTINCT last_iata::text AS iata FROM observers WHERE last_iata IS NOT NULL ORDER BY 1 +` + +func (q *Queries) ListHeardIATAs(ctx context.Context) ([]string, error) { + rows, err := q.db.Query(ctx, listHeardIATAs) + if err != nil { + return nil, err + } + defer rows.Close() + items := []string{} + for rows.Next() { + var iata string + if err := rows.Scan(&iata); err != nil { + return nil, err + } + items = append(items, iata) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listIATAs = `-- name: ListIATAs :many SELECT iata, display_name, approx_lat, approx_lng, added_at, border FROM iata_codes ORDER BY iata ` From 951e0b0de6e39033519ab7fd077694f27eae5353 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 16:49:09 -0400 Subject: [PATCH 134/141] docs: regenerate swagger --- docs/docs.go | 14 ++++++++++---- docs/swagger.json | 14 ++++++++++---- docs/swagger.yaml | 19 ++++++++++++------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 43664243..466c4885 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2618,7 +2618,7 @@ const docTemplate = `{ }, "/stats/paths": { "get": { - "description": "Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads use materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and missing hours are omitted.", + "description": "Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly path rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted.", "produces": [ "application/json" ], @@ -2937,7 +2937,7 @@ const docTemplate = `{ }, "/stats/signal": { "get": { - "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly signal rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", "produces": [ "application/json" ], @@ -3311,10 +3311,16 @@ const docTemplate = `{ }, { "type": "integer", - "description": "last_heard_at epoch ms of last item for pagination", + "description": "lastHeardAt (epoch ms) of the last item; returns older tags", "name": "cursor", "in": "query" }, + { + "type": "string", + "description": "traceTag (hex) of the last item; with cursor, also returns later tags sharing that millisecond", + "name": "cursorTag", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -5325,7 +5331,7 @@ const docTemplate = `{ "type": "integer" }, "windowHours": { - "description": "always 24 for now", + "description": "complete rolled hours the totals cover, at most 24", "type": "integer" } } diff --git a/docs/swagger.json b/docs/swagger.json index 0d7a5fc9..63ce584f 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2616,7 +2616,7 @@ }, "/stats/paths": { "get": { - "description": "Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads use materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and missing hours are omitted.", + "description": "Counts stored receptions in [since, until), at most 30 days. Hash widths count only validated nonempty ordinary header paths (1/2/3 bytes). Empty paths do not vote for a width. TRACE header paths contain signal readings and are separate. Missing payload type or inconsistent/unsupported path metadata is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly path rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted.", "produces": [ "application/json" ], @@ -2935,7 +2935,7 @@ }, "/stats/signal": { "get": { - "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads use hourly materialized snapshots refreshed by background.view_refresh; the current partial hour is excluded and absent hours are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", + "description": "Aggregates stored observer receptions in [since, until), at most 30 days. SNR is dB; RSSI is dBm. Null/non-finite readings and the zero/zero unavailable sentinel are excluded per metric; actual zero SNR with nonzero RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads the hourly signal rollup; an hour appears only after it is rolled, about 95 minutes after it ends, and hours without data are omitted. These last-hop readings do not measure end-to-end quality or packet loss.", "produces": [ "application/json" ], @@ -3309,10 +3309,16 @@ }, { "type": "integer", - "description": "last_heard_at epoch ms of last item for pagination", + "description": "lastHeardAt (epoch ms) of the last item; returns older tags", "name": "cursor", "in": "query" }, + { + "type": "string", + "description": "traceTag (hex) of the last item; with cursor, also returns later tags sharing that millisecond", + "name": "cursorTag", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -5323,7 +5329,7 @@ "type": "integer" }, "windowHours": { - "description": "always 24 for now", + "description": "complete rolled hours the totals cover, at most 24", "type": "integer" } } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index a0963a4a..132e0f88 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1350,7 +1350,7 @@ definitions: description: epoch ms, exclusive end of the window type: integer windowHours: - description: always 24 for now + description: complete rolled hours the totals cover, at most 24 type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.StatsSeries: @@ -3406,9 +3406,9 @@ paths: is unclassified. The four categories partition total receptions. Flood paths accumulate entries; direct paths contain remaining entries, so counts do not measure distance or a complete traversed route. Both boundaries round down - to UTC hours and the response reports that effective window. Reads use materialized - snapshots refreshed by background.view_refresh; the current partial hour is - excluded and missing hours are omitted. + to UTC hours and the response reports that effective window. Reads the hourly + path rollup; an hour appears only after it is rolled, about 95 minutes after + it ends, and hours without data are omitted. parameters: - description: Inclusive start, epoch milliseconds (0 through 253402300799999) in: query @@ -3643,8 +3643,8 @@ paths: RSSI remains valid. Averages are null without samples. Histogram bounds are lower-inclusive/upper-exclusive, with null for unbounded ends. Both boundaries round down to UTC hours and the response reports that effective window. Reads - use hourly materialized snapshots refreshed by background.view_refresh; the - current partial hour is excluded and absent hours are omitted. These last-hop + the hourly signal rollup; an hour appears only after it is rolled, about 95 + minutes after it ends, and hours without data are omitted. These last-hop readings do not measure end-to-end quality or packet loss. parameters: - description: Inclusive start, epoch milliseconds (0 through 253402300799999) @@ -3897,10 +3897,15 @@ paths: in: query name: until type: integer - - description: last_heard_at epoch ms of last item for pagination + - description: lastHeardAt (epoch ms) of the last item; returns older tags in: query name: cursor type: integer + - description: traceTag (hex) of the last item; with cursor, also returns later + tags sharing that millisecond + in: query + name: cursorTag + type: string - description: Max results (default 50); must be positive, values above 200 are clamped in: query From 5b493564376bd60eb1b4edbf53e62a810bfdccdb Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 16:49:09 -0400 Subject: [PATCH 135/141] docs: update for 2.0.0 --- CONTRIBUTING.md | 4 +- README.md | 247 ++++++++++++++++++++++++++++----------- SHOULDERS.md | 11 ++ config.yaml.example | 13 ++- docs/historical-stats.md | 2 +- env.example | 6 +- 6 files changed, 209 insertions(+), 74 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3656d682..04209e74 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,9 @@ go build ./... # must compile gofmt -l . # must be empty (no unformatted files) go vet ./... # no warnings go test ./... # all tests pass -swag init # if you changed any handler or api type (see below) +govulncheck ./... # no reachable vulnerabilities +swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency + # if you changed any handler or api type (see below) ``` --- diff --git a/README.md b/README.md index 5e664f69..80c00d61 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ in PostgreSQL, and streams live events to WebSocket clients. - Seeds regions, IATA display names, and channel keys from a YAML config file on startup -For deployment instructions including the frontend app, see the deployment docs. +For deployment instructions including the frontend app and reverse proxy +examples, see [beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs). For a bounded private database and saved-config bundle, see [backup export](docs/backup-export.md). A standalone export tool is available; the @@ -71,12 +72,33 @@ define your regions, IATA display names, channel keys, and retention settings. ### 2. Start PostgreSQL +Any PostgreSQL 16 database works. For a local throwaway one: + ```bash -docker compose up postgres -d +docker run -d --name beacon-postgres -p 5432:5432 \ + -e POSTGRES_USER=beacon -e POSTGRES_PASSWORD=beacon -e POSTGRES_DB=beacon \ + postgres:16-alpine ``` Database migrations are applied automatically on startup. +### Upgrading from 1.x + +2.0.0 starts from a new schema baseline and **needs a fresh database**. Pointed +at a 1.x database it refuses to start with `database schema predates Beacon +2.0.0; 2.0.0 needs a fresh database`. 1.x history does not carry over: create a +new, empty database (Docker: stop the stack and move or remove the Postgres +data directory) before starting 2.0.0. + +Config changes to review when upgrading: + +- Every manually configured `scopes:` entry needs a `region`. +- `meshmapper.scopes.sources` is ignored; MeshMapper refresh intervals must be + within 1h–24h for scopes and 24h–168h for zones, or startup fails. +- REST rate limiting is on by default. Behind a reverse proxy, set + `server.trusted_proxies` (see [Reverse proxies and rate limits](#reverse-proxies-and-rate-limits)). +- `packets.retention` now defaults to 7 days. + ### 3. Run ```bash @@ -90,6 +112,8 @@ docker pull ghcr.io/meshcore-beacon/beacon-server:latest ``` The image is public on GitHub Container Registry — no `docker login` required. +`latest` tracks stable releases; pin a version tag (e.g. `2.0.0`) for +predictable upgrades, and `dev` follows the development branch. Beacon will: @@ -139,7 +163,9 @@ Global CORS preflights remain public. Use a long, randomly generated key, keep it out of source control and logs, and send it only in the Authorization header, never the URL or request body. Require HTTPS at the reverse proxy and restrict direct access to Beacon's HTTP listener to that proxy or a private connection. -Changing the key requires a restart. No API key is issued automatically. +Changing the key requires a restart. No API key is issued automatically. The +key must be at least 16 characters with no inner whitespace (surrounding +whitespace is trimmed); an unusable key prevents startup. `PUT /api/v1/admin/config` accepts only `{"cors":{"allowed_origins":["https://example.org"]}}`. It replaces the entire @@ -188,6 +214,11 @@ even when the same bearer-authenticated request works with curl. | `MQTT_BROKER_2_URL` | — | Broker 2 WebSocket URL | | `MQTT_BROKER_2_USERNAME` | — | Broker 2 username | | `MQTT_BROKER_2_PASSWORD` | — | Broker 2 password | +| `BEACON_API_KEY` | — | Admin bearer key; overrides `auth.api_key` when set | +| `LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error`; overrides `log.level` | +| `LOG_FORMAT` | `text` | `text` or `json`; overrides `log.format` | +| `BEACON_CPU_PROFILE_DIR` | — | Private CPU capture directory (see [PROFILING.md](PROFILING.md)) | +| `BEACON_CPU_PROFILE_UNTIL` | — | RFC 3339 deadline for CPU captures, at most 72h ahead | ### Config file (`config.yaml`) @@ -277,9 +308,20 @@ presence: flush_interval: 30s # how often coalesced bumps are flushed (default: 30s) packet_ttl: 30s # how long a quiet packet stays coalesced before writing through again (default: 30s) +# Direct proxy peers allowed to set the client IP through X-Real-IP. +server: + trusted_proxies: [172.30.0.0/24] # default: [] (direct access) + +# Per-client REST limits for /api/v1/*. +ratelimit: + enabled: true # default: true + requests_per_minute: 300 # default: 300 + # WebSocket settings. websocket: max_connections_per_ip: 5 # default: 5 + max_connects_per_minute: 10 # upgrade attempts per IP (default: 10) + allowed_origins: [https://beacon.example.com] # other sites allowed to open /ws (default: same host only) # Node staleness, deletion, and clock-drift thresholds. nodes: @@ -323,7 +365,38 @@ ingest: IATAs are auto-created on first packet arrival. The config file adds display names, coordinates, and optional region borders. Regions and channel keys -must be defined here — they are not auto-created. +must be defined here unless MeshMapper imports them (`meshmapper.zones.import_groups`, +`meshmapper.channels`). + +--- + +### Reverse proxies and rate limits + +REST requests under `/api/v1` are rate limited per client IP (300/min by +default; IPv6 clients share a /64). Exhausted clients get `429` with +`rate_limited` and `Retry-After`. WebSocket upgrades are limited separately by +`websocket.max_connects_per_minute`. + +Beacon never trusts `X-Forwarded-For` or `True-Client-IP`. Behind a reverse +proxy, list the proxy's address in `server.trusted_proxies` (CIDR, e.g. +`127.0.0.1/32` or the Docker network subnet) and have the proxy overwrite +`X-Real-IP` with the connecting client's address: + +```nginx +proxy_set_header X-Real-IP $remote_addr; +``` + +```caddy +reverse_proxy app:8080 { + header_up X-Real-IP {remote_host} +} +``` + +Without this every visitor shares the proxy's budget and the site gets 429s +under normal load. Beacon logs a warning at startup when limits are on and no +proxy is trusted, and once at runtime when a request carries forwarding headers +it is ignoring. Full nginx and Caddy examples are in +[beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs). --- @@ -331,10 +404,11 @@ must be defined here — they are not auto-created. Set `nodes.mark_foreign: true` to expose `possiblyForeign` on repeater nodes. The local operating area is the union of **all configured** -`iatas..borderFile` GeoJSON Polygon/MultiPolygon features. IATAs without -border files do not add an area; airport coordinates and the current API region -filter are not boundaries. Enabling this with no borders, missing files or -invalid geometry fails startup. Border changes require a restart. +`iatas..borderFile` GeoJSON Polygon/MultiPolygon features, with +`meshmapper.zones` boundaries replacing them where MeshMapper has one. IATAs +without a border do not add an area; airport coordinates and the current API +region filter are not boundaries. Enabling this with no border source, missing +files or invalid geometry fails startup. Border file changes require a restart. Inside any polygon (including its edges) means `false`; outside the entire union means `true`. Hole interiors are outside; hole edges are local. Other node roles, @@ -359,9 +433,9 @@ silently treating them as the complementary global area. ## Authentication -API authentication is not yet implemented. Beacon is intended for trusted -internal network or reverse-proxy deployments. Do not expose it directly to the -public internet without an authentication layer in front of it. +Public reads and the WebSocket need no credentials. Only the `/api/v1/admin` +subtree is protected, by a bearer key (see [Admin authentication](#admin-authentication)). +Serve Beacon behind an HTTPS reverse proxy. --- @@ -395,14 +469,17 @@ unsubscribing. "regionIds": ["1"], "regionSlugs": ["western-canada"], "payloadTypes": [4, 5], + "routeTypes": [1, 2], "channelHashes": ["11"], + "observerIds": [""], "events": ["packetObservation", "channelMessage"] } } ``` -All scope fields are optional. Omitted means no filter on that dimension (match -everything). Empty array means match nothing on that dimension. `regionIds` and +All scope fields are optional; omitted or empty means no filter on that +dimension. `routeTypes` and `observerIds` filter `packetObservation` events +(`observerIds` also `observerStatus`), and `channelHashes` only `channelMessage`. `regionIds` and `regionSlugs` are both expanded to their member IATAs server-side. **Unsubscribe** — remove a specific subscription by ID. @@ -416,21 +493,27 @@ everything). Empty array means match nothing on that dimension. `regionIds` and } ``` -**Configure** — toggle connection-wide settings. Currently just `resolvePath`, -which enables per-hop node resolution on `packetObservation` events (see -below). Unlike `subscribe`, this is a single flag for the whole connection, -not additive across calls — each `configure` sets it to exactly the value -sent, and it can be flipped on or off as many times as you like for the life -of the connection. Default is `false`. +The server replies `{ "v": 1, "type": "unsubscribed", "id": "unsub-1", "subscriptionId": "..." }`. + +**Configure** — connection-wide flags, all default `false`: + +- `resolvePath` adds per-hop node resolution to `packetObservation` events (see below). +- `includeObserverKey` adds `observation.observerPublicKey`. +- `includeRepeats` also streams later hearings of an already-stored observation + over a new path, as `packetObservation` events with `packet.isRepeat: true` + and `observationCount: 0`. + +Unlike `subscribe`, this is not additive: each `configure` sets all three flags +to exactly the values sent, so an omitted flag turns off. ```json -{ "v": 1, "type": "configure", "id": "cfg-1", "resolvePath": true } +{ "v": 1, "type": "configure", "id": "cfg-1", "resolvePath": true, "includeRepeats": true } ``` The server replies: ```json -{ "v": 1, "type": "configured", "id": "cfg-1", "resolvePath": true } +{ "v": 1, "type": "configured", "id": "cfg-1", "resolvePath": true, "includeObserverKey": false, "includeRepeats": true } ``` **Ping** @@ -466,21 +549,28 @@ notice: { "v": 1, "type": "lagged", "droppedCount": 12, "since": 1234567890000 } ``` -Clients should respond by re-fetching the relevant REST endpoint using `afterId` -to backfill missed events, then resume streaming. +Clients should respond by backfilling over REST, then resume streaming: +`/api/v1/packets/backfill?afterObservationId=` and +`/api/v1/messages/backfill?afterId=`. ### Reconnection Subscriptions are not persisted — they exist only for the lifetime of the connection. On any disconnect the client should reconnect with backoff, re-issue -all subscriptions, and backfill via REST using -`afterId=`. +all subscriptions, and backfill via the same REST endpoints. ### Connection limits -By default a maximum of 5 concurrent WebSocket connections are allowed per IP -address. Connections beyond this limit receive `HTTP 429`. The limit is -configurable via `websocket.max_connections_per_ip` in `config.yaml`. +- At most `websocket.max_connections_per_ip` (default 5) concurrent connections + per IP. A connection over the cap is accepted and then closed with code + `1013` (try again later) before `hello`. +- At most `websocket.max_connects_per_minute` (default 10) upgrade attempts per + IP; beyond that the handshake gets `429` with `Retry-After: 60`. +- Browsers may only connect from the same host unless their origin is listed in + `websocket.allowed_origins` (`https://*.example.com` covers every subdomain). + +Behind a reverse proxy both limits need `server.trusted_proxies`, or every +visitor counts as the proxy's IP. --- @@ -492,52 +582,69 @@ All list endpoints support cursor-based pagination via `cursor` and `limit` query params. See the Swagger UI at `http://localhost:8080/swagger/index.html` for full parameter documentation. -### Authentication +### Authentication and limits -Not yet implemented — see the Authentication section above. +Only `/admin/*` needs `Authorization: Bearer `. All `/api/v1` routes are +rate limited per client IP; see [Reverse proxies and rate limits](#reverse-proxies-and-rate-limits). +List `limit` values are clamped to 1–200. ### Endpoints -| Method | Path | Description | -| ------ | ----------------------------------- | -------------------------------------------------------------------------------------------------- | -| `GET` | `/brokers` | List MQTT brokers and connection status | -| `GET` | `/channels` | List channels (optional: `?hash=&iata=&limit=50`) | -| `GET` | `/channels/{id}` | Get channel detail by integer ID | -| `GET` | `/channels/{id}/messages` | List messages for a channel (optional: `?since=&iata=&limit=50`) | -| `GET` | `/iatas` | List all known IATA codes | -| `GET` | `/iatas/{iata}` | Get a single IATA code | -| `GET` | `/iatas/{iata}/border` | Get an IATA's GeoJSON region border, if configured (204 if not) | -| `GET` | `/messages` | List all messages (optional: `?channelId=&channelHash=&iata=&since=&limit=50`) | -| `GET` | `/messages/backfill` | Backfill messages after a given message ID | -| `GET` | `/nodes` | List nodes (optional: `?pubkeyPrefix=&neighbors&iatas=&pubkey=`) | -| `GET` | `/nodes/{nodeId}` | Get node detail | -| `GET` | `/nodes/{nodeId}/neighbors` | List neighboring nodes observed in the mesh | -| `GET` | `/nodes/{nodeId}/observations` | List observations for a node | -| `GET` | `/observers` | List observers (optional: `?iata=&type=&broker=&status=online\|offline`) | -| `GET` | `/observers/{observerId}` | Get observer detail including broker last-seen timestamps | -| `GET` | `/observers/{observerId}/adverts` | Adverts heard by observer | -| `GET` | `/observers/{observerId}/telemetry` | Observer telemetry history (optional: `?range=24h&interval=1h\|6h\|24h`) | -| `GET` | `/packets` | List packets (optional: `?payloadTypes=&routeTypes=&scopes=` accept plural, comma-separated values alongside the singular params) | -| `GET` | `/packets/backfill` | Backfill packets after a given observation ID | -| `GET` | `/packets/{packetHash}` | Get packet with all observations | -| `GET` | `/regions` | List all regions (summary) | -| `GET` | `/regions/{id}` | Get a single region with IATA list | -| `GET` | `/routes` | List known routes (all hops high confidence) | -| `GET` | `/routes/search` | Search routes by source and destination hash | -| `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | -| `GET` | `/scopes` | List transport scope names; IATA/region filters use configured regions and MeshMapper catalogues | -| `GET` | `/scopes/{name}` | Get scope detail | -| `GET` | `/stats/observations` | Hourly observation counts per IATA (last 7 days by default) | -| `GET` | `/stats/overview` | Network overview over the last 24 rolled hours | -| `GET` | `/stats/payload-breakdown` | Observation counts by payload type (last 24h by default) | -| `GET` | `/stats/scopes` | Configured region scopes and breakdown of packets, nodes, observers (last 7 days by default) | -| `GET` | `/stats/series` | Hourly card metrics and window summary from the rollups (`since`/`until` required) | -| `GET` | `/stats/top-advertisers` | Top N nodes by distinct ADVERT packet count (last 24h by default) | -| `GET` | `/stats/top-nodes` | Top N nodes by advert hearings (last 7 days by default) | -| `GET` | `/stats/top-observers` | Top N observers by observation count (last 24h by default) | -| `GET` | `/stats/top-talkers` | Top N companion names by decrypted channel message count (last 24h by default) | -| `GET` | `/traces` | List trace tags with filters (optional: ?type=TRACE\|PING) | -| `GET` | `/traces/{tag}` | Get full trace detail with resolved routes | +| Method | Path | Description | +| -------- | --------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `GET` | `/admin/accounts` | List operator accounts (bearer key) | +| `POST` | `/admin/accounts` | Create an operator account (bearer key) | +| `GET` | `/admin/accounts/{id}` | Get an operator account (bearer key) | +| `DELETE` | `/admin/accounts/{id}` | Deactivate an operator account (bearer key) | +| `GET` | `/admin/backup` | Download a private database and saved-config backup (bearer key, opt-in) | +| `GET` | `/admin/config` | Inspect selected running configuration (bearer key) | +| `PUT` | `/admin/config` | Replace runtime CORS origins (bearer key) | +| `GET` | `/brokers` | List MQTT brokers and connection status | +| `GET` | `/channels` | List channels (`hash`, `iata`/`iatas`, `keyKnown`, `pageCursor`, `limit`) | +| `GET` | `/channels/{channelID}` | Get channel detail by integer ID | +| `GET` | `/channels/{channelID}/messages` | List messages for a channel (`since`, `iatas`, `region`/`regionId`, `scope`, `cursor`, `limit`) | +| `GET` | `/iatas` | List all known IATA codes | +| `GET` | `/iatas/{iata}` | Get a single IATA code | +| `GET` | `/iatas/{iata}/border` | Get an IATA's GeoJSON border, if any (204 if not) | +| `GET` | `/messages` | List channel messages (`channelID`, `channelHash`, `since`, `iatas`, `region`/`regionId`, `scope`, `cursor`, `limit`) | +| `GET` | `/messages/backfill` | Messages after `afterId` | +| `GET` | `/nodes` | List nodes (type, IATA/region, name, scope, `pubkey`/`pubkeyPrefix`, capability flags, `neighbors`) | +| `GET` | `/nodes/{nodeId}` | Get node detail | +| `GET` | `/nodes/{nodeId}/neighbors` | List neighboring nodes observed in the mesh | +| `GET` | `/nodes/{nodeId}/observations` | List observations of packets from a node | +| `GET` | `/observers` | List observers (IATA/region, `type`, `broker`, `status=online\|offline`, `name`, `scope`) | +| `GET` | `/observers/{observerId}` | Get observer detail including broker last-seen timestamps | +| `GET` | `/observers/{observerId}/activity` | Heard-activity history (`range`, `interval`, `until`) | +| `GET` | `/observers/{observerId}/adverts` | Adverts heard by observer | +| `GET` | `/observers/{observerId}/telemetry` | Telemetry history (`range`, `interval=1h\|6h\|24h`, `afterId`) | +| `GET` | `/packets` | List packets (payload/route type, IATA/region, scope, `since`/`until`; plural params take CSV) | +| `GET` | `/packets/backfill` | Packets after `afterObservationId` | +| `GET` | `/packets/{packetHash}` | Get packet with all observations | +| `GET` | `/regions` | List all regions (summary) | +| `GET` | `/regions/{regionId}` | Get a single region with IATA list | +| `GET` | `/routes` | List known routes (`iata`, `hopCount`) | +| `GET` | `/routes/search` | Search routes by source and destination hash | +| `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | +| `GET` | `/routes/{iata}/{pathKey}/observations` | Retained observations matching a saved route (see below) | +| `GET` | `/scopes` | List transport scope names; IATA/region filters use configured regions and MeshMapper catalogues | +| `GET` | `/scopes/{name}` | Get scope detail | +| `GET` | `/stats/clock-drift` | Repeaters and room servers whose clock drifted past the threshold, worst first | +| `GET` | `/stats/node-types` | Node type breakdown | +| `GET` | `/stats/observations` | Hourly observation counts per IATA (last 7 days by default) | +| `GET` | `/stats/observer-comparison` | Compare flood packets reported by two observers | +| `GET` | `/stats/overview` | Network overview over the last 24 rolled hours | +| `GET` | `/stats/paths` | Path-entry and hash-width distributions | +| `GET` | `/stats/payload-breakdown` | Observation counts by payload type (last 24h by default) | +| `GET` | `/stats/radio-presets` | Radio preset usage by IATA | +| `GET` | `/stats/scopes` | Region scopes with hourly packet, node and observer counts (last 7 days by default) | +| `GET` | `/stats/series` | Hourly card metrics and window summary (`since`/`until` required) | +| `GET` | `/stats/signal` | Reception signal distributions and hourly trends | +| `GET` | `/stats/top-advertisers` | Top N nodes by distinct ADVERT packets, split flood/direct (last 24h by default) | +| `GET` | `/stats/top-nodes` | Top N nodes by advert hearings (last 7 days by default) | +| `GET` | `/stats/top-observers` | Top N observers by observation count (last 24h by default) | +| `GET` | `/stats/top-talkers` | Top N companion names by decrypted channel message count (last 24h by default) | +| `GET` | `/traces` | List trace tags (`type=TRACE\|PING`, IATA/region, scope, `since`/`until`, `cursor`+`cursorTag`) | +| `GET` | `/traces/{tag}` | Get full trace detail with resolved routes | --- diff --git a/SHOULDERS.md b/SHOULDERS.md index ef2b34e7..ad3b0ec4 100644 --- a/SHOULDERS.md +++ b/SHOULDERS.md @@ -24,6 +24,17 @@ projects that make this possible: - [google/uuid](https://github.com/google/uuid) — UUID generation - [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) — singleflight for collapsing concurrent cache misses +- [paulmach/orb](https://github.com/paulmach/orb) — GeoJSON border geometry and + point-in-polygon checks +- [swaggo/http-swagger](https://github.com/swaggo/http-swagger) — Swagger UI + handler + +## Testing + +- [go.uber.org/mock](https://github.com/uber-go/mock) — gomock mocks for the + generated queries +- [miniredis](https://github.com/alicebob/miniredis) — In-memory Redis for cache + tests ## Data diff --git a/config.yaml.example b/config.yaml.example index 3d91c8ff..d68edb8a 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,8 +1,12 @@ # Beacon configuration file # Copy to config.yaml and adjust as needed. +# Durations are Go strings (30s, 1h, 168h); negative values prevent startup +# (observers.delete_after excepted, where nonpositive disables it). # Public reads stay available. No key means admin routes return 503. # Prefer BEACON_API_KEY in the service environment; never commit a real key. +# At least 16 characters with no inner whitespace (surrounding whitespace is +# trimmed); a shorter key prevents startup. auth: api_key: "" @@ -77,7 +81,7 @@ ratelimit: # zone list, unless iatas: below sets a name or lat/lng for them. # import_groups turns MeshMapper zone groups (e.g. ONQC: YOW, YQB, YUL) into # regions, from the zone lists already fetched. A group is imported when at least -# one member is a known IATA, with its full member list (members Beacon hasn't +# one member is heard by an observer or configured here, with its full member list (members Beacon hasn't # heard yet are added as IATAs); slug = lowercased group code, centered on the # members' average location, listed after the regions below. A region below with # the same slug wins. @@ -199,6 +203,13 @@ packets: analytics: rollup_retention: 2160h # 90 days +# Presence write coalescing: observer last_seen and packet last_heard_at bumps are +# batched in memory and flushed on an interval. An unclean shutdown loses at most +# one interval of presence freshness. +#presence: +# flush_interval: 30s # default: 30s +# packet_ttl: 30s # how long a quiet packet stays coalesced (default: 30s) + # Known-route retention. Routes are distilled packet history (the path a packet # took, hop by hop), so they may outlive packets.retention; there is no required # relationship between the two. diff --git a/docs/historical-stats.md b/docs/historical-stats.md index 99812aa1..f9e03176 100644 --- a/docs/historical-stats.md +++ b/docs/historical-stats.md @@ -76,7 +76,7 @@ region or all IATAs. ## Adding a metric -1. Add a family table to `db/migrations/001_baseline.sql` (on 2.x, a new migration): `hour` +1. Add a family table in a new numbered migration (`001_baseline.sql` is frozen from 2.0.0): `hour` first in the primary key, no foreign keys, and a set table with `CHECK (iata_set_is_canonical(iatas))` plus a GIN index if it needs distinct counts. 2. Add a delete in `DeleteRollupHour`, a `Roll*` insert over `rollup_obs` in diff --git a/env.example b/env.example index b918e6e3..f2d930d4 100644 --- a/env.example +++ b/env.example @@ -1,6 +1,10 @@ LISTEN_ADDR=:8080 -# Optional admin bearer key. If set, overrides auth.api_key (even when empty). +# Path to the YAML config (default: config.yaml). +# CONFIG_PATH=config.yaml + +# Optional admin bearer key, at least 16 characters with no whitespace. +# If set, overrides auth.api_key (even when empty). # BEACON_API_KEY= # Optional application logging overrides (defaults: info / text). From a1aff810e9eb23e7781f4432443bd6252d5b18b4 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 18:58:50 -0400 Subject: [PATCH 136/141] fix(api): answer 404 only for missing rows Lookups turned every error into a 404, and region filters into a 400, so a transient database failure looked final to clients that don't retry 4xx. Missing rows stay 404/400; anything else is a 500. Empty /regions and /iatas lists are [] rather than null. --- internal/api/handlers/channels.go | 9 +- internal/api/handlers/iatas.go | 23 +++- internal/api/handlers/iatas_test.go | 4 +- internal/api/handlers/lookup_errors_test.go | 116 +++++++++++++++++++ internal/api/handlers/messages.go | 2 +- internal/api/handlers/nodes.go | 11 +- internal/api/handlers/observer_comparison.go | 2 +- internal/api/handlers/observers.go | 9 +- internal/api/handlers/packets.go | 12 +- internal/api/handlers/paths.go | 2 +- internal/api/handlers/paths_test.go | 3 +- internal/api/handlers/regions.go | 26 ++++- internal/api/handlers/scope_stats_test.go | 5 +- internal/api/handlers/scopes.go | 2 +- internal/api/handlers/signal.go | 2 +- internal/api/handlers/signal_test.go | 3 +- internal/api/handlers/stats.go | 22 ++-- internal/api/handlers/stats_series.go | 2 +- internal/api/handlers/stats_series_test.go | 3 +- internal/api/handlers/traces.go | 2 +- 20 files changed, 220 insertions(+), 40 deletions(-) create mode 100644 internal/api/handlers/lookup_errors_test.go diff --git a/internal/api/handlers/channels.go b/internal/api/handlers/channels.go index 9bf80d59..a43489aa 100644 --- a/internal/api/handlers/channels.go +++ b/internal/api/handlers/channels.go @@ -5,6 +5,7 @@ package handlers import ( "encoding/hex" + "errors" "net/http" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) // ChannelsRouter mounts all /channels routes onto a subrouter. @@ -117,6 +119,7 @@ func listChannels(reader api.Reader) http.HandlerFunc { // @Success 200 {object} api.Channel // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /channels/{channelID} [get] func getChannel(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -130,9 +133,13 @@ func getChannel(reader api.Reader) http.HandlerFunc { id = i } channel, err := reader.GetChannel(r.Context(), int32(id)) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && channel == nil: respondError(w, http.StatusNotFound, "channel not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, channel) } diff --git a/internal/api/handlers/iatas.go b/internal/api/handlers/iatas.go index 1294c029..19d2375b 100644 --- a/internal/api/handlers/iatas.go +++ b/internal/api/handlers/iatas.go @@ -4,11 +4,13 @@ package handlers import ( + "errors" "net/http" "strings" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) // IATAsRouter mounts all /iatas routes onto a subrouter. @@ -45,15 +47,18 @@ func pathIATA(r *http.Request) (string, bool) { // @Tags IATAs // @Produce json // @Success 200 {array} api.IATA -// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /iatas [get] func listIATAs(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { iatas, err := reader.ListIATAs(r.Context()) if err != nil { - respondError(w, http.StatusNotFound, "no IATAs found") + respondError(w, http.StatusInternalServerError, "internal server error") return } + if iatas == nil { + iatas = []api.IATA{} + } respond(w, http.StatusOK, iatas) } } @@ -66,6 +71,7 @@ func listIATAs(reader api.Reader) http.HandlerFunc { // @Param iata path string true "3-letter IATA code" // @Success 200 {object} api.IATA // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /iatas/{iata} [get] func getIATA(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -75,9 +81,13 @@ func getIATA(reader api.Reader) http.HandlerFunc { return } result, err := reader.GetIATA(r.Context(), iata) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && result == nil: respondError(w, http.StatusNotFound, "IATA not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, result) } @@ -92,6 +102,7 @@ func getIATA(reader api.Reader) http.HandlerFunc { // @Success 200 {object} object "GeoJSON Feature (Polygon or MultiPolygon geometry, with bbox)" // @Success 204 "IATA exists but has no border configured" // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /iatas/{iata}/border [get] func getIATABorder(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -101,9 +112,13 @@ func getIATABorder(reader api.Reader) http.HandlerFunc { return } border, err := reader.GetIATABorder(r.Context(), iata) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows): respondError(w, http.StatusNotFound, "IATA not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } // A cache round-trip re-encodes a nil json.RawMessage as the literal 4-byte JSON // "null" rather than leaving it empty, so both must be treated as "no border set". diff --git a/internal/api/handlers/iatas_test.go b/internal/api/handlers/iatas_test.go index b6717710..f6e26dff 100644 --- a/internal/api/handlers/iatas_test.go +++ b/internal/api/handlers/iatas_test.go @@ -6,13 +6,13 @@ package handlers import ( "context" "encoding/json" - "errors" "net/http" "net/http/httptest" "testing" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) func TestListIATAs_OK(t *testing.T) { @@ -148,7 +148,7 @@ func TestGetIATABorder_NotFound(t *testing.T) { r := chi.NewRouter() r.Get("/iatas/{iata}/border", getIATABorder(stubReader{ getIATABorder: func(_ context.Context, _ string) (json.RawMessage, error) { - return nil, errors.New("not found") + return nil, pgx.ErrNoRows }, })) req := httptest.NewRequest(http.MethodGet, "/iatas/ZZZ/border", nil) diff --git a/internal/api/handlers/lookup_errors_test.go b/internal/api/handlers/lookup_errors_test.go new file mode 100644 index 00000000..4c2431d5 --- /dev/null +++ b/internal/api/handlers/lookup_errors_test.go @@ -0,0 +1,116 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +// Lookups answer 404 only for a missing row; any other failure is a retryable 500. +func TestLookupErrors(t *testing.T) { + id := uuid.NewString() + for _, tc := range []struct { + name string + router func(error) http.Handler + path string + missing int // status for pgx.ErrNoRows + }{ + {"region", func(err error) http.Handler { + return RegionsRouter(stubReader{getRegion: func(context.Context, int32) (*api.Region, error) { return nil, err }}) + }, "/1", http.StatusNotFound}, + {"regions", func(err error) http.Handler { + return RegionsRouter(stubReader{listRegions: func(context.Context) ([]api.RegionSummary, error) { return nil, err }}) + }, "/", http.StatusInternalServerError}, + {"iata", func(err error) http.Handler { + return IATAsRouter(stubReader{getIATA: func(context.Context, string) (*api.IATA, error) { return nil, err }}) + }, "/YOW", http.StatusNotFound}, + {"iata border", func(err error) http.Handler { + return IATAsRouter(stubReader{getIATABorder: func(context.Context, string) (json.RawMessage, error) { return nil, err }}) + }, "/YOW/border", http.StatusNotFound}, + {"iatas", func(err error) http.Handler { + return IATAsRouter(stubReader{listIATAs: func(context.Context) ([]api.IATA, error) { return nil, err }}) + }, "/", http.StatusInternalServerError}, + {"channel", func(err error) http.Handler { + return ChannelsRouter(stubReader{getChannel: func(context.Context, int32) (*api.Channel, error) { return nil, err }}) + }, "/7", http.StatusNotFound}, + {"observer", func(err error) http.Handler { + return ObserversRouter(stubReader{getObserver: func(context.Context, uuid.UUID) (*api.Observer, error) { return nil, err }}) + }, "/" + id, http.StatusNotFound}, + {"node", func(err error) http.Handler { + return NodesRouter(stubReader{getNode: func(context.Context, uuid.UUID) (*api.Node, error) { return nil, err }}) + }, "/" + id, http.StatusNotFound}, + {"packet", func(err error) http.Handler { + return PacketsRouter(stubReader{getPacket: func(context.Context, []byte) (*api.Packet, error) { return nil, err }}) + }, "/abcd", http.StatusNotFound}, + } { + for _, c := range []struct { + err error + want int + }{ + {pgx.ErrNoRows, tc.missing}, + {context.DeadlineExceeded, http.StatusInternalServerError}, + } { + t.Run(tc.name+"/"+c.err.Error(), func(t *testing.T) { + w := httptest.NewRecorder() + tc.router(c.err).ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) + if w.Code != c.want { + t.Fatalf("status = %d, want %d", w.Code, c.want) + } + var body map[string]APIError + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || body["error"].Message == "" { + t.Fatalf("not a JSON error: %s", w.Body.String()) + } + }) + } + } +} + +// An unknown region filter is the caller's mistake; a failed lookup is retryable. +func TestRegionFilterErrors(t *testing.T) { + for _, tc := range []struct { + err error + want int + }{ + {pgx.ErrNoRows, http.StatusBadRequest}, + {context.DeadlineExceeded, http.StatusInternalServerError}, + } { + reader := stubReader{getRegionBySlug: func(context.Context, string) (*api.Region, error) { return nil, tc.err }} + for name, h := range map[string]http.Handler{ + "packets": PacketsRouter(reader), + "nodes": NodesRouter(reader), + "traces": TracesRouter(reader), + } { + t.Run(name+"/"+tc.err.Error(), func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/?region=nowhere", nil)) + if w.Code != tc.want { + t.Fatalf("status = %d, want %d", w.Code, tc.want) + } + }) + } + } +} + +func TestEmptyListsAreArrays(t *testing.T) { + for name, h := range map[string]http.Handler{ + "regions": RegionsRouter(stubReader{}), + "iatas": IATAsRouter(stubReader{}), + } { + t.Run(name, func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/", nil)) + if w.Code != http.StatusOK || w.Body.String() != "[]\n" { + t.Fatalf("got %d %q, want 200 []", w.Code, w.Body.String()) + } + }) + } +} diff --git a/internal/api/handlers/messages.go b/internal/api/handlers/messages.go index 7d10fb82..2bd627bf 100644 --- a/internal/api/handlers/messages.go +++ b/internal/api/handlers/messages.go @@ -147,7 +147,7 @@ func listMessagesBackfill(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index d844a307..1d2f09aa 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -5,6 +5,7 @@ package handlers import ( "encoding/hex" + "errors" "net/http" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" "github.com/google/uuid" + "github.com/jackc/pgx/v5" ) // NodesRouter mounts all /nodes routes onto a subrouter. @@ -99,7 +101,7 @@ func listNodes(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -156,6 +158,7 @@ func listNodes(reader api.Reader) http.HandlerFunc { // @Success 200 {object} api.Node // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /nodes/{nodeId} [get] func getNode(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -165,9 +168,13 @@ func getNode(reader api.Reader) http.HandlerFunc { return } node, err := reader.GetNode(r.Context(), nodeID) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && node == nil: respondError(w, http.StatusNotFound, "node not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, node) } diff --git a/internal/api/handlers/observer_comparison.go b/internal/api/handlers/observer_comparison.go index 6988d50a..ff03db42 100644 --- a/internal/api/handlers/observer_comparison.go +++ b/internal/api/handlers/observer_comparison.go @@ -64,7 +64,7 @@ func getObserverComparison(reader api.Reader) http.HandlerFunc { if q.Get("regionId") != "" || q.Get("region") != "" { regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, "region not found") + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 89fddd68..089c6e4b 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -79,7 +79,7 @@ func listObservers(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -102,6 +102,7 @@ func listObservers(reader api.Reader) http.HandlerFunc { // @Success 200 {object} api.Observer // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /observers/{observerId} [get] func getObserver(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -112,9 +113,13 @@ func getObserver(reader api.Reader) http.HandlerFunc { return } obs, err := reader.GetObserver(r.Context(), id) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && obs == nil: respondError(w, http.StatusNotFound, "observer not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, obs) } diff --git a/internal/api/handlers/packets.go b/internal/api/handlers/packets.go index 40ea63c7..82f10262 100644 --- a/internal/api/handlers/packets.go +++ b/internal/api/handlers/packets.go @@ -5,6 +5,7 @@ package handlers import ( "encoding/hex" + "errors" "net/http" "strconv" "strings" @@ -12,6 +13,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) // PacketsRouter mounts all /packets routes onto a subrouter. @@ -102,7 +104,7 @@ func listPackets(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -172,7 +174,7 @@ func listPacketsBackfill(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -207,9 +209,13 @@ func getPacket(reader api.Reader) http.HandlerFunc { return } packet, err := reader.GetPacket(r.Context(), hash) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && packet == nil: respondError(w, http.StatusNotFound, "packet not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, packet) } diff --git a/internal/api/handlers/paths.go b/internal/api/handlers/paths.go index 7dfcc2b0..7f25169a 100644 --- a/internal/api/handlers/paths.go +++ b/internal/api/handlers/paths.go @@ -43,7 +43,7 @@ func getPathStats(reader api.Reader) http.HandlerFunc { if q.Get("regionId") != "" || q.Get("region") != "" { regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, "region not found") + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) diff --git a/internal/api/handlers/paths_test.go b/internal/api/handlers/paths_test.go index fb3b0eeb..b7be4205 100644 --- a/internal/api/handlers/paths_test.go +++ b/internal/api/handlers/paths_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" ) type pathReader struct { @@ -53,7 +54,7 @@ func TestPathsFiltersErrorsAndCancellation(t *testing.T) { return &api.Region{}, nil } if slug == "missing" { - return nil, errors.New("not found") + return nil, pgx.ErrNoRows } return &api.Region{IATAs: []string{"YVR"}}, nil }}, read: func(ctx context.Context, since, until time.Time, iatas []string) (*api.PathStats, error) { diff --git a/internal/api/handlers/regions.go b/internal/api/handlers/regions.go index f8366e82..8437f41b 100644 --- a/internal/api/handlers/regions.go +++ b/internal/api/handlers/regions.go @@ -5,6 +5,7 @@ package handlers import ( "context" + "errors" "fmt" "net/http" "strconv" @@ -13,6 +14,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" ) // RegionsRouter mounts all /regions routes onto a subrouter. @@ -35,15 +37,18 @@ func RegionsRouter(reader api.Reader) http.Handler { // @Tags Regions // @Produce json // @Success 200 {array} api.RegionSummary -// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /regions [get] func listRegions(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { regions, err := reader.ListRegions(r.Context()) if err != nil { - respondError(w, http.StatusNotFound, "no regions found") + respondError(w, http.StatusInternalServerError, "internal server error") return } + if regions == nil { + regions = []api.RegionSummary{} + } respond(w, http.StatusOK, regions) } } @@ -57,6 +62,7 @@ func listRegions(reader api.Reader) http.HandlerFunc { // @Success 200 {object} api.Region // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError // @Router /regions/{regionId} [get] func getRegion(reader api.Reader) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -67,9 +73,13 @@ func getRegion(reader api.Reader) http.HandlerFunc { return } region, err := reader.GetRegion(r.Context(), int32(regionInt)) - if err != nil { + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && region == nil: respondError(w, http.StatusNotFound, "region not found") return + case err != nil: + respondError(w, http.StatusInternalServerError, "internal server error") + return } respond(w, http.StatusOK, region) } @@ -191,3 +201,13 @@ func resolveRegionIATAs(ctx context.Context, regionID, regionSlug string, reader } return region.IATAs, nil } + +// respondRegionError reports an unknown or malformed region filter as 400 and a +// failed lookup as a retryable 500. +func respondRegionError(w http.ResponseWriter, err error) { + if errors.Is(err, pgx.ErrNoRows) || errors.Is(err, strconv.ErrSyntax) || errors.Is(err, strconv.ErrRange) { + respondError(w, http.StatusBadRequest, err.Error()) + return + } + respondError(w, http.StatusInternalServerError, "internal server error") +} diff --git a/internal/api/handlers/scope_stats_test.go b/internal/api/handlers/scope_stats_test.go index 363b1c8b..9d30e3f0 100644 --- a/internal/api/handlers/scope_stats_test.go +++ b/internal/api/handlers/scope_stats_test.go @@ -15,6 +15,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" ) func TestGetStatsScopes_RegionErrors(t *testing.T) { @@ -22,10 +23,10 @@ func TestGetStatsScopes_RegionErrors(t *testing.T) { t.Run(query, func(t *testing.T) { reader := stubReader{ getRegion: func(context.Context, int32) (*api.Region, error) { - return nil, errors.New("not found") + return nil, pgx.ErrNoRows }, getRegionBySlug: func(context.Context, string) (*api.Region, error) { - return nil, errors.New("not found") + return nil, pgx.ErrNoRows }, } w := httptest.NewRecorder() diff --git a/internal/api/handlers/scopes.go b/internal/api/handlers/scopes.go index 9ca791e1..d6293234 100644 --- a/internal/api/handlers/scopes.go +++ b/internal/api/handlers/scopes.go @@ -55,7 +55,7 @@ func listScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc { if regionID != "" || region != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionID, region, reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) diff --git a/internal/api/handlers/signal.go b/internal/api/handlers/signal.go index 34995f00..1a2563dc 100644 --- a/internal/api/handlers/signal.go +++ b/internal/api/handlers/signal.go @@ -43,7 +43,7 @@ func getSignalStats(reader api.Reader) http.HandlerFunc { if q.Get("regionId") != "" || q.Get("region") != "" { regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, "region not found") + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) diff --git a/internal/api/handlers/signal_test.go b/internal/api/handlers/signal_test.go index 7d29d4bc..5cc234bb 100644 --- a/internal/api/handlers/signal_test.go +++ b/internal/api/handlers/signal_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" ) type signalReader struct { @@ -56,7 +57,7 @@ func TestSignalFiltersAndErrors(t *testing.T) { case "empty": return &api.Region{}, nil case "missing": - return nil, errors.New("not found") + return nil, pgx.ErrNoRows default: return &api.Region{IATAs: []string{"YVR"}}, nil } diff --git a/internal/api/handlers/stats.go b/internal/api/handlers/stats.go index 43164d04..426ad836 100644 --- a/internal/api/handlers/stats.go +++ b/internal/api/handlers/stats.go @@ -86,7 +86,7 @@ func getStatsOverview(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -123,7 +123,7 @@ func getStatsObservations(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -164,7 +164,7 @@ func getStatsPayloadBreakdown(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -207,7 +207,7 @@ func getStatsTopNodes(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -254,7 +254,7 @@ func getStatsTopObservers(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -302,7 +302,7 @@ func getStatsTopAdvertisers(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -348,7 +348,7 @@ func getStatsClockDrift(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -390,7 +390,7 @@ func getStatsTopTalkers(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -437,7 +437,7 @@ func getStatsRadioPresets(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -475,7 +475,7 @@ func getStatsScopes(reader api.Reader, scopes ScopeMembership) http.HandlerFunc if regionID := r.URL.Query().Get("regionId"); regionID != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionID, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) @@ -522,7 +522,7 @@ func getStatsNodeTypes(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), regionIDStr, r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) diff --git a/internal/api/handlers/stats_series.go b/internal/api/handlers/stats_series.go index 564a7013..961d58ad 100644 --- a/internal/api/handlers/stats_series.go +++ b/internal/api/handlers/stats_series.go @@ -54,7 +54,7 @@ func getStatsSeries(reader api.Reader, maxWindow time.Duration) http.HandlerFunc if q.Get("regionId") != "" || q.Get("region") != "" { regionIATAs, err := resolveRegionIATAs(ctx, q.Get("regionId"), q.Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, "region not found") + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) diff --git a/internal/api/handlers/stats_series_test.go b/internal/api/handlers/stats_series_test.go index 3ba2624e..68b34f80 100644 --- a/internal/api/handlers/stats_series_test.go +++ b/internal/api/handlers/stats_series_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" ) func TestGetStatsSeries(t *testing.T) { @@ -49,7 +50,7 @@ func TestGetStatsSeries(t *testing.T) { case "empty": return &api.Region{}, nil case "missing": - return nil, errors.New("not found") + return nil, pgx.ErrNoRows } return &api.Region{IATAs: []string{"YVR"}}, nil }, diff --git a/internal/api/handlers/traces.go b/internal/api/handlers/traces.go index 9f131b5e..9ce002db 100644 --- a/internal/api/handlers/traces.go +++ b/internal/api/handlers/traces.go @@ -83,7 +83,7 @@ func listTraceTags(reader api.Reader) http.HandlerFunc { if regionIDStr := r.URL.Query().Get("regionId"); regionIDStr != "" || r.URL.Query().Get("region") != "" { regionIATAs, err := resolveRegionIATAs(r.Context(), r.URL.Query().Get("regionId"), r.URL.Query().Get("region"), reader) if err != nil { - respondError(w, http.StatusBadRequest, err.Error()) + respondRegionError(w, err) return } iatas = append(iatas, regionIATAs...) From 4ad869d711b41ff6bb3ef726098b5c5b329d8531 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 18:59:09 -0400 Subject: [PATCH 137/141] fix(db): page known routes by (last_seen, id) The cursor was the last row's last_seen in ms, so routes sharing that millisecond (common within one upsert batch) never came back. GET /routes takes an optional cursorId, backed by keyset indexes; cursor alone keeps working as before. --- db/migrations/001_baseline.sql | 4 ++ db/queries/queries.sql | 21 ++++-- db/routes.go | 3 +- db/routes_integration_test.go | 2 +- db/routes_pagination_integration_test.go | 78 +++++++++++++++++++++++ db/sqlc/queries.sql.go | 23 +++++-- internal/api/handlers/limits_test.go | 2 +- internal/api/handlers/routes.go | 17 ++++- internal/api/handlers/routes_test.go | 36 ++++++++++- internal/api/handlers/stub_reader_test.go | 6 +- internal/api/reader.go | 3 +- internal/cache/cache_test.go | 2 +- internal/cache/reader.go | 4 +- 13 files changed, 175 insertions(+), 26 deletions(-) create mode 100644 db/routes_pagination_integration_test.go diff --git a/db/migrations/001_baseline.sql b/db/migrations/001_baseline.sql index 71e42f26..4126ad77 100644 --- a/db/migrations/001_baseline.sql +++ b/db/migrations/001_baseline.sql @@ -761,6 +761,10 @@ CREATE INDEX idx_known_routes_hop_count ON known_routes USING btree (iata, hop_c CREATE INDEX idx_known_routes_iata_last_seen ON known_routes USING btree (iata, last_seen DESC); +CREATE INDEX idx_known_routes_iata_keyset ON known_routes USING btree (iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC); + +CREATE INDEX idx_known_routes_keyset ON known_routes USING btree (date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC); + CREATE INDEX idx_known_routes_last_seen ON known_routes USING btree (last_seen DESC); CREATE INDEX idx_known_routes_reconfirm ON known_routes USING btree (last_reconfirmed_at); diff --git a/db/queries/queries.sql b/db/queries/queries.sql index ecdcffe1..079593c7 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1462,13 +1462,18 @@ ON CONFLICT (iata, path_key) DO UPDATE SET -- Only one branch runs. Keep the IATA range ordered by the composite index: -- generic plans can otherwise prefer scanning the global timestamp index. -- The text equality preserves exact input matching, including trailing spaces. -( +-- Pages by (last_seen to the ms, id) so routes sharing the cursor's millisecond +-- aren't skipped; cursor id 0 keeps the plain timestamp cursor. +SELECT r.id, r.node_ids, r.hash_prefix, r.iata, r.hop_count, r.first_seen, r.last_seen, r.observation_count +FROM (( SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes WHERE $1 = '' AND ($2 = 0 OR hop_count = $2) - AND ($3::timestamptz IS NULL OR last_seen < $3) -ORDER BY last_seen DESC + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC LIMIT $4 ) UNION ALL @@ -1478,11 +1483,13 @@ FROM known_routes WHERE $1 <> '' AND iata::text = $1 AND iata >= $1::bpchar AND iata <= $1::bpchar AND ($2 = 0 OR hop_count = $2) - AND ($3::timestamptz IS NULL OR last_seen < $3) -ORDER BY iata, last_seen DESC + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC LIMIT $4 -) -ORDER BY last_seen DESC +)) r +ORDER BY date_trunc('milliseconds', r.last_seen, 'UTC') DESC, r.id DESC LIMIT $4; -- name: SearchKnownRoutes :many diff --git a/db/routes.go b/db/routes.go index 69183fb4..73b17c54 100644 --- a/db/routes.go +++ b/db/routes.go @@ -37,7 +37,7 @@ func (s *Store) UpsertKnownRoute(ctx context.Context, nodeIDs []uuid.UUID, hashP }) } -func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) { +func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]api.KnownRoute, error) { var cursorTS pgtype.Timestamptz if !cursor.IsZero() { cursorTS = pgtype.Timestamptz{Time: cursor, Valid: true} @@ -47,6 +47,7 @@ func (s *Store) ListKnownRoutes(ctx context.Context, iata string, hopCount int32 Column2: hopCount, Column3: cursorTS, Limit: limit, + Column5: cursorID, }) if err != nil { return nil, err diff --git a/db/routes_integration_test.go b/db/routes_integration_test.go index 6d6addf2..7e6594c4 100644 --- a/db/routes_integration_test.go +++ b/db/routes_integration_test.go @@ -111,7 +111,7 @@ ANALYZE known_routes;`) } for _, iata := range []string{"GPT", "YYZ", ""} { var planJSON []byte - if err := tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE route_page('"+iata+"',0,NULL,50)").Scan(&planJSON); err != nil { + if err := tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) EXECUTE route_page('"+iata+"',0,NULL,50,0)").Scan(&planJSON); err != nil { t.Fatal(err) } var plans []struct { diff --git a/db/routes_pagination_integration_test.go b/db/routes_pagination_integration_test.go new file mode 100644 index 00000000..81513292 --- /dev/null +++ b/db/routes_pagination_integration_test.go @@ -0,0 +1,78 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "slices" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" +) + +// Routes upserted in one batch share last_seen; paging must not skip any of them. +func TestKnownRoutePaginationTiesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN for the PostgreSQL regression test") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + applyBaseline(t, ctx, tx) + // Seven routes in one millisecond (two with sub-ms offsets), one older, one in another IATA. + if _, err := tx.Exec(ctx, ` +INSERT INTO iata_codes (iata) VALUES ('YYZ'), ('YOW') ON CONFLICT DO NOTHING; +INSERT INTO known_routes (id,path_key,node_ids,hash_prefix,iata,hop_count,first_seen,last_seen) +SELECT i, decode(md5(i::text),'hex'), ARRAY[]::uuid[], ARRAY['\x01'::bytea], + CASE WHEN i = 9 THEN 'YOW' ELSE 'YYZ' END, 2, + '2026-01-01 00:00:00+00'::timestamptz, + CASE WHEN i = 8 THEN '2026-01-01 00:00:04+00'::timestamptz + WHEN i IN (3, 6) THEN '2026-01-01 00:00:05.000400+00'::timestamptz + ELSE '2026-01-01 00:00:05+00'::timestamptz END +FROM generate_series(1,9) i;`); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + for _, iata := range []string{"", "YYZ"} { + t.Run("iata="+iata, func(t *testing.T) { + var got []int64 + var cursor time.Time + var cursorID int64 + for page := 0; page < 10; page++ { + routes, err := store.ListKnownRoutes(ctx, iata, 0, cursor, cursorID, 3) + if err != nil { + t.Fatal(err) + } + if len(routes) == 0 { + break + } + for _, r := range routes { + got = append(got, r.ID) + } + last := routes[len(routes)-1] + cursor, cursorID = time.UnixMilli(last.LastSeen), last.ID + } + want := []int64{7, 6, 5, 4, 3, 2, 1, 8} + if iata == "" { + want = []int64{9, 7, 6, 5, 4, 3, 2, 1, 8} + } + if !slices.Equal(got, want) { + t.Fatalf("paged ids = %v, want %v", got, want) + } + }) + } +} diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 435bb941..df93bc29 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2968,13 +2968,16 @@ func (q *Queries) ListIATAs(ctx context.Context) ([]IataCode, error) { } const listKnownRoutes = `-- name: ListKnownRoutes :many -( +SELECT r.id, r.node_ids, r.hash_prefix, r.iata, r.hop_count, r.first_seen, r.last_seen, r.observation_count +FROM (( SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes WHERE $1 = '' AND ($2 = 0 OR hop_count = $2) - AND ($3::timestamptz IS NULL OR last_seen < $3) -ORDER BY last_seen DESC + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC LIMIT $4 ) UNION ALL @@ -2984,11 +2987,13 @@ FROM known_routes WHERE $1 <> '' AND iata::text = $1 AND iata >= $1::bpchar AND iata <= $1::bpchar AND ($2 = 0 OR hop_count = $2) - AND ($3::timestamptz IS NULL OR last_seen < $3) -ORDER BY iata, last_seen DESC + AND ($3::timestamptz IS NULL + OR date_trunc('milliseconds', last_seen, 'UTC') < $3 + OR ($5::bigint > 0 AND date_trunc('milliseconds', last_seen, 'UTC') = $3 AND id < $5)) +ORDER BY iata, date_trunc('milliseconds', last_seen, 'UTC') DESC, id DESC LIMIT $4 -) -ORDER BY last_seen DESC +)) r +ORDER BY date_trunc('milliseconds', r.last_seen, 'UTC') DESC, r.id DESC LIMIT $4 ` @@ -2997,6 +3002,7 @@ type ListKnownRoutesParams struct { Column2 interface{} `json:"column_2"` Column3 pgtype.Timestamptz `json:"column_3"` Limit int32 `json:"limit"` + Column5 int64 `json:"column_5"` } type ListKnownRoutesRow struct { @@ -3013,12 +3019,15 @@ type ListKnownRoutesRow struct { // Only one branch runs. Keep the IATA range ordered by the composite index: // generic plans can otherwise prefer scanning the global timestamp index. // The text equality preserves exact input matching, including trailing spaces. +// Pages by (last_seen to the ms, id) so routes sharing the cursor's millisecond +// aren't skipped; cursor id 0 keeps the plain timestamp cursor. func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) { rows, err := q.db.Query(ctx, listKnownRoutes, arg.Column1, arg.Column2, arg.Column3, arg.Limit, + arg.Column5, ) if err != nil { return nil, err diff --git a/internal/api/handlers/limits_test.go b/internal/api/handlers/limits_test.go index b5267694..e904e5c3 100644 --- a/internal/api/handlers/limits_test.go +++ b/internal/api/handlers/limits_test.go @@ -99,7 +99,7 @@ func TestListLimits(t *testing.T) { got = limit return nil, nil }, - listKnownRoutes: func(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) { + listKnownRoutes: func(ctx context.Context, iata string, hopCount int32, cursor time.Time, _ int64, limit int32) ([]api.KnownRoute, error) { calls++ got = limit return nil, nil diff --git a/internal/api/handlers/routes.go b/internal/api/handlers/routes.go index 77f1117e..86a7c861 100644 --- a/internal/api/handlers/routes.go +++ b/internal/api/handlers/routes.go @@ -34,6 +34,7 @@ func RoutesRouter(reader api.Reader) http.Handler { // @Param iata query string false "Filter by IATA code" // @Param hopCount query int false "Filter by exact hop count" // @Param cursor query int false "Epoch ms timestamp of last item for pagination" +// @Param cursorId query int false "id of the last item; with cursor, also returns later routes sharing that millisecond" // @Param limit query int false "Max results (default 50); must be positive, values above 200 are clamped" minimum(1) maximum(200) // @Failure 400 {object} handlers.APIError // @Success 200 {object} []api.KnownRoute @@ -54,12 +55,26 @@ func listKnownRoutes(reader api.Reader) http.HandlerFunc { cursor = time.UnixMilli(ms) } } + // cursorId breaks ties within the cursor's millisecond. + var cursorID int64 + if v := r.URL.Query().Get("cursorId"); v != "" { + id, err := strconv.ParseInt(v, 10, 64) + if err != nil || id <= 0 { + respondError(w, http.StatusBadRequest, "cursorId must be a positive integer") + return + } + if cursor.IsZero() { + respondError(w, http.StatusBadRequest, "cursorId requires cursor") + return + } + cursorID = id + } limit, err := parseLimit(r, 50) if err != nil { respondError(w, http.StatusBadRequest, err.Error()) return } - routes, err := reader.ListKnownRoutes(r.Context(), iata, hopCount, cursor, limit) + routes, err := reader.ListKnownRoutes(r.Context(), iata, hopCount, cursor, cursorID, limit) if err != nil { respondError(w, http.StatusInternalServerError, "internal server error") return diff --git a/internal/api/handlers/routes_test.go b/internal/api/handlers/routes_test.go index 699d5455..9860d6d6 100644 --- a/internal/api/handlers/routes_test.go +++ b/internal/api/handlers/routes_test.go @@ -60,10 +60,44 @@ func TestSearchCrossIATARoutes_MissingParams(t *testing.T) { } } +func TestListKnownRoutes_CursorID(t *testing.T) { + var gotCursor time.Time + var gotID int64 + r := chi.NewRouter() + r.Get("/routes", listKnownRoutes(stubReader{ + listKnownRoutes: func(_ context.Context, _ string, _ int32, cursor time.Time, cursorID int64, _ int32) ([]api.KnownRoute, error) { + gotCursor, gotID = cursor, cursorID + return nil, nil + }, + })) + for _, tc := range []struct { + query string + code int + id int64 + }{ + {"cursor=1700000000000&cursorId=42", http.StatusOK, 42}, + {"cursor=1700000000000", http.StatusOK, 0}, + {"cursor=1700000000000&cursorId=0", http.StatusBadRequest, 0}, + {"cursor=1700000000000&cursorId=x", http.StatusBadRequest, 0}, + {"cursorId=42", http.StatusBadRequest, 0}, + } { + gotID = 0 + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/routes?"+tc.query, nil)) + if w.Code != tc.code { + t.Errorf("%s: code %d, want %d", tc.query, w.Code, tc.code) + continue + } + if tc.code == http.StatusOK && (gotID != tc.id || gotCursor.UnixMilli() != 1700000000000) { + t.Errorf("%s: cursor %v id %d", tc.query, gotCursor, gotID) + } + } +} + func TestListKnownRoutes_OK(t *testing.T) { r := chi.NewRouter() r.Get("/routes", listKnownRoutes(stubReader{ - listKnownRoutes: func(_ context.Context, _ string, _ int32, _ time.Time, _ int32) ([]api.KnownRoute, error) { + listKnownRoutes: func(_ context.Context, _ string, _ int32, _ time.Time, _ int64, _ int32) ([]api.KnownRoute, error) { return []api.KnownRoute{{IATA: "YVR"}}, nil }, })) diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 6ae0ae4e..efb5cb8f 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -59,7 +59,7 @@ type stubReader struct { getScopeByName func(ctx context.Context, name string) (*api.ScopeDetail, error) listTraceTags func(ctx context.Context, iatas []string, scope, traceType string, since, until time.Time, cursor time.Time, cursorTag string, limit int32) ([]api.TraceTagSummary, error) getTraceByTag func(ctx context.Context, tag string) (*api.TraceDetail, error) - listKnownRoutes func(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) + listKnownRoutes func(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]api.KnownRoute, error) searchKnownRoutes func(ctx context.Context, iata, fromHash, toHash string) ([]api.KnownRoute, error) getKnownRoutesByNode func(ctx context.Context, iata string, nodeID uuid.UUID) ([]api.KnownRoute, error) getCrossIATANeighbors func(ctx context.Context, nodeID uuid.UUID, iata string) ([]api.NodeNeighbor, error) @@ -369,9 +369,9 @@ func (s stubReader) GetTraceByTag(ctx context.Context, tag string) (*api.TraceDe return nil, nil } -func (s stubReader) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) { +func (s stubReader) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]api.KnownRoute, error) { if s.listKnownRoutes != nil { - return s.listKnownRoutes(ctx, iata, hopCount, cursor, limit) + return s.listKnownRoutes(ctx, iata, hopCount, cursor, cursorID, limit) } return nil, nil } diff --git a/internal/api/reader.go b/internal/api/reader.go index 36c67e07..dc71e63f 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -225,7 +225,8 @@ type Reader interface { GetTraceByTag(ctx context.Context, tag string) (*TraceDetail, error) // ListKnownRoutes returns known routes filtered by IATA and optional hop count. - ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]KnownRoute, error) + // A nonzero cursorID breaks ties within the cursor's millisecond. + ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]KnownRoute, error) // SearchKnownRoutes returns known routes containing a path from source to destination hash. SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash string) ([]KnownRoute, error) diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 3af439d5..b5dcc7f5 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -208,7 +208,7 @@ func (s *stubReader) ListPacketsAfterID(_ context.Context, _ int64, _, _ int16, return nil, nil } -func (s *stubReader) ListKnownRoutes(_ context.Context, _ string, _ int32, _ time.Time, _ int32) ([]api.KnownRoute, error) { +func (s *stubReader) ListKnownRoutes(_ context.Context, _ string, _ int32, _ time.Time, _ int64, _ int32) ([]api.KnownRoute, error) { return nil, nil } diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 372cae62..e491152d 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -355,8 +355,8 @@ func (cr *CachedReader) ListPacketsAfterID(ctx context.Context, afterObservation } // ListKnownRoutes implements [api.Reader]. -func (cr *CachedReader) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, limit int32) ([]api.KnownRoute, error) { - return cr.inner.ListKnownRoutes(ctx, iata, hopCount, cursor, limit) +func (cr *CachedReader) ListKnownRoutes(ctx context.Context, iata string, hopCount int32, cursor time.Time, cursorID int64, limit int32) ([]api.KnownRoute, error) { + return cr.inner.ListKnownRoutes(ctx, iata, hopCount, cursor, cursorID, limit) } // SearchKnownRoutes implements [api.Reader]. From ff8758b932f0b96ad4c72361f4f0b692c4daccd1 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 18:59:09 -0400 Subject: [PATCH 138/141] fix(db): read observer activity raw rows back to the rollup watermark Hourly activity read raw rows only 3h back, so a longer rollup backlog left hours drawn as zero. The raw tail now starts at the newest complete rollup hour, capped at the 24h raw holdback, and the response reports rolledUntil and rawFrom so clients can show anything still uncovered. --- db/observer_metrics_integration_test.go | 41 +++++++++++++++++++++++++ db/observer_window_test.go | 2 ++ db/observers.go | 27 +++++++++++----- db/observers_test.go | 5 +++ db/queries/queries.sql | 20 +++++------- db/queries/series.sql | 3 ++ db/sqlc/mock/querier.go | 15 +++++++++ db/sqlc/querier.go | 5 ++- db/sqlc/queries.sql.go | 28 +++++++---------- db/sqlc/series.sql.go | 11 +++++++ internal/api/observers.go | 4 +++ 11 files changed, 123 insertions(+), 38 deletions(-) diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 0ca707e3..96ec8a5c 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -67,6 +67,47 @@ func TestObserverActivityUnrolledTailPostgres(t *testing.T) { } } +// A rollup backlog longer than the usual lag must still be read from raw rows. +func TestObserverActivityRollupBacklogPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + applyBaseline(t, ctx, tx) + now := time.Now().UTC() + rolled := now.Truncate(time.Hour).Add(-10 * time.Hour) + if _, err := tx.Exec(ctx, ` + INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); + INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,last_heard_at,first_heard_at) SELECT int4send(i),4,0,1,'\x00','\x00',NOW(),NOW() FROM generate_series(1,3) i;`); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, ` + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type) VALUES + (int4send(1),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz+interval '10 minutes',0,1,0,4);`, rolled); err != nil { + t.Fatal(err) + } + rollTxHours(t, ctx, tx) + if _, err := tx.Exec(ctx, ` + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type) VALUES + (int4send(2),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz-interval '8 hours',0,1,0,4), + (int4send(3),'00000000-0000-0000-0000-000000000001','YOW',$1::timestamptz-interval '6 hours',0,1,0,4);`, now); err != nil { + t.Fatal(err) + } + store := &Store{q: sqlc.New(tx)} + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + activity, err := store.GetObserverActivity(ctx, id, 24*time.Hour, time.Hour, time.Time{}) + if err != nil { + t.Fatal(err) + } + if activity.Summary.RecordedPackets != 3 { + t.Errorf("recorded packets = %d, want 3 (1 rolled + 2 in the backlog)", activity.Summary.RecordedPackets) + } + wantRolled := rolled.Add(time.Hour).UnixMilli() + if activity.RolledUntil == nil || *activity.RolledUntil != wantRolled { + t.Errorf("rolledUntil = %v, want %d", activity.RolledUntil, wantRolled) + } + if activity.RawFrom == nil || *activity.RawFrom != wantRolled { + t.Errorf("rawFrom = %v, want %d (no gap)", activity.RawFrom, wantRolled) + } +} + func TestObserverMetricsPostgres(t *testing.T) { ctx, tx := retentionTx(t) applyBaseline(t, ctx, tx) diff --git a/db/observer_window_test.go b/db/observer_window_test.go index 205d1ef8..03f920ae 100644 --- a/db/observer_window_test.go +++ b/db/observer_window_test.go @@ -11,6 +11,7 @@ import ( sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" "go.uber.org/mock/gomock" ) @@ -26,6 +27,7 @@ func TestObserverActivityLiveAndFixedWindow(t *testing.T) { mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()).Return(nil, nil) mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) } else { + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()).Return(nil, nil) mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) } diff --git a/db/observers.go b/db/observers.go index 888410f9..a458560b 100644 --- a/db/observers.go +++ b/db/observers.go @@ -209,9 +209,9 @@ func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uui return api.BucketTelemetry(raw.Points, since, time.Duration(bucketHours)*time.Hour), nil } -// activityRawTail caps how far back hourly activity reads raw rows for hours not yet rolled -// (normally under rollupDelay + 1h). -const activityRawTail = 3 * time.Hour +// activityRawTail caps how far back hourly activity reads raw rows for hours not yet rolled. +// Cleanup holds raw rows for unrolled hours up to the same 24h (background rawHoldbackCap). +const activityRawTail = 24 * time.Hour // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. // Buckets of an hour or coarser come from the hourly rollup plus raw rows for the unrolled tail; @@ -264,13 +264,26 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w binWidth := pgtype.Interval{Microseconds: interval.Microseconds(), Valid: true} if interval >= time.Hour { - // Unrolled hours read raw rows, bounded to the normal rollup lag. - tailFloor := pgtype.Timestamptz{Time: until.Add(-activityRawTail), Valid: true} + // Unrolled hours read raw rows from the rollup watermark, at most activityRawTail back. + watermark, err := s.q.GetLatestCompleteRollupHour(ctx) + if err != nil { + return nil, err + } + tailStart := until.Add(-activityRawTail) + if watermark.Valid { + rolledUntil := watermark.Time.Add(time.Hour) + activity.RolledUntil = new(rolledUntil.UnixMilli()) + if rolledUntil.After(tailStart) { + tailStart = rolledUntil + } + } + activity.RawFrom = new(tailStart.UnixMilli()) + tailStartTS := pgtype.Timestamptz{Time: tailStart, Valid: true} rows, err := s.q.GetObserverActivityHourly(ctx, sqlc.GetObserverActivityHourlyParams{ ObserverID: observerID, Column2: sinceTS, Column3: binWidth, - TailFloor: tailFloor, + TailStart: tailStartTS, Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { @@ -303,7 +316,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityHourlyPayloadTypes(ctx, sqlc.GetObserverActivityHourlyPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, - TailFloor: tailFloor, + TailStart: tailStartTS, Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { diff --git a/db/observers_test.go b/db/observers_test.go index c60572d5..0943a9a3 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -597,6 +597,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { mock.EXPECT().GetObserverByID(gomock.Any(), observerID). Return(activityObserver(i16(10), i16(5), f32(62.5), f32(910.525)), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyRow{{ Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, @@ -667,6 +668,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { mock.EXPECT().GetObserverByID(gomock.Any(), observerID). Return(activityObserver(i16(7), i16(5), f32(250), nil), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyRow{{ Bucket: pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true}, @@ -837,6 +839,7 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { mock.EXPECT().GetObserverByID(gomock.Any(), observerID). Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyRow{}, nil) mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). @@ -893,6 +896,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { mock.EXPECT().GetObserverByID(gomock.Any(), observerID). Return(activityObserver(tc.sf, tc.cr, tc.bw, nil), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyRow{}, nil) mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). @@ -921,6 +925,7 @@ func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { var gotInterval pgtype.Interval mock.EXPECT().GetObserverByID(gomock.Any(), observerID). Return(activityObserver(i16(10), i16(5), f32(62.5), nil), nil) + mock.EXPECT().GetLatestCompleteRollupHour(gomock.Any()).Return(pgtype.Timestamptz{}, nil) mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()). DoAndReturn(func(_ context.Context, arg sqlc.GetObserverActivityHourlyParams) ([]sqlc.GetObserverActivityHourlyRow, error) { gotSince, gotInterval = arg.Column2.Time, arg.Column3 diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 079593c7..2a2c470b 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -266,21 +266,18 @@ ORDER BY count DESC; -- name: GetObserverActivityHourly :many -- Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. --- Hours after the newest complete one aren't rolled yet; that tail, from no earlier than @tail_floor, reads raw rows. -WITH tail AS ( - SELECT GREATEST(max(hour) + INTERVAL '1 hour', @tail_floor::timestamptz) AS tail_start - FROM analytics_rollup_hours WHERE status = 'complete' -), src AS ( +-- Hours from @tail_start on aren't rolled yet and read raw rows. +WITH src AS ( SELECT a.hour AS t, a.observations, a.airtime_ms, a.airtime_n, a.snr_sum, a.snr_n, a.snr_min, a.rssi_sum, a.rssi_n FROM analytics_hourly_observer_activity a - WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST(@until::timestamptz, (SELECT tail_start FROM tail)) + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST(@until::timestamptz, @tail_start::timestamptz) UNION ALL SELECT o.heard_at, 1::bigint, o.airtime_ms, (o.airtime_ms IS NOT NULL)::int::bigint, s.snr, (s.snr IS NOT NULL)::int::bigint, s.snr, s.rssi::bigint, (s.rssi IS NOT NULL)::int::bigint FROM packet_observations o CROSS JOIN LATERAL (SELECT CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.snr END AS snr, CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.rssi END AS rssi) s - WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, (SELECT tail_start FROM tail)) AND o.heard_at < @until::timestamptz + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, @tail_start::timestamptz) AND o.heard_at < @until::timestamptz ) SELECT date_bin($3::interval, t, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, @@ -298,17 +295,14 @@ ORDER BY 1; -- name: GetObserverActivityHourlyPayloadTypes :many -- Same rollup/raw-tail split as GetObserverActivityHourly. -WITH tail AS ( - SELECT GREATEST(max(hour) + INTERVAL '1 hour', @tail_floor::timestamptz) AS tail_start - FROM analytics_rollup_hours WHERE status = 'complete' -), src AS ( +WITH src AS ( SELECT a.payload_type, a.observations AS n FROM analytics_hourly_observer_activity a - WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST(@until::timestamptz, (SELECT tail_start FROM tail)) + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST(@until::timestamptz, @tail_start::timestamptz) UNION ALL SELECT COALESCE(o.payload_type, -1)::smallint, 1::bigint FROM packet_observations o - WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, (SELECT tail_start FROM tail)) AND o.heard_at < @until::timestamptz + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, @tail_start::timestamptz) AND o.heard_at < @until::timestamptz ) SELECT payload_type, SUM(n)::bigint AS count FROM src diff --git a/db/queries/series.sql b/db/queries/series.sql index a22ca8e1..154d8db1 100644 --- a/db/queries/series.sql +++ b/db/queries/series.sql @@ -89,3 +89,6 @@ ORDER BY k.hour NULLS LAST; -- name: GetEarliestCompleteRollupHour :one SELECT min(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete'; + +-- name: GetLatestCompleteRollupHour :one +SELECT max(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete'; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 71191e67..b34664f2 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -522,6 +522,21 @@ func (mr *MockQuerierMockRecorder) GetKnownRoutesByNode(ctx, arg any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetKnownRoutesByNode", reflect.TypeOf((*MockQuerier)(nil).GetKnownRoutesByNode), ctx, arg) } +// GetLatestCompleteRollupHour mocks base method. +func (m *MockQuerier) GetLatestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLatestCompleteRollupHour", ctx) + ret0, _ := ret[0].(pgtype.Timestamptz) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLatestCompleteRollupHour indicates an expected call of GetLatestCompleteRollupHour. +func (mr *MockQuerierMockRecorder) GetLatestCompleteRollupHour(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLatestCompleteRollupHour", reflect.TypeOf((*MockQuerier)(nil).GetLatestCompleteRollupHour), ctx) +} + // GetNodeByID mocks base method. func (m *MockQuerier) GetNodeByID(ctx context.Context, id uuid.UUID) (db.GetNodeByIDRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 4f56b0f7..f3c54d3c 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -77,6 +77,7 @@ type Querier interface { // not-found distinction GetIATA already makes. GetIATABorder(ctx context.Context, iata string) (json.RawMessage, error) GetKnownRoutesByNode(ctx context.Context, arg GetKnownRoutesByNodeParams) ([]GetKnownRoutesByNodeRow, error) + GetLatestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) GetNodeByID(ctx context.Context, id uuid.UUID) (GetNodeByIDRow, error) GetNodeByPubkey(ctx context.Context, publicKey []byte) (uuid.UUID, error) // Returns the neighbors of a node with details, ordered by most recently seen. @@ -84,7 +85,7 @@ type Querier interface { GetNodesByIDs(ctx context.Context, dollar_1 []uuid.UUID) ([]GetNodesByIDsRow, error) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]GetNodesByPubkeysRow, error) // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. - // Hours after the newest complete one aren't rolled yet; that tail, from no earlier than @tail_floor, reads raw rows. + // Hours from @tail_start on aren't rolled yet and read raw rows. GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) // Same rollup/raw-tail split as GetObserverActivityHourly. GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) @@ -205,6 +206,8 @@ type Querier interface { // Only one branch runs. Keep the IATA range ordered by the composite index: // generic plans can otherwise prefer scanning the global timestamp index. // The text equality preserves exact input matching, including trailing spaces. + // Pages by (last_seen to the ms, id) so routes sharing the cursor's millisecond + // aren't skipped; cursor id 0 keeps the plain timestamp cursor. ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams) ([]ListKnownRoutesRow, error) // Returns messages after the given message ID, ordered oldest first. // Used for WS reconnect backfill. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index df93bc29..46b13b50 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -813,20 +813,17 @@ func (q *Queries) GetNodesByPubkeys(ctx context.Context, pubkeys [][]byte) ([]Ge } const getObserverActivityHourly = `-- name: GetObserverActivityHourly :many -WITH tail AS ( - SELECT GREATEST(max(hour) + INTERVAL '1 hour', $4::timestamptz) AS tail_start - FROM analytics_rollup_hours WHERE status = 'complete' -), src AS ( +WITH src AS ( SELECT a.hour AS t, a.observations, a.airtime_ms, a.airtime_n, a.snr_sum, a.snr_n, a.snr_min, a.rssi_sum, a.rssi_n FROM analytics_hourly_observer_activity a - WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST($5::timestamptz, (SELECT tail_start FROM tail)) + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST($4::timestamptz, $5::timestamptz) UNION ALL SELECT o.heard_at, 1::bigint, o.airtime_ms, (o.airtime_ms IS NOT NULL)::int::bigint, s.snr, (s.snr IS NOT NULL)::int::bigint, s.snr, s.rssi::bigint, (s.rssi IS NOT NULL)::int::bigint FROM packet_observations o CROSS JOIN LATERAL (SELECT CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.snr END AS snr, CASE WHEN NOT (COALESCE(o.rssi, 0) = 0 AND COALESCE(o.snr, 0) = 0) THEN o.rssi END AS rssi) s - WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, (SELECT tail_start FROM tail)) AND o.heard_at < $5::timestamptz + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, $5::timestamptz) AND o.heard_at < $4::timestamptz ) SELECT date_bin($3::interval, t, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, @@ -847,8 +844,8 @@ type GetObserverActivityHourlyParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` - TailFloor pgtype.Timestamptz `json:"tail_floor"` Until pgtype.Timestamptz `json:"until"` + TailStart pgtype.Timestamptz `json:"tail_start"` } type GetObserverActivityHourlyRow struct { @@ -864,14 +861,14 @@ type GetObserverActivityHourlyRow struct { } // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. -// Hours after the newest complete one aren't rolled yet; that tail, from no earlier than @tail_floor, reads raw rows. +// Hours from @tail_start on aren't rolled yet and read raw rows. func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) { rows, err := q.db.Query(ctx, getObserverActivityHourly, arg.ObserverID, arg.Column2, arg.Column3, - arg.TailFloor, arg.Until, + arg.TailStart, ) if err != nil { return nil, err @@ -902,17 +899,14 @@ func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserver } const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many -WITH tail AS ( - SELECT GREATEST(max(hour) + INTERVAL '1 hour', $3::timestamptz) AS tail_start - FROM analytics_rollup_hours WHERE status = 'complete' -), src AS ( +WITH src AS ( SELECT a.payload_type, a.observations AS n FROM analytics_hourly_observer_activity a - WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST($4::timestamptz, (SELECT tail_start FROM tail)) + WHERE a.observer_id = $1 AND a.hour >= $2::timestamptz AND a.hour < LEAST($3::timestamptz, $4::timestamptz) UNION ALL SELECT COALESCE(o.payload_type, -1)::smallint, 1::bigint FROM packet_observations o - WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, (SELECT tail_start FROM tail)) AND o.heard_at < $4::timestamptz + WHERE o.observer_id = $1 AND o.heard_at >= GREATEST($2::timestamptz, $4::timestamptz) AND o.heard_at < $3::timestamptz ) SELECT payload_type, SUM(n)::bigint AS count FROM src @@ -923,8 +917,8 @@ ORDER BY count DESC type GetObserverActivityHourlyPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` - TailFloor pgtype.Timestamptz `json:"tail_floor"` Until pgtype.Timestamptz `json:"until"` + TailStart pgtype.Timestamptz `json:"tail_start"` } type GetObserverActivityHourlyPayloadTypesRow struct { @@ -937,8 +931,8 @@ func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2, - arg.TailFloor, arg.Until, + arg.TailStart, ) if err != nil { return nil, err diff --git a/db/sqlc/series.sql.go b/db/sqlc/series.sql.go index cf4e6905..8bd7875c 100644 --- a/db/sqlc/series.sql.go +++ b/db/sqlc/series.sql.go @@ -22,6 +22,17 @@ func (q *Queries) GetEarliestCompleteRollupHour(ctx context.Context) (pgtype.Tim return column_1, err } +const getLatestCompleteRollupHour = `-- name: GetLatestCompleteRollupHour :one +SELECT max(hour)::timestamptz FROM analytics_rollup_hours WHERE status = 'complete' +` + +func (q *Queries) GetLatestCompleteRollupHour(ctx context.Context) (pgtype.Timestamptz, error) { + row := q.db.QueryRow(ctx, getLatestCompleteRollupHour) + var column_1 pgtype.Timestamptz + err := row.Scan(&column_1) + return column_1, err +} + const getStatsSeries = `-- name: GetStatsSeries :many WITH hours AS ( SELECT g.h::timestamptz AS hour, COALESCE(r.status, 'missing')::text AS status diff --git a/internal/api/observers.go b/internal/api/observers.go index 080b74d7..995ef04d 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -110,6 +110,10 @@ type ObserverActivity struct { GeneratedAt int64 `json:"generatedAt"` // response computation time, not proof of continuous coverage Source string `json:"source"` // raw or hourly; missing records do not prove an outage Summary *ObserverActivitySummary `json:"summary,omitempty"` + // Hourly only. Buckets before rolledUntil come from rollups and from rawFrom on from raw rows; + // when rawFrom is later, the hours between are uncovered, not quiet. + RolledUntil *int64 `json:"rolledUntil,omitempty"` // end of the newest complete rollup hour, epoch ms + RawFrom *int64 `json:"rawFrom,omitempty"` // start of the raw tail, epoch ms Range string `json:"range"` Interval string `json:"interval"` From af20bebfe580acff719aa6c63dd8d0d004b67fb4 Mon Sep 17 00:00:00 2001 From: MrAlders0n Date: Fri, 2 Oct 2026 18:59:09 -0400 Subject: [PATCH 139/141] docs: document route cursorId and activity coverage fields --- README.md | 6 +++-- docs/docs.go | 58 +++++++++++++++++++++++++++++++++++++--- docs/historical-stats.md | 3 ++- docs/swagger.json | 58 +++++++++++++++++++++++++++++++++++++--- docs/swagger.yaml | 45 ++++++++++++++++++++++++++++--- 5 files changed, 155 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 80c00d61..084cc3d5 100644 --- a/README.md +++ b/README.md @@ -622,7 +622,7 @@ List `limit` values are clamped to 1–200. | `GET` | `/packets/{packetHash}` | Get packet with all observations | | `GET` | `/regions` | List all regions (summary) | | `GET` | `/regions/{regionId}` | Get a single region with IATA list | -| `GET` | `/routes` | List known routes (`iata`, `hopCount`) | +| `GET` | `/routes` | List known routes (`iata`, `hopCount`, `cursor`+`cursorId`) | | `GET` | `/routes/search` | Search routes by source and destination hash | | `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | | `GET` | `/routes/{iata}/{pathKey}/observations` | Retained observations matching a saved route (see below) | @@ -696,7 +696,9 @@ The existing observer `observationCount` remains a legacy cumulative presence counter for compatibility, including status/neighbour events. It is not a period packet total. Broker presence and packet-arrival timestamps are now updated separately; this cannot reconstruct previously overwritten timestamps. -Hourly activity comes from the analytics rollups and includes unknown-type activity. +Hourly activity comes from the analytics rollups, plus raw rows for hours not yet rolled +(at most 24h back), and includes unknown-type activity. `rolledUntil` is the end of the newest +complete rollup hour and `rawFrom` the start of the raw tail; hours between them are uncovered. ### Saved-route evidence diff --git a/docs/docs.go b/docs/docs.go index 466c4885..1d20bed3 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -692,6 +692,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -799,8 +805,8 @@ const docTemplate = `{ } } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -838,6 +844,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -875,6 +887,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -1204,6 +1222,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -1442,6 +1466,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -1905,8 +1935,8 @@ const docTemplate = `{ } } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -1950,6 +1980,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -1982,6 +2018,12 @@ const docTemplate = `{ "name": "cursor", "in": "query" }, + { + "type": "integer", + "description": "id of the last item; with cursor, also returns later routes sharing that millisecond", + "name": "cursorId", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -4221,6 +4263,14 @@ const docTemplate = `{ "range": { "type": "string" }, + "rawFrom": { + "description": "start of the raw tail, epoch ms", + "type": "integer" + }, + "rolledUntil": { + "description": "Hourly only. Buckets before rolledUntil come from rollups and from rawFrom on from raw rows;\nwhen rawFrom is later, the hours between are uncovered, not quiet.", + "type": "integer" + }, "source": { "description": "raw or hourly; missing records do not prove an outage", "type": "string" diff --git a/docs/historical-stats.md b/docs/historical-stats.md index f9e03176..c039ad5a 100644 --- a/docs/historical-stats.md +++ b/docs/historical-stats.md @@ -69,7 +69,8 @@ region or all IATAs. - Packet, trace and route detail, the packet list, observer comparison, and observer activity below one hour. Hourly observer activity reads raw rows for the hours after the - newest complete one (at most the last 3 hours), so the unrolled tail isn't shown as zero. + newest complete one (at most the last 24 hours, the raw holdback), so the unrolled tail isn't + shown as zero. `rolledUntil` and `rawFrom` in the response mark any hours still uncovered. - Current memberships: scope observer and node counts, node types, clock drift, radio presets. - `/traces` reads `trace_tags`, a per-tag summary kept at ingest, not a rollup. A new packet adds its count, type and payload in the statement that stores it; hearings set the times. diff --git a/docs/swagger.json b/docs/swagger.json index 63ce584f..809064f6 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -690,6 +690,12 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -797,8 +803,8 @@ } } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -836,6 +842,12 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -873,6 +885,12 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -1202,6 +1220,12 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -1440,6 +1464,12 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -1903,8 +1933,8 @@ } } }, - "404": { - "description": "Not Found", + "500": { + "description": "Internal Server Error", "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } @@ -1948,6 +1978,12 @@ "schema": { "$ref": "#/definitions/internal_api_handlers.APIError" } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } } } } @@ -1980,6 +2016,12 @@ "name": "cursor", "in": "query" }, + { + "type": "integer", + "description": "id of the last item; with cursor, also returns later routes sharing that millisecond", + "name": "cursorId", + "in": "query" + }, { "maximum": 200, "minimum": 1, @@ -4219,6 +4261,14 @@ "range": { "type": "string" }, + "rawFrom": { + "description": "start of the raw tail, epoch ms", + "type": "integer" + }, + "rolledUntil": { + "description": "Hourly only. Buckets before rolledUntil come from rollups and from rawFrom on from raw rows;\nwhen rawFrom is later, the hours between are uncovered, not quiet.", + "type": "integer" + }, "source": { "description": "raw or hourly; missing records do not prove an outage", "type": "string" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 132e0f88..3e872fd8 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -590,6 +590,14 @@ definitions: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio' range: type: string + rawFrom: + description: start of the raw tail, epoch ms + type: integer + rolledUntil: + description: |- + Hourly only. Buckets before rolledUntil come from rollups and from rawFrom on from raw rows; + when rawFrom is later, the hours between are uncovered, not quiet. + type: integer source: description: raw or hourly; missing records do not prove an outage type: string @@ -2083,6 +2091,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get channel detail tags: - Channels @@ -2155,8 +2167,8 @@ paths: items: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.IATA' type: array - "404": - description: Not Found + "500": + description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' summary: List all IATA codes @@ -2181,6 +2193,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get a single IATA code tags: - IATAs @@ -2205,6 +2221,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get an IATA's GeoJSON border, if configured tags: - IATAs @@ -2428,6 +2448,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get node detail tags: - Nodes @@ -2587,6 +2611,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get observer detail tags: - Observers @@ -2904,8 +2932,8 @@ paths: items: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RegionSummary' type: array - "404": - description: Not Found + "500": + description: Internal Server Error schema: $ref: '#/definitions/internal_api_handlers.APIError' summary: List all regions @@ -2934,6 +2962,10 @@ paths: description: Not Found schema: $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' summary: Get a single region tags: - Regions @@ -2952,6 +2984,11 @@ paths: in: query name: cursor type: integer + - description: id of the last item; with cursor, also returns later routes sharing + that millisecond + in: query + name: cursorId + type: integer - description: Max results (default 50); must be positive, values above 200 are clamped in: query From be35b36679502551cae930850452ba95a1cff7d1 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:42:17 -0400 Subject: [PATCH 140/141] docs: condense backup, README and contributor docs (#205) --- CONTRIBUTING.md | 39 ++--- PROFILING.md | 95 +++++------- README.md | 315 +++++++++++++++++---------------------- docs/backup-export.md | 284 +++++++++++++++-------------------- docs/packet-summaries.md | 32 ++-- 5 files changed, 333 insertions(+), 432 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 04209e74..382b7352 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,8 +55,6 @@ swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency ## Code style -- Run `gofmt -w .` before committing — CI will fail on unformatted files -- Run `go vet ./...` — no warnings - Follow the existing patterns in each package before introducing new ones - Keep functions small and single-purpose - Prefer explicit error handling over panic @@ -69,9 +67,10 @@ swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency no side effects) should have unit tests. See `internal/hub/hub_test.go`, `internal/api/nodes_test.go`, and `internal/keystore/keystore_test.go` for examples of the style we use. -- Integration tests (requiring a real DB) are not yet required but are welcome. - They will be gated before release. -- Run `go test ./...` before opening a PR. All tests must pass. +- Integration tests (`*_integration_test.go`) need a real PostgreSQL 16 and skip + unless `BEACON_TEST_POSTGRES_DSN` is set. CI runs a selected set; run the ones + touching your change locally: + `BEACON_TEST_POSTGRES_DSN=postgres://... go test ./db/ -run Postgres` - If you are fixing a bug, add a test that would have caught it. --- @@ -118,9 +117,8 @@ that boundary. `/config` remains owned by the shared admin router. No global registration or `init` side effects are needed. When concurrent PRs touch the same startup wiring, stack them in an explicit -merge order and link each feature-only comparison. Rebase the stack together -after prerequisite merges, regenerate Swagger rather than hand-merging it, and -compile the combined result even when GitHub reports it as mergeable. +merge order, regenerate Swagger rather than hand-merging it, and build the +combined result even when GitHub says it merges cleanly. Any new or modified REST endpoint must have swagger annotations and regenerated docs: @@ -129,12 +127,8 @@ docs: `// @Router` comments on the handler function - Response types are defined in `internal/api/` — add new types there, not inline in handlers -- After changing any handler or API type, regenerate the swagger docs and commit - the updated `docs/` directory alongside your changes: - -```bash -swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency -``` +- After changing any handler or API type, rerun the `swag init` command from the + checklist and commit the updated `docs/` directory Install swag if you don't have it: @@ -242,6 +236,7 @@ Scopes are optional but helpful for larger codebases. Common scopes: `api`, ``` cmd/beacon/ — main entry point, wiring, startup +cmd/beacon-backup/ — standalone backup export/verify command db/ — store layer: sqlc-generated code + thin mapping layer migrations/ — SQL schema (2.0.0 baseline + numbered files, append only) queries/ — SQL queries (input to sqlc) @@ -249,15 +244,25 @@ db/ — store layer: sqlc-generated code + thin mapping layer internal/ api/ — response types and Reader interface handlers/ — HTTP handlers (validation, routing, response) + middleware/ — auth, rate limiting, logging router/ — chi router wiring + background/ — periodic maintenance (retention, rollups, age-out) + backup/ — database + config bundle export and verification + borders/ — foreign-repeater classification against IATA borders + cache/ — Redis caching layer over the Reader config/ — config loading and scope key derivation hub/ — WebSocket fan-out broker - ingest/ — MQTT packet ingestion and side effects iatadb/ — in-memory IATA airport lookup + ingest/ — MQTT packet ingestion and side effects keystore/ — channel key lookup + logging/ — slog setup + lora/ — LoRa airtime calculation + meshmapper/ — MeshMapper scope and zone imports + presence/ — coalesced last-seen writes + profiling/ — opt-in CPU profiling scopestore/ — transport scope key lookup ws/ — WebSocket connection handling -docs/ — generated swagger docs (do not edit by hand) +docs/ — generated swagger and design notes (swagger: do not edit by hand) ``` Key patterns to understand before contributing: @@ -282,7 +287,7 @@ release. Do not open PRs directly against `main`. ### Cutting a release -> NOTE: release are done manually, not from GH. Commits for releases should be +> NOTE: releases are done manually, not from GH. Commits for releases should be > signed. 1. Ensure all changes are committed and CI is green on `dev` diff --git a/PROFILING.md b/PROFILING.md index a9989b4a..6fbe7ae3 100644 --- a/PROFILING.md +++ b/PROFILING.md @@ -1,66 +1,55 @@ # Production CPU profiling -CPU profiling is supported on Linux and macOS and is disabled by default. It writes private files inside the container; -there is no HTTP endpoint or published profiling port. Profiling failures stop the -recorder and log a warning without stopping ingest. +Beacon can record bounded CPU profiles on Linux and macOS. It's off by default, writes +private files only (there's no HTTP endpoint or port), and a profiling failure stops the +recorder with a warning without affecting ingest. -## Enable a bounded session +## Enable a session -Create a dedicated directory writable by the container user with mode `0700`. -Mount it into the app container and set both environment variables: +Create a dedicated `0700` directory writable by the container user, mount it, and set both +variables: ```yaml services: app: environment: BEACON_CPU_PROFILE_DIR: /profiles - BEACON_CPU_PROFILE_UNTIL: "2026-10-02T12:00:00Z" # Replace with your fixed deadline. + BEACON_CPU_PROFILE_UNTIL: "2026-10-02T12:00:00Z" # fixed deadline volumes: - ./profiles:/profiles ``` -Choose an RFC3339 deadline no more than 72 hours in the future. The same deadline -remains in effect after restarts; expired settings are inactive. Do not generate a -fresh deadline automatically on each startup. Restarting the app is required to -change these settings. Preserve the rest of the deployment configuration. +- The deadline is RFC 3339, at most 72 hours ahead. It survives restarts and is inactive + once past. Don't generate a fresh deadline on every start. +- Changing either setting needs a restart. +- Start with a deadline about a minute out, and compare process CPU, packet freshness, queue + drops and database timeouts with an unprofiled period before running a longer session. +- One Beacon process per directory. Keep it out of web roots, public backups and source + control, and don't run another CPU profiler in the process meanwhile. -Start with a deadline about one minute away to assess the first capture's overhead. -Check process CPU, packet freshness, queue drops and database timeouts against a -comparable period with profiling disabled. After reviewing that capture, an operator -can enable a longer session. Profiling adds overhead while a capture is active. +## What gets captured -## Capture behavior +- A 30-second sample at start, another 35 minutes later, then every 30 minutes (the extra + five minutes keeps a maintenance trigger from landing in the cooldown). +- An extra sample when route reconfirmation maintenance starts (including its retention + step). +- A five-minute cooldown between capture starts. A periodic sample due during it runs when it + ends; a triggered one is skipped, and any sample cancels a periodic one that's waiting. +- Background task stacks carry a `task` label. +- Shutdown or the deadline ends the active sample early and saves it. -- One 30-second sample immediately, then every 30 minutes, offset five minutes past - the half hour so a maintenance trigger due on the hour is not lost to the cooldown. -- Route reconfirmation requests an additional sample when the maintenance task starts. - This includes the retention step before route validation. A five-minute cooldown - between capture starts prevents overlap and repeated triggers from increasing load; - a periodic sample due during the cooldown runs when it ends. A triggered sample - during the cooldown is skipped, and any sample cancels a periodic one waiting on - the cooldown. -- Background task stacks carry a `task` label while profiling is enabled. -- Shutdown or expiry stops the active sample and saves the shorter profile. -- Each profile is limited to 8 MiB. The dedicated directory is limited to 256 MiB - and 512 files, including metadata and files left by interrupted runs. The recorder - reserves space for a full capture before starting and stops when a limit is reached. - Files are never automatically deleted. Existing unrelated regular files consume the budget; subdirectories are ignored. -- Profiles and metadata are written with mode `0600`. A `.partial` file indicates - an interrupted capture and is not a completed profile. +Limits: 8 MiB per profile; 256 MiB and 512 files per directory, counting metadata, leftovers +and unrelated regular files (subdirectories are ignored). The recorder reserves room for a +full capture first and stops at a limit. Nothing is deleted automatically. Files are `0600`; +a `.partial` file is an interrupted capture. -Only one Beacon process should write to a profiling directory. Keep the mount -private and out of web roots, backups intended for public download, and source control. -Do not run another CPU profiler in the same process during a session. +## Reading the output -## Interpret the output +Each `.pprof` has a `.pprof.json` sidecar with UTC start and end, the trigger, Go version, +process CPU counters, goroutine count, and database-pool counters before and after (pool +values are cumulative, so subtract). No SQL text or credentials. -Each `.pprof` has a `.pprof.json` sidecar containing UTC start/end times, the trigger, -Go version, process CPU counters (Linux/macOS), goroutine count, and database-pool -counters before and after the capture. Pool acquire duration and counts are cumulative; -use differences between the two snapshots. They do not contain SQL text or credentials. - -Copy completed files privately off the host. Record the container's image revision -and digest alongside them. On a workstation with Go installed: +Copy completed files off the host privately, noting the image revision and digest. Then: ```sh go tool pprof -top capture.pprof @@ -68,16 +57,14 @@ go tool pprof -top -cum capture.pprof go tool pprof -tags capture.pprof ``` -CPU samples show work inside Beacon, not CPU used by PostgreSQL or time waiting on -locks. Correlate each UTC interval with separately collected host/container CPU and -I/O, PostgreSQL wait states, observation throughput, and Beacon timeout/drop logs. -Do not enable SQL text logging or run full-table counts for this purpose. The recorder -makes no diagnostic database queries and does not change the ingest path. +Profiles show CPU inside Beacon, not PostgreSQL work or lock waits. Correlate each interval +with host/container CPU and I/O, PostgreSQL wait states, observation throughput and Beacon's +timeout/drop logs. Don't turn on SQL text logging or run full-table counts for this; the +recorder itself makes no database queries. -Review captures from quiet periods, bursts, and maintenance separately before -combining them. Additional maintenance samples deliberately bias an aggregate profile. -Thirty-second windows can miss brief stalls; the absence of a stack is not proof that -it never consumes CPU. +Review quiet, burst and maintenance captures separately: maintenance samples bias an +aggregate. Thirty-second windows can miss brief stalls, so a missing stack isn't proof it +never runs. -At the deadline, check for `CPU profiling stopped` in the app log. Export the files, -then remove the environment variables and mount during the next planned deployment. +When the deadline passes, look for `CPU profiling stopped` in the log, export the files, and +remove the variables and mount at the next deployment. diff --git a/README.md b/README.md index 084cc3d5..c83ab37f 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,14 @@ in PostgreSQL, and streams live events to WebSocket clients. - Seeds regions, IATA display names, and channel keys from a YAML config file on startup -For deployment instructions including the frontend app and reverse proxy -examples, see [beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs). +More documentation: -For a bounded private database and saved-config bundle, see -[backup export](docs/backup-export.md). A standalone export tool is available; the -protected download API is opt-in; browser login and import remain separate follow-ups. +- [beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs): deployment with the + frontend and reverse proxy examples +- [Backup and export](docs/backup-export.md): private database and config bundles +- [Historical stats](docs/historical-stats.md): how the hourly rollups work +- [Packet summaries](docs/packet-summaries.md): the `summary` field on packet lists +- [CPU profiling](PROFILING.md): bounded production captures --- @@ -82,23 +84,6 @@ docker run -d --name beacon-postgres -p 5432:5432 \ Database migrations are applied automatically on startup. -### Upgrading from 1.x - -2.0.0 starts from a new schema baseline and **needs a fresh database**. Pointed -at a 1.x database it refuses to start with `database schema predates Beacon -2.0.0; 2.0.0 needs a fresh database`. 1.x history does not carry over: create a -new, empty database (Docker: stop the stack and move or remove the Postgres -data directory) before starting 2.0.0. - -Config changes to review when upgrading: - -- Every manually configured `scopes:` entry needs a `region`. -- `meshmapper.scopes.sources` is ignored; MeshMapper refresh intervals must be - within 1h–24h for scopes and 24h–168h for zones, or startup fails. -- REST rate limiting is on by default. Behind a reverse proxy, set - `server.trusted_proxies` (see [Reverse proxies and rate limits](#reverse-proxies-and-rate-limits)). -- `packets.retention` now defaults to 7 days. - ### 3. Run ```bash @@ -122,82 +107,37 @@ Beacon will: - Connect to the configured MQTT brokers - Start the HTTP server on `LISTEN_ADDR` (default `:8080`) -### Cold start and path resolution +### Upgrading from 1.x + +2.0.0 starts from a new schema baseline and **needs a fresh, empty database**; 1.x history +does not carry over. Pointed at a 1.x database it refuses to start with `database schema +predates Beacon 2.0.0; 2.0.0 needs a fresh database` (Docker: stop the stack and move or +remove the Postgres data directory). + +Config changes to review: + +- Every manually configured `scopes:` entry needs a `region`. +- `meshmapper.scopes.sources` is ignored; MeshMapper refresh intervals must be within + 1h–24h for scopes and 24h–168h for zones, or startup fails. +- REST rate limiting is on by default. Behind a reverse proxy, set + `server.trusted_proxies` (see [Reverse proxies and rate limits](#reverse-proxies-and-rate-limits)). +- `packets.retention` now defaults to 7 days. + +### Cold start -Path resolution, firmware capability detection, and known route storage all -depend on nodes having advertised at least once to a local observer. On a fresh -deployment `resolvedPath` will show `"confidence": "none"` for all hops and -`supportsMultibytePaths` will be `false` for all nodes until advert traffic -arrives and populates `node_short_ids`. This is expected behaviour — resolution -improves automatically as the mesh is observed over time. +Path resolution, capability detection and known routes depend on nodes having advertised +to a local observer. On a fresh database every hop shows `"confidence": "none"` and +`supportsMultibytePaths` is `false` until adverts populate `node_short_ids`. That's +expected and fills in as the mesh is observed. -Similarly, GRP_TXT packets whose channel key isn't yet known at ingest time are -stored as hash-only, undecrypted rows. Adding the channel's key to `config.yaml` -doesn't retroactively decrypt that history immediately — it's picked up -automatically on the next restart, when Beacon scans for undecrypted packets -matching a now-known channel and decrypts them. Watch the startup log for -`backfilled N previously-undecrypted channel message(s)`. +GRP_TXT packets whose channel key isn't known yet are stored hash-only. After adding the +key to `config.yaml`, the next restart decrypts the matching history; look for +`backfilled N previously-undecrypted channel message(s)` in the log. --- ## Configuration -### Admin authentication - -The `/api/v1/admin` subtree requires `Authorization: Bearer `. Set the -operator key with `BEACON_API_KEY` or `auth.api_key` in YAML. A set environment -variable overrides YAML; an explicitly empty value disables admin access. -With no key, admin requests return JSON 503 while public reads and WebSockets -continue normally. With a key, missing, incorrect or duplicate Authorization -headers return JSON 401 with `WWW-Authenticate: Bearer`. - -`GET /api/v1/admin/config` returns selected running settings: CORS options with -Beacon defaults applied, `auth.configured`, and `ingest.broker_count` (configured -broker workers, not connection status or a tunable processing-worker pool). -The CORS lists are the options supplied to the middleware; its normal matching -normalization still applies. The response excludes -credential fields, broker addresses, channel material, database settings and -other configuration. Unknown admin paths return 404 and unsupported -methods on the config endpoint return 405 after authentication. -Global CORS preflights remain public. Use a long, randomly generated key, keep -it out of source control and logs, and send it only in the Authorization header, -never the URL or request body. Require HTTPS at the reverse proxy and restrict -direct access to Beacon's HTTP listener to that proxy or a private connection. -Changing the key requires a restart. No API key is issued automatically. The -key must be at least 16 characters with no inner whitespace (surrounding -whitespace is trimmed); an unusable key prevents startup. - -`PUT /api/v1/admin/config` accepts only -`{"cors":{"allowed_origins":["https://example.org"]}}`. It replaces the entire -origin list immediately and updates the reported configuration with the same -policy. Requests already in progress may use the previous policy. Concurrent -valid updates are serialized; updates take effect one at a time. The response -contains `config`, `persisted: false` and `requires_restart: false`. - -Updates are **runtime-only**: no file or database is written, and restarting -reloads the saved configuration. Keep 1–32 ASCII HTTP(S) origins, at most 512 bytes -each, with an optional single hostname wildcard; a sole `*` permits all origins. -Empty/null lists, URL paths/queries/credentials, control characters and unknown -fields are rejected. Requests must be JSON, at most 16 KiB. Other CORS options, -auth/credential fields and broker count cannot be changed here; there is no -configurable `ingest.worker_count`. Cross-origin admin clients need PUT allowed -in the saved CORS methods. CORS controls browser access, not authentication. - -Operator accounts are available at `GET/POST /api/v1/admin/accounts` and -`GET/DELETE /api/v1/admin/accounts/{id}`. POST accepts a JSON `name` field in a -body up to 4 KiB; names are trimmed, case-sensitive and limited to 128 Unicode -characters without control characters. Active names are unique. DELETE soft -deactivates the record (204); missing IDs return 404 and an already inactive -record returns 409. A deactivated name may be reused by a new account. -Lists include active and inactive records, newest first, without pagination. -These are operator-defined records; no login, session or API token is created. -Cross-origin account clients need both `POST` and `DELETE` in the saved -`cors.allowed_methods`; the default `GET, HEAD, OPTIONS` is read-only. For an -admin UI that also updates configuration, use `[GET, HEAD, OPTIONS, POST, PUT, -DELETE]`, restrict `cors.allowed_origins` to that UI, and allow `Authorization` -and `Content-Type` headers. Otherwise browser preflight blocks these requests -even when the same bearer-authenticated request works with curl. - ### Environment variables (`.env`) | Variable | Default | Description | @@ -368,7 +308,47 @@ names, coordinates, and optional region borders. Regions and channel keys must be defined here unless MeshMapper imports them (`meshmapper.zones.import_groups`, `meshmapper.channels`). ---- +### Admin API + +The `/api/v1/admin` subtree requires `Authorization: Bearer `. Everything else, +including the WebSocket and CORS preflights, is public. + +- Set the key with `BEACON_API_KEY` or `auth.api_key`. A set environment variable wins; an + explicitly empty one disables admin access. No key is generated for you. +- The key must be at least 16 characters with no inner whitespace (surrounding whitespace is + trimmed). An unusable key fails startup. Changing it needs a restart. +- With no key, admin requests return 503. A missing, wrong or duplicated header returns 401 + with `WWW-Authenticate: Bearer`. After auth, unknown paths are 404 and unsupported methods + 405. +- Use a long random key, send it only in the header (never the URL or body), and keep it out + of source control and logs. Terminate HTTPS at the proxy and keep Beacon's listener + private to it. + +Endpoints: + +- **`GET /admin/config`** returns the CORS options with defaults applied, `auth.configured` + and `ingest.broker_count` (configured broker workers, not connection status). No + credentials, broker addresses, channel material or database settings. +- **`PUT /admin/config`** accepts only `{"cors":{"allowed_origins":[...]}}` and replaces the + origin list immediately. It is **runtime-only**: nothing is written, and a restart reloads + the file. The response carries `config`, `persisted: false` and `requires_restart: false`. + Send 1–32 ASCII http(s) origins of at most 512 bytes each, with an optional single + hostname wildcard; a lone `*` allows all. Empty or null lists, paths, queries, credentials, + control characters, unknown fields and bodies over 16 KiB are rejected. Concurrent updates + apply one at a time; requests already in flight may see the old list. +- **`GET/POST /admin/accounts`, `GET/DELETE /admin/accounts/{id}`** manage operator account + records (no login, session or token). POST takes `{"name": "..."}` (body ≤ 4 KiB; name + trimmed, case-sensitive, ≤ 128 characters, no control characters, unique among active + accounts). DELETE deactivates: 204, or 404 if missing, 409 if already inactive. The name + can then be reused. The list returns active and inactive accounts, newest first, + unpaginated. +- **`GET /admin/backup`** is opt-in; see [Backup and export](docs/backup-export.md). + +Browser admin clients need the matching methods in the saved `cors.allowed_methods`. The +default `GET, HEAD, OPTIONS` is read-only. For an admin UI, use +`[GET, HEAD, OPTIONS, POST, PUT, DELETE]`, restrict `cors.allowed_origins` to that UI and +allow the `Authorization` and `Content-Type` headers, or preflight blocks requests that work +from curl. CORS controls browser access, not authentication. ### Reverse proxies and rate limits @@ -398,44 +378,50 @@ proxy is trusted, and once at runtime when a request carries forwarding headers it is ignoring. Full nginx and Caddy examples are in [beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs). ---- +### Analytics retention + +Historical stats read hourly rollups kept for `analytics.rollup_retention` (default 90 +days), independently of `packets.retention`. Each UTC hour is rolled about 95 minutes after +it closes, and packet cleanup holds back up to 24h of raw rows until their hours are rolled. +An hour that lost raw rows first is reported as partial, without values. Drill-downs, +sub-hour observer activity and observer comparison still read raw data. Details: +[docs/historical-stats.md](docs/historical-stats.md). + +### Logging + +Logs go to stderr; collect and rotate them with Docker or systemd. Set `log.level` +(`debug`, `info`, `warn`, `error`; default `info`) and `log.format` (`text` or `json`) in +`config.yaml`, or override with `LOG_LEVEL` / `LOG_FORMAT`. Invalid values fail startup. + +Every record has a `component`. Ingest records add the broker name; HTTP completion records +add the validated client address, route, status and duration. Query strings and WebSocket +hello payloads are never logged. Expected ingest skips and routine WebSocket lifecycle +events are debug-level. Reverse-proxy access logs (and fail2ban rules on them) are +unaffected. ### Foreign repeater indication -Set `nodes.mark_foreign: true` to expose `possiblyForeign` on repeater nodes. -The local operating area is the union of **all configured** -`iatas..borderFile` GeoJSON Polygon/MultiPolygon features, with -`meshmapper.zones` boundaries replacing them where MeshMapper has one. IATAs -without a border do not add an area; airport coordinates and the current API -region filter are not boundaries. Enabling this with no border source, missing -files or invalid geometry fails startup. Border file changes require a restart. - -Inside any polygon (including its edges) means `false`; outside the entire union -means `true`. Hole interiors are outside; hole edges are local. Other node roles, -missing or invalid positions have no classification (an explicit 0/0 advert -clears the stored position). -This is a hint based on reported position, not proof of a repeater's origin. -Packet ingestion, heard-in IATAs and route matching remain unchanged. - -Node list/detail reads apply the current geometry after cache reads, so existing -historical nodes need no backfill. A `nodeUpdate` includes `possiblyForeign` -when its advert provides a position: a boolean for known positions, `null` to -clear it when the advert reports 0/0 or an invalid position. Omission retains the -previous value when a repeater's advert omits its position. A change to another -role also sends `null`. `lat`/`lng` follow the same rule: omitted keeps the -client's position, explicit `null` clears it. -The field is omitted everywhere when the feature is disabled (the default). - -Use longitude/latitude coordinate order and split antimeridian-crossing borders -into MultiPolygons as described in [RFC 7946 section 3.1.9](https://www.rfc-editor.org/rfc/rfc7946#section-3.1.9). -Classification rejects unsplit edges spanning more than 180 degrees rather than -silently treating them as the complementary global area. - -## Authentication - -Public reads and the WebSocket need no credentials. Only the `/api/v1/admin` -subtree is protected, by a bearer key (see [Admin authentication](#admin-authentication)). -Serve Beacon behind an HTTPS reverse proxy. +Set `nodes.mark_foreign: true` to add `possiblyForeign` to repeater nodes. It is `true` when +the repeater's reported position is outside the union of all configured +`iatas..borderFile` polygons, with `meshmapper.zones` borders replacing them where +MeshMapper has one. IATAs without a border add no area, and airport coordinates aren't used. +Enabling it with no border source, a missing file or invalid geometry fails startup. Border +changes need a restart. + +- Points on an edge (including hole edges) are inside; hole interiors are outside. +- Other roles and missing or invalid positions get no value. A 0/0 advert clears the stored + position. +- It's a hint from the reported position, not proof of origin. Ingest, heard-in IATAs and + route matching are unaffected. +- Computed at read time, so existing nodes need no backfill. The field is omitted when the + feature is off (the default). +- In `nodeUpdate`, it's a boolean when the advert carries a position, `null` when the + position is 0/0 or invalid or the node stops being a repeater, and omitted to keep the + previous value. `lat`/`lng` follow the same rule. + +GeoJSON uses longitude/latitude order. Split antimeridian-crossing borders into +MultiPolygons per [RFC 7946 §3.1.9](https://www.rfc-editor.org/rfc/rfc7946#section-3.1.9); +edges spanning more than 180° are rejected. --- @@ -646,62 +632,33 @@ List `limit` values are clamped to 1–200. | `GET` | `/traces` | List trace tags (`type=TRACE\|PING`, IATA/region, scope, `since`/`until`, `cursor`+`cursorTag`) | | `GET` | `/traces/{tag}` | Get full trace detail with resolved routes | ---- - -## Acknowledgements - -See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people who have helped build -Beacon. +### Observer activity -Beacon stands on the shoulders of giants. See [SHOULDERS.md](SHOULDERS.md) for -the full list of open source projects that make this possible. +`GET /observers/{observerId}/activity` returns buckets over `[windowStart, windowEnd)` (live +requests include the current partial bucket), plus `generatedAt`, `source` (`raw` or +`hourly`) and `summary`. -## Application logging +- `recordedPackets` counts stored observations in the window; repeated broker deliveries of + the same packet count once. Unknown payload types show as `-1`. +- Freshness fields (`lastCompleteHour` with its start and end, `latestRecordedAt`) are + measured at `generatedAt`, even for historical `until` requests. +- `until` (epoch ms, within the last 30 days) lines up two observers' charts. +- Hourly mode reads the rollups plus raw rows for hours not rolled yet (at most 24h). + `rolledUntil` ends the rolled range and `rawFrom` starts the raw tail; hours between them + are uncovered. +- Missing records don't prove downtime. The observer's `observationCount` is a legacy + cumulative presence counter (including status and neighbour events), not a packet total. -Beacon writes application logs to stderr. Configure the minimum level and output format in `config.yaml`: - -```yaml -log: - level: info # debug, info, warn, error - format: text # text or json -``` - -`LOG_LEVEL` and `LOG_FORMAT` override file settings; empty settings use `info` and `text`. Invalid values prevent startup. Configuration-loading failures can use the bootstrap text logger before file settings are available; failures after initialization retain error severity at every supported level. - -Records include a component field. Ingest workers also include their broker name, and HTTP completion records include the validated client address, route, status and duration. Query strings and protocol hello payloads are excluded. Expected ingest skips and routine WebSocket lifecycle details are debug-level. Changing the application's format does not change Caddy/Apache access logs or their fail2ban configuration. Collect/rotate stderr through Docker or systemd. +### Saved-route evidence -### Analytics retention +`GET /routes/{iata}/{pathKey}/observations` returns retained observations matching a saved +route's full path bytes, hash size and hop count within its IATA. `range` defaults to `24h` +(max `720h`) and `limit` to 50 (max 200); page with `nextPageCursor` → `pageCursor`. +Evidence expires with raw packets. -Historical stats (traffic, payload, observer activity, talker, advertiser, node, -scope, Signal and Paths) are read from hourly rollup tables kept for -`analytics.rollup_retention` (default 90 days), independently of `packets.retention`. -One background task rolls each UTC hour about 95 minutes after it closes; -packet cleanup waits for unrolled hours (up to 24h) so they can be rolled in full. -An hour whose raw rows were deleted first is marked partial and reported without values. -Packet drill-down, sub-hour observer activity and observer comparison still read -retained raw data. See [docs/historical-stats.md](docs/historical-stats.md). - -### Observer monitoring metrics - -Observer activity returns buckets in `[windowStart, windowEnd)`, including the current partial bucket for live requests, plus -`generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the -sum of stored observations in those buckets; repeated broker delivery of the -same retained packet/observer pair counts once. Unknown payload types appear as -`-1` rather than disappearing. Freshness fields are measured at `generatedAt` even for historical `until` requests; `recordedPackets` alone follows the selected window. `lastCompleteHour` uses the previous complete UTC -hour and includes its own start/end; `latestRecordedAt` is the latest retained -reception timestamp. Missing records do not prove downtime. The optional `until` -(epoch milliseconds, within the last 30 days) aligns two observers' charts. - -The existing observer `observationCount` remains a legacy cumulative presence -counter for compatibility, including status/neighbour events. It is not a -period packet total. Broker presence and packet-arrival timestamps are now -updated separately; this cannot reconstruct previously overwritten timestamps. -Hourly activity comes from the analytics rollups, plus raw rows for hours not yet rolled -(at most 24h back), and includes unknown-type activity. `rolledUntil` is the end of the newest -complete rollup hour and `rawFrom` the start of the raw tail; hours between them are uncovered. +--- -### Saved-route evidence +## Acknowledgements -`GET /api/v1/routes/{iata}/{pathKey}/observations` returns retained observations that match a saved -route's full path bytes, hash size and hop count within its IATA. `range` defaults to `24h` (max `720h`), -`limit` to 50 (max 200); page with `nextPageCursor` → `pageCursor`. Evidence expires with raw packets. +See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people who have helped build Beacon, and +[SHOULDERS.md](SHOULDERS.md) for the open source projects it stands on. diff --git a/docs/backup-export.md b/docs/backup-export.md index 54551f05..d7d24dac 100644 --- a/docs/backup-export.md +++ b/docs/backup-export.md @@ -1,178 +1,136 @@ -# Database and saved-config export and verification - -`beacon-backup` is the export and archive-validation foundation for issue #72. It -produces a private, versioned `.tar.gz` using PostgreSQL's `pg_dump` and can verify -an existing bundle offline. It does not yet provide a web interface, account -login, scheduled/remote storage or automatic import. - -Build it with `go build ./cmd/beacon-backup`. For exports, install `pg_dump` in the -same runtime as this command; an installation on the Docker host does not install -it inside an app container. Use a client of the same major version as the source -PostgreSQL server; an older client cannot dump a newer server. - -Set libpq's standard `PGHOST`, `PGPORT`, `PGDATABASE`, `PGUSER` and TLS settings. -`PGDATABASE` is required and must be a plain database name. Prefer a private -`PGPASSFILE` (0600 on Unix) or an existing libpq service configuration for secrets. -The command does not load `.env`, read `POSTGRES_DSN`, start Beacon, run migrations, -subscribe to MQTT or connect to Redis. Connection settings are not command-line -arguments and client stderr is not printed because it can contain private data. - -For example, after configuring those connection settings: +# Backup and export + +Beacon exports its PostgreSQL database and saved `config.yaml` as a private, versioned +`.tar.gz`, either with the standalone `beacon-backup` command or an opt-in admin download. +`beacon-backup -verify` checks an archive offline. There is no import, scheduling or remote +storage. + +## Archive format 1 + +Exactly three regular files: + +- `manifest.json`: format and exporter version, creation time, payload sizes and SHA-256 + hashes, and the exclusions below. +- `database.sql`: one consistent `pg_dump` snapshot (schema, data, migration journal), + without ownership, ACLs or tablespaces so it restores under any role. +- `config.yaml`: the saved file byte-for-byte, read just before the snapshot. Avoid config + edits during an export. + +**It is sensitive and unencrypted** (keys, message data); keep it private. It is not a full +recovery package: `.env` and environment variables, `borderFile` and TLS files, runtime-only +changes, PostgreSQL roles and cluster settings, Redis and deployment files are not included. + +## Requirements and limits + +- `pg_dump` must be in the **same runtime** as the exporter (the CLI's environment, or the + server's PATH for the download), with a major version matching the database. A client on + the Docker host or in another container doesn't count. +- The Docker image ships the PostgreSQL 16 client. Its Alpine 3.19 base has no 17/18 + packages, so a newer database needs a different runtime, not just `POSTGRES_CLIENT_MAJOR`. +- The temp directory (`TMPDIR`) needs about twice the SQL limit free. A small tmpfs won't do. +- The output filesystem must support hard links (used to publish atomically). Existing + destinations, symlinks included, are never replaced. Files are 0600 and staging dirs 0700; + on Windows, restrict the directory ACLs yourself. + +| Limit | Default | +| --- | --- | +| Timeout | 10 min (`-timeout`) | +| Uncompressed SQL | 1 GiB (`-max-bytes`, up to 1 TiB) | +| Saved YAML / manifest | 1 MiB / 64 KiB | +| Compressed archive (verify) | SQL limit + 1% + 2 MiB | +| Lock wait | 5 s | + +Failures, timeouts and interrupts clean up and publish nothing. A SIGKILL or power loss can +leave a private staging dir: `.beacon-backup-*` next to the CLI output, or +`beacon-download-*` in the server's temp dir, which Beacon sweeps on its next start. + +## CLI export ```sh +go build ./cmd/beacon-backup beacon-backup -config /private/config.yaml -output /private/beacon-20260913.tar.gz ``` -Use an output directory controlled by the operator. Staging directories are -0700 and files are 0600 on Unix; Windows operators must use a directory with -appropriately restricted ACLs. Existing destinations, including symlinks, are -never replaced. A hard link publishes the finished archive atomically, so the -destination filesystem must support hard links. Unsupported filesystems fail -without publishing an output. Normal failure, timeout and handled interruption -remove temporary files; a power loss or SIGKILL can leave a private -`.beacon-backup-*` staging directory for the operator to inspect. - -## Protected download API - -Set `backup.enabled: true` in saved YAML and configure the existing admin bearer -key to enable `GET /api/v1/admin/backup`. Supply the key only in the `Authorization: -Bearer ...` header over HTTPS. No query parameters or request body are accepted. -This is an operator API, not account login or a browser admin panel. Keep it off -on public previews that do not need private exports. - -The server requires `pg_dump` on its own PATH when this option is enabled. The -Docker image includes PostgreSQL 16's client by default. Its Alpine 3.19 base does -not provide PostgreSQL 17/18 packages: changing `POSTGRES_CLIENT_MAJOR` alone is -not an upgrade path. For a newer database, supply a runtime with a compatible -client. Startup compares `pg_dump --version` with `server_version_num` and leaves -backup unavailable if the client is missing, unrecognized or older than the server. -Match the source database major version. A client on the host or in a different container is not -sufficient. No Docker socket or host-command bridge is used. - -The download uses the startup `POSTGRES_DSN`, not the standalone export's ambient -`PGDATABASE`. This first adapter supports a single-host `postgres://` or -`postgresql://` URL with an explicit user and database. Passwords, IPv6 and URL -escaping are supported. It copies TLS settings, `passfile`, `connect_timeout`, -`application_name`, `target_session_attrs` and `options`; known `pool_*` controls -are omitted from libpq. Supported PG environment defaults are captured at startup -and URL values override them. Prefer explicit `sslmode=verify-full` and a trusted -`sslrootcert` for remote databases; password-file/TLS paths must be readable in -the same runtime. Their contents are not included in the archive. - -Keyword DSNs, service indirection, multiple hosts, duplicate/unknown query options, -newer protocol negotiation settings and service-file values with line breaks or -edge whitespace are rejected rather than silently changing targets or TLS policy. -Ambient `PGSERVICE`, `PGSERVICEFILE`, `PGSSLNEGOTIATION`, `PGMINPROTOCOLVERSION`, -`PGMAXPROTOCOLVERSION` and `PGTZ` are also unsupported for this adapter. The -standalone command retains native PG* support for those deployments. Errors do -not echo the DSN. Any failed backup prerequisite logs its reason and leaves only -this feature unavailable; Beacon still starts ingest, WebSocket and other APIs. -Fix the prerequisite and restart to enable downloads. Native connection settings live only in a 0600 service file in -private staging; process arguments contain no connection string or credential, -and no request changes Beacon's environment. - -Only one export or transfer is allowed at a time; another request receives 409. -The existing ten-minute/1 GiB SQL/1 MiB YAML limits apply, and disconnecting -cancels an in-progress export. The completed transfer has a ten-minute write -deadline. Temporary files are removed after completion/failure/cancellation; -abrupt process termination can leave private staging. Responses use `no-store` -and a fixed attachment filename. A failed export returns an error without any -partial archive. Authentication missing/invalid is 401, unconfigured access is -503, invalid input is 400, export failure is 500, timeout is 504 and exceeding the -configured export size limit is 507. Other export failures retain sanitized -operator diagnostics in the server log; raw pg_dump stderr remains discarded. - -Provision a private writable temporary directory (`TMPDIR` on Unix) with space -for roughly twice the SQL limit plus overhead. A small read-only-container tmpfs -is insufficient for a large backup. Exports include saved YAML verbatim and may -include keys/message data; the exclusions below still apply. This API does not -implement archive upload, import/overwrite, schedules or remote storage. - -## Format 1 - -The tar contains exactly three regular files with fixed names: - -- `manifest.json`: format version, creation time, exporter version, dump format, - uncompressed payload sizes/SHA-256 hashes and explicit exclusions. -- `database.sql`: one consistent `pg_dump` snapshot of schema and data, including - Beacon's migration journal. Ownership, ACLs and tablespace placement are omitted - so objects can be restored under the destination operator. -- `config.yaml`: the supplied saved file, byte-for-byte, including comments and - any keys. It is captured before the database snapshot; avoid configuration edits - during export if the files must describe the same deployment state. - -This is sensitive, unencrypted data. Store and transfer it privately. The bundle -does not include deployment environment variables, `.env`, external files such as -`borderFile` inputs or TLS keys, runtime-only changes, PostgreSQL roles/cluster -settings, Redis or service/deployment files. Retain those separately. A config -export is not the sanitized admin-config response and is not a complete server -recovery package by itself. - -Defaults are ten minutes and a 1 GiB uncompressed SQL limit. Saved YAML is capped -at 1 MiB. `-timeout` and `-max-bytes` set finite positive limits (SQL maximum 1 TiB). -Allow disk space for both the uncompressed SQL and compressed bundle, roughly -twice the chosen SQL limit plus overhead. A five-second lock-wait limit prevents -waiting indefinitely behind schema changes. Export failure publishes no backup; -check the client version, privileges, connection settings and available capacity -privately. The command deliberately does not expose raw client diagnostics. - -## Verify an existing archive +Connection comes only from the standard libpq environment (`PGHOST`, `PGPORT`, +`PGDATABASE`, `PGUSER`, TLS settings); `PGDATABASE` is required. Keep the password in a 0600 +`PGPASSFILE` or a service file. The command ignores `.env` and `POSTGRES_DSN`, runs no +migrations, and never prints `pg_dump` stderr since it can leak private data. If it fails, +check the client version, privileges, connection settings and free space. + +## Download API + +`GET /api/v1/admin/backup` streams a fresh archive. Enable it with `backup.enabled: true` and +the admin bearer key, then restart. Send the key only in the `Authorization` header over +HTTPS; the endpoint takes no query or body. Leave it off on public previews. + +At startup Beacon checks `pg_dump --version` against the server. If that or another +prerequisite fails, it logs why and disables only backup; fix it and restart. + +**Connection.** The download uses `POSTGRES_DSN`, which must be a single-host `postgres://` +URL with an explicit user and database. TLS options, `passfile`, `connect_timeout`, +`application_name`, `target_session_attrs` and `options` are passed through; `pool_*` +options are dropped; supported `PG*` env defaults apply under the URL. Keyword DSNs, +services, multiple hosts, unknown or duplicate options, protocol-negotiation settings and +the `PGSERVICE`, `PGSERVICEFILE`, `PGSSLNEGOTIATION`, `PG{MIN,MAX}PROTOCOLVERSION` and `PGTZ` +env vars are rejected. Use the CLI for those setups. Prefer `sslmode=verify-full` for remote +databases. Credentials go to a 0600 service file in staging, never into process arguments, +logs or errors. + +**Behaviour.** One export at a time. Disconnecting cancels it, and the transfer has a +10-minute write deadline. Responses are `no-store`. A failed export sends no partial archive +and logs a sanitized reason. + +| Status | Meaning | +| --- | --- | +| 400 | Query string or body sent | +| 401 | Missing or invalid key | +| 409 | Another export is running | +| 500 | Export failed | +| 503 | Backup unavailable, no admin key, or shutting down | +| 504 | Timed out | +| 507 | SQL size limit exceeded | + +## Verify ```sh beacon-backup -verify /private/beacon-20260913.tar.gz ``` -Verification reads a regular file without extracting members, writing files, -executing SQL or connecting to PostgreSQL, Redis or MQTT. It needs neither a -configuration file nor `pg_dump` or connection credentials. Do not combine -`-verify` with the export-only `-config` or `-output` flags. - -The verifier accepts the native format-1 bundle: exactly the three fixed regular -members with 0600 modes and USTAR headers (or the size-only PAX header needed for -SQL over 8 GiB). It rejects duplicates, unexpected paths, links, devices, other -extended metadata, incomplete tar/gzip endings and trailing or concatenated -streams. Manifest fields must be known, with no duplicate keys or case aliases; -the payload sizes and SHA-256 hashes must match the bytes actually read. - -The same `-max-bytes` SQL limit applies, with a separate 1 MiB config limit and -64 KiB manifest limit. Compressed input is capped at the SQL limit plus 1% and -2 MiB, matching the exporter. Headers and decompressed data are bounded too; -SQL and config are hashed as streams, without keeping their contents in memory. -The ten-minute default `-timeout` and interrupt handling check cancellation -between reads; they cannot interrupt a blocked filesystem read. - -Exit status 0 means the archive structure and checksums passed. Failure returns -a nonzero status with no payload, manifest values or input path in its diagnostic. -The manifest is not signed: verification does **not** establish authenticity, -safe SQL, configuration validity or restorability. It does not make an untrusted -dump safe to restore, and does not provide import/overwrite or deployment recovery. - -## Restore verification - -Use trusted bundles only: PostgreSQL dumps can contain executable SQL. Run the -archive verification above, then extract into private staging. For a -**new, empty disposable database**, with its own explicit `PGDATABASE` and -target-role connection settings: +Reads the file without extracting, running SQL or connecting to anything, so it needs no +config, `pg_dump` or credentials. `-max-bytes` and `-timeout` apply. It requires exactly the +three members as 0600 regular files with USTAR headers (size-only PAX allowed for SQL over +8 GiB). It rejects duplicates, links, devices, extra metadata, truncated or concatenated +streams, and unknown or duplicate manifest keys. Sizes and SHA-256 hashes must match. + +Exit 0 means structure and checksums passed. Errors don't include payload, manifest values +or the path. The manifest is unsigned, so a pass does **not** prove authenticity, safe SQL or +that the dump restores. + +## Restore + +Only restore trusted bundles, because a dump can run arbitrary SQL. Verify, extract +privately, and load into a **new, empty database** on a compatible PostgreSQL with the +needed extensions: ```sh psql -X --set ON_ERROR_STOP=on --single-transaction --file database.sql ``` -Restore with a compatible PostgreSQL version and the required extensions already -available. Reconcile the migration journal and representative records/relationships -before relying on the bundle. Review saved configuration and restore external -secrets/files separately before starting a new Beacon server. Never restore over -live data simply to test an export; overwrite/import requires a separate workflow. - -PostgreSQL references: [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.html), -[connection environment](https://www.postgresql.org/docs/16/libpq-envars.html), -[password files](https://www.postgresql.org/docs/16/libpq-pgpass.html). - -CI runs the compiled command and its PostgreSQL round-trip test with a dedicated -PostgreSQL 16 service. The test also verifies the export offline with no client on -PATH or connection settings, and rejects a truncated copy. To repeat it privately, -build the command, set the `PG*` connection variables for an isolated test server and set -`BEACON_BACKUP_TEST_POSTGRES=1` plus `BEACON_BACKUP_TEST_BINARY` to the command's -absolute path. Run `go test ./internal/backup -run '^TestExportPostgres$' -v`. -The test role needs permission to create/drop its two randomly named databases; -the test migrates and restores only those databases and removes them afterward. +Check the migration journal and some records, review the config, and restore external +secrets and files before pointing a new Beacon at it. Never test an export by restoring over +live data. See [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.html) and +[libpq environment](https://www.postgresql.org/docs/16/libpq-envars.html). + +## Testing + +CI runs export, restore round-trip, offline verify and truncated-archive rejection against +PostgreSQL 16. Locally, with `PG*` pointing at a throwaway server whose role can create +databases: + +```sh +go build -o "$PWD/beacon-backup" ./cmd/beacon-backup +BEACON_BACKUP_TEST_POSTGRES=1 BEACON_BACKUP_TEST_BINARY="$PWD/beacon-backup" \ + go test ./internal/backup -run '^TestExportPostgres$' -v +``` + +The test creates, migrates, restores and drops two randomly named databases. diff --git a/docs/packet-summaries.md b/docs/packet-summaries.md index e954765c..34c7ccfd 100644 --- a/docs/packet-summaries.md +++ b/docs/packet-summaries.md @@ -1,28 +1,22 @@ # Packet summaries -Packet lists, regional lists, reconnect backfill and live `packetObservation` -events share an optional `summary` string. It describes metadata carried by that -packet, rather than looking up a node's current name or decoding history again. +Packet lists, regional lists, reconnect backfill and live `packetObservation` events carry +an optional `summary` string. It comes from the packet's own stored metadata, never from a +node's current name or a fresh decode. | Payload | Summary | Source | |---|---|---| | ADVERT | Advertised name | Saved `appData.name` / verified decoded advert | | ACK | `ACK 01020304` | Four-byte acknowledgement checksum | | TRACE | `TRACE efbeadde` | Trace tag | -| TRACE classified as PING | `PING efbeadde` | Trace tag, with the existing single-destination classification | +| TRACE classified as PING | `PING efbeadde` | Trace tag | -References use eight lowercase hexadecimal characters in the same byte order -as packet detail's `checksum` / `traceTag`. Extended ACK retry bytes do not change -the displayed checksum. Zero is a valid reference. These short references are -not globally unique packet identifiers or proof of delivery/authentication; no -message body, trace auth code or inferred sender/recipient is added to the summary. - -Historical reads require the expected stored type and a string containing -exactly eight hex characters. Missing, malformed or unsupported references omit -the field. Advert behavior is unchanged. List/backfill queries project the text -from saved JSON in their existing query, without a schema change, backfill job -or per-packet database lookup. The existing web summary display needs no new API -field or client release to render these values. - -Other packet types remain separate follow-ups under issue #99. A missing summary -does not mean the packet is invalid or unreadable in packet detail. +- References are eight lowercase hex characters in the same byte order as packet detail's + `checksum` / `traceTag`. Zero is valid. Extended ACK retry bytes don't change the checksum. +- They are short references, not unique packet IDs or proof of delivery. No message body, + trace auth code or inferred sender/recipient is included. +- The text is projected from saved JSON in the existing list queries: no extra column, + backfill or per-packet lookup. A stored value of the wrong type or not exactly eight hex + characters omits the field. +- Other payload types have no summary yet. A missing summary doesn't mean the packet is + invalid. From 1f83df8583e919ad79e4faba5e3905e1137a0a14 Mon Sep 17 00:00:00 2001 From: MrAlders0n <55921894+MrAlders0n@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:43:03 -0400 Subject: [PATCH 141/141] docs: move operator docs to beacon-docs (#206) * docs: keep only server-specific contributor notes * docs: trim README to the code and link to beacon-docs * chore: strip tracking parameters from two airport website URLs --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/docker-publish.yml | 2 +- CODE_OF_CONDUCT.md | 10 +- CONTRIBUTING.md | 183 ++---- CONTRIBUTORS.md | 2 +- PROFILING.md | 70 --- README.md | 645 ++-------------------- SECURITY.md | 2 +- SHOULDERS.md | 46 +- config.yaml.example | 7 +- docs/backup-export.md | 136 ----- docs/packet-summaries.md | 22 - env.example | 5 +- internal/iatadb/gen/airports.csv | 4 +- 16 files changed, 151 insertions(+), 989 deletions(-) delete mode 100644 PROFILING.md delete mode 100644 docs/backup-export.md delete mode 100644 docs/packet-summaries.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 64ec8161..c7bf29c4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -34,5 +34,5 @@ paste logs here ## Additional context -Any other context — config snippets (redact credentials), MQTT broker details, +Any other context: config snippets (redact credentials), MQTT broker details, mesh topology, etc. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 433c4232..f2fdb7f3 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -15,7 +15,7 @@ related to an existing issue or limitation, link it here. ## Proposed solution -How you'd like it to work. API shape, config options, WS event format — as much +How you'd like it to work. API shape, config options, WS event format, as much detail as you have. ## Alternatives considered diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 16463f3b..ef6d6c03 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -23,7 +23,7 @@ issues: `Closes #123` or `Related to #456`. - [ ] API changes include swagger annotations and `swag init` has been run - [ ] `docs/` is committed if swagger was regenerated - [ ] New dependencies are added to `SHOULDERS.md` -- [ ] I have read [CONTRIBUTING.md](CONTRIBUTING.md) +- [ ] I have read [CONTRIBUTING.md](../CONTRIBUTING.md) ## Testing notes diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0cc526c8..70f27ec4 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,7 +38,7 @@ jobs: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} labels: | org.opencontainers.image.title=beacon-server - org.opencontainers.image.description=Beacon — MeshCore LoRa mesh observation server + org.opencontainers.image.description=Beacon: MeshCore LoRa mesh observation server org.opencontainers.image.licenses=AGPL-3.0-or-later flavor: | latest=false diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c8610310..e50faa7d 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -25,7 +25,7 @@ Examples of behaviour that contributes to a positive environment: Examples of unacceptable behaviour: -- Hate speech or discriminatory language or jokes of any kind — including but +- Hate speech or discriminatory language or jokes of any kind, including but not limited to those targeting race, ethnicity, nationality, gender identity or expression, sexual orientation, disability, religion, age, or socioeconomic background @@ -34,7 +34,7 @@ Examples of unacceptable behaviour: - Trolling, insulting or derogatory comments, and personal or political attacks - Public or private harassment of any kind - Deliberate misgendering or use of rejected names -- Publishing others' private information — such as a physical or email address — +- Publishing others' private information, such as a physical or email address, without their explicit permission - Threats of violence or incitement of violence toward any individual or group - Dismissing or minimising reports of harassment or discrimination @@ -55,15 +55,15 @@ when appropriate. ## Scope -This Code of Conduct applies within all project spaces — GitHub issues, pull -requests, discussions, and the MeshCore Canada Discord server — and also when an +This Code of Conduct applies within all project spaces (GitHub issues, pull +requests, discussions, and the MeshCore Canada Discord server) and also when an individual is representing the project in public spaces. ## Reporting Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported by contacting the maintainers directly on the MeshCore Canada Discord -server at https://discord.gg/Gz3KvJx2hf — reach out to **dedskelly** privately. +server at https://discord.gg/Gz3KvJx2hf; reach out to **dedskelly** privately. All reports will be handled with discretion and confidentiality. Maintainers are obligated to respect the privacy and safety of the reporter. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 382b7352..46702834 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,40 +6,16 @@ before opening a PR. --- -## Before you start - -**Open or comment on an issue before starting work.** This avoids duplicate -effort and lets maintainers flag if something is already in progress or out of -scope. For small bug fixes a brief comment is fine; for larger features please -discuss the approach first. - -**One thing per PR.** Each pull request should cover one logical change — a bug -fix, a new endpoint, a refactor, a new test. PRs that touch many unrelated parts -of the codebase are hard to review and hard to revert if something goes wrong. - -**No fully AI-generated contributions.** We welcome developers who use AI tools -to assist their work, but PRs should reflect the author's own understanding and -judgement. PRs that appear to be unreviewed AI output may be closed without -further comment. - ---- - -## Branches - -- `main` — stable releases only, protected. Never target this directly. -- `dev` — active development. All PRs target `dev`. - ## Workflow -1. Fork or create a branch from `dev` -2. Make your changes -3. Run the checklist below -4. Open a pull request against `dev` with a clear description of what changed - and why, referencing any related issues - ---- +Branches, the pull request flow, commit message format and the release process are the same +for every Beacon repo and live in +[beacon-docs/CONTRIBUTING.md](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/CONTRIBUTING.md). +In short: open or comment on an issue first, branch from `dev`, keep each PR to one change, +write conventional commit messages, and open the PR against `dev`. Running beacon-web against +a local server is described there too. -## Checklist before opening a PR +Before opening a PR here: ``` go build ./... # must compile @@ -75,13 +51,31 @@ swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency --- +## Backup export tests + +CI runs export, restore round-trip, offline verify and truncated-archive rejection against +PostgreSQL 16. Locally, with `PG*` pointing at a throwaway server whose role can create +databases: + +```sh +go build -o "$PWD/beacon-backup" ./cmd/beacon-backup +BEACON_BACKUP_TEST_POSTGRES=1 BEACON_BACKUP_TEST_BINARY="$PWD/beacon-backup" \ + go test ./internal/backup -run '^TestExportPostgres$' -v +``` + +The test creates, migrates, restores and drops two randomly named databases. The export +format and operator usage are documented in +[Backup and export](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/backup-export.md). + +--- + ## Database changes All schema changes must include a proper migration path: - Add a new migration file to `db/migrations/` following the existing naming convention (e.g. `002_add_observation_count.sql`). `001_baseline.sql` is the - 2.0.0 schema; do not modify existing migration files — append only via new files. + 2.0.0 schema; do not modify existing migration files; append only via new files. - Update `db/queries/queries.sql` with any new or modified queries - Re-run `sqlc generate` to regenerate `db/sqlc/` - Update the store layer in `db/` to expose the new functionality @@ -89,7 +83,7 @@ All schema changes must include a proper migration path: - Historical stats read hourly rollups, never raw observations; follow [docs/historical-stats.md](docs/historical-stats.md) to add a metric -Never edit files under `db/sqlc/` by hand — they are generated by sqlc and will +Never edit files under `db/sqlc/` by hand; they are generated by sqlc and will be overwritten. If you need to work around a sqlc limitation, document it clearly in the query comment. @@ -125,7 +119,7 @@ docs: - Add or update `// @Summary`, `// @Param`, `// @Success`, `// @Failure`, and `// @Router` comments on the handler function -- Response types are defined in `internal/api/` — add new types there, not +- Response types are defined in `internal/api/`; add new types there, not inline in handlers - After changing any handler or API type, rerun the `swag init` command from the checklist and commit the updated `docs/` directory @@ -214,55 +208,37 @@ AIRPORTS_CSV=/path/to/airports.csv go run ./internal/iatadb/gen --- -## Commit messages - -Use the conventional commits format: - -``` -feat(routes): add observation_count to known routes response -fix(ingest): correct lat/lon divisor for advert payloads -refactor(hub): collapse RegionIATAs into IATAs in Scope -test(api): add unit tests for NodeTypeName and NodeTypeFromString -chore: update airports.csv -docs: expand CONTRIBUTING.md -``` - -Scopes are optional but helpful for larger codebases. Common scopes: `api`, -`db`, `ingest`, `hub`, `ws`, `handlers`, `config`, `keystore`. - ---- - ## Project structure ``` -cmd/beacon/ — main entry point, wiring, startup -cmd/beacon-backup/ — standalone backup export/verify command -db/ — store layer: sqlc-generated code + thin mapping layer - migrations/ — SQL schema (2.0.0 baseline + numbered files, append only) - queries/ — SQL queries (input to sqlc) - sqlc/ — generated Go code (do not edit by hand) +cmd/beacon/ main entry point, wiring, startup +cmd/beacon-backup/ standalone backup export/verify command +db/ store layer: sqlc-generated code + thin mapping layer + migrations/ SQL schema (2.0.0 baseline + numbered files, append only) + queries/ SQL queries (input to sqlc) + sqlc/ generated Go code (do not edit by hand) internal/ - api/ — response types and Reader interface - handlers/ — HTTP handlers (validation, routing, response) - middleware/ — auth, rate limiting, logging - router/ — chi router wiring - background/ — periodic maintenance (retention, rollups, age-out) - backup/ — database + config bundle export and verification - borders/ — foreign-repeater classification against IATA borders - cache/ — Redis caching layer over the Reader - config/ — config loading and scope key derivation - hub/ — WebSocket fan-out broker - iatadb/ — in-memory IATA airport lookup - ingest/ — MQTT packet ingestion and side effects - keystore/ — channel key lookup - logging/ — slog setup - lora/ — LoRa airtime calculation - meshmapper/ — MeshMapper scope and zone imports - presence/ — coalesced last-seen writes - profiling/ — opt-in CPU profiling - scopestore/ — transport scope key lookup - ws/ — WebSocket connection handling -docs/ — generated swagger and design notes (swagger: do not edit by hand) + api/ response types and Reader interface + handlers/ HTTP handlers (validation, routing, response) + middleware/ auth, rate limiting, logging + router/ chi router wiring + background/ periodic maintenance (retention, rollups, age-out) + backup/ database + config bundle export and verification + borders/ foreign-repeater classification against IATA borders + cache/ Redis caching layer over the Reader + config/ config loading and scope key derivation + hub/ WebSocket fan-out broker + iatadb/ in-memory IATA airport lookup + ingest/ MQTT packet ingestion and side effects + keystore/ channel key lookup + logging/ slog setup + lora/ LoRa airtime calculation + meshmapper/ MeshMapper scope and zone imports + presence/ coalesced last-seen writes + profiling/ opt-in CPU profiling + scopestore/ transport scope key lookup + ws/ WebSocket connection handling +docs/ generated swagger and design notes (swagger: do not edit by hand) ``` Key patterns to understand before contributing: @@ -272,7 +248,7 @@ Key patterns to understand before contributing: put business logic here. - **Ingest layer** (`internal/ingest/`): processes raw MQTT packets, calls the store, and broadcasts hub events. The `DB` interface in `ingest.go` defines - exactly what the ingest layer needs from the store — keep it minimal. + exactly what the ingest layer needs from the store; keep it minimal. - **Hub** (`internal/hub/`): pure fan-out broker. Events are pre-serialised JSON before entering the hub so the broadcast loop never touches encoding. - **Reader interface** (`internal/api/reader.go`): defines everything the API @@ -282,54 +258,15 @@ Key patterns to understand before contributing: ## Releases -Merges from `dev` to `main` are done by maintainers and represent a versioned -release. Do not open PRs directly against `main`. - -### Cutting a release - -> NOTE: releases are done manually, not from GH. Commits for releases should be -> signed. - -1. Ensure all changes are committed and CI is green on `dev` -2. Bump the version string in the `@version` swagger annotation in - `cmd/beacon/main.go` -3. Regenerate swagger docs: +The flow is in +[Releases and versioning](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/releases.md#cutting-a-release). +The server-specific step is bumping `@version` in `cmd/beacon/main.go` and regenerating Swagger: ```bash swag init -g cmd/beacon/main.go -o docs --parseInternal --parseDependency ``` -4. Commit the version bump and updated docs: - -``` -chore: bump version to vX.Y.Z -``` - -5. Merge `dev` into `main` (fast-forward only): - -```bash -git checkout main -git merge --ff-only dev -``` - -6. Tag the release and push: - -```bash -git tag vX.Y.Z -git push origin main --tags -``` - -Pushing the tag triggers the release CI workflow, which builds binaries for all -supported platforms and attaches them to a draft GitHub release. - -7. Open the draft release on GitHub, paste the release notes, and publish. - -8. Rebase `dev` on `main` to keep histories in sync for future releases: - -```bash -git checkout dev -git rebase main -``` +--- ## Recognition diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 929596ed..0f7e5d98 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -4,7 +4,7 @@ Beacon is built by its contributors. Thank you to everyone who has helped. ## Core -- [ded](https://hackers.town/@ded) — co-founder, lead developer +- [ded](https://hackers.town/@ded), co-founder, lead developer ## Contributors diff --git a/PROFILING.md b/PROFILING.md deleted file mode 100644 index 6fbe7ae3..00000000 --- a/PROFILING.md +++ /dev/null @@ -1,70 +0,0 @@ -# Production CPU profiling - -Beacon can record bounded CPU profiles on Linux and macOS. It's off by default, writes -private files only (there's no HTTP endpoint or port), and a profiling failure stops the -recorder with a warning without affecting ingest. - -## Enable a session - -Create a dedicated `0700` directory writable by the container user, mount it, and set both -variables: - -```yaml -services: - app: - environment: - BEACON_CPU_PROFILE_DIR: /profiles - BEACON_CPU_PROFILE_UNTIL: "2026-10-02T12:00:00Z" # fixed deadline - volumes: - - ./profiles:/profiles -``` - -- The deadline is RFC 3339, at most 72 hours ahead. It survives restarts and is inactive - once past. Don't generate a fresh deadline on every start. -- Changing either setting needs a restart. -- Start with a deadline about a minute out, and compare process CPU, packet freshness, queue - drops and database timeouts with an unprofiled period before running a longer session. -- One Beacon process per directory. Keep it out of web roots, public backups and source - control, and don't run another CPU profiler in the process meanwhile. - -## What gets captured - -- A 30-second sample at start, another 35 minutes later, then every 30 minutes (the extra - five minutes keeps a maintenance trigger from landing in the cooldown). -- An extra sample when route reconfirmation maintenance starts (including its retention - step). -- A five-minute cooldown between capture starts. A periodic sample due during it runs when it - ends; a triggered one is skipped, and any sample cancels a periodic one that's waiting. -- Background task stacks carry a `task` label. -- Shutdown or the deadline ends the active sample early and saves it. - -Limits: 8 MiB per profile; 256 MiB and 512 files per directory, counting metadata, leftovers -and unrelated regular files (subdirectories are ignored). The recorder reserves room for a -full capture first and stops at a limit. Nothing is deleted automatically. Files are `0600`; -a `.partial` file is an interrupted capture. - -## Reading the output - -Each `.pprof` has a `.pprof.json` sidecar with UTC start and end, the trigger, Go version, -process CPU counters, goroutine count, and database-pool counters before and after (pool -values are cumulative, so subtract). No SQL text or credentials. - -Copy completed files off the host privately, noting the image revision and digest. Then: - -```sh -go tool pprof -top capture.pprof -go tool pprof -top -cum capture.pprof -go tool pprof -tags capture.pprof -``` - -Profiles show CPU inside Beacon, not PostgreSQL work or lock waits. Correlate each interval -with host/container CPU and I/O, PostgreSQL wait states, observation throughput and Beacon's -timeout/drop logs. Don't turn on SQL text logging or run full-table counts for this; the -recorder itself makes no database queries. - -Review quiet, burst and maintenance captures separately: maintenance samples bias an -aggregate. Thirty-second windows can miss brief stalls, so a missing stack isn't proof it -never runs. - -When the deadline passes, look for `CPU profiling stopped` in the log, export the files, and -remove the variables and mount at the next deployment. diff --git a/README.md b/README.md index c83ab37f..dca6275e 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ in PostgreSQL, and streams live events to WebSocket clients. - Subscribes to MeshCore MQTT brokers and decodes incoming LoRa packets using [meshcore-go](https://github.com/meshcore-go/meshcore-go) - Stores packets, observations, nodes, observers, traces, routes and channel - messages in PostgreSQL (more backends to come) -- Deduplicates observations across multiple brokers (same packet heard by two + messages in PostgreSQL +- Deduplicates observations across multiple brokers (the same packet heard by two brokers is one observation per observer) - Decrypts group text messages for known channel keys - Detects firmware capability flags from path hash sizes @@ -25,16 +25,9 @@ in PostgreSQL, and streams live events to WebSocket clients. - Seeds regions, IATA display names, and channel keys from a YAML config file on startup -More documentation: - -- [beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs): deployment with the - frontend and reverse proxy examples -- [Backup and export](docs/backup-export.md): private database and config bundles -- [Historical stats](docs/historical-stats.md): how the hourly rollups work -- [Packet summaries](docs/packet-summaries.md): the `summary` field on packet lists -- [CPU profiling](PROFILING.md): bounded production captures - ---- +This repo is the code. Deploying, configuring and operating Beacon is documented in +[beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs); see +[Documentation](#documentation) below. ## Stack @@ -51,612 +44,68 @@ More documentation: | Config | YAML via gopkg.in/yaml.v3 | | Env | godotenv | ---- - -## Getting started - -### Prerequisites +## Running it locally -- Go 1.26+ -- Docker and Docker Compose - -### 1. Clone and configure +You need Go 1.26+ and a PostgreSQL 16 database. Docker is the easy way to get the database. ```bash -git clone https://github.com/MeshCore-Beacon/beacon-server.git -cd beacon-server +git clone https://github.com/MeshCore-Beacon/beacon-server.git && cd beacon-server cp env.example .env cp config.yaml.example config.yaml -``` - -Edit `.env` with your broker credentials and database DSN. Edit `config.yaml` to -define your regions, IATA display names, channel keys, and retention settings. - -### 2. Start PostgreSQL - -Any PostgreSQL 16 database works. For a local throwaway one: - -```bash docker run -d --name beacon-postgres -p 5432:5432 \ - -e POSTGRES_USER=beacon -e POSTGRES_PASSWORD=beacon -e POSTGRES_DB=beacon \ - postgres:16-alpine -``` - -Database migrations are applied automatically on startup. - -### 3. Run - -```bash + -e POSTGRES_USER=beacon -e POSTGRES_PASSWORD=beacon -e POSTGRES_DB=beacon postgres:16-alpine go run ./cmd/beacon ``` -Or pull and run the Docker image: - -```bash -docker pull ghcr.io/meshcore-beacon/beacon-server:latest -``` - -The image is public on GitHub Container Registry — no `docker login` required. -`latest` tracks stable releases; pin a version tag (e.g. `2.0.0`) for -predictable upgrades, and `dev` follows the development branch. - -Beacon will: - -- Load `.env` and `config.yaml` -- Connect to PostgreSQL and seed config data -- Connect to the configured MQTT brokers -- Start the HTTP server on `LISTEN_ADDR` (default `:8080`) - -### Upgrading from 1.x - -2.0.0 starts from a new schema baseline and **needs a fresh, empty database**; 1.x history -does not carry over. Pointed at a 1.x database it refuses to start with `database schema -predates Beacon 2.0.0; 2.0.0 needs a fresh database` (Docker: stop the stack and move or -remove the Postgres data directory). - -Config changes to review: - -- Every manually configured `scopes:` entry needs a `region`. -- `meshmapper.scopes.sources` is ignored; MeshMapper refresh intervals must be within - 1h–24h for scopes and 24h–168h for zones, or startup fails. -- REST rate limiting is on by default. Behind a reverse proxy, set - `server.trusted_proxies` (see [Reverse proxies and rate limits](#reverse-proxies-and-rate-limits)). -- `packets.retention` now defaults to 7 days. - -### Cold start - -Path resolution, capability detection and known routes depend on nodes having advertised -to a local observer. On a fresh database every hop shows `"confidence": "none"` and -`supportsMultibytePaths` is `false` until adverts populate `node_short_ids`. That's -expected and fills in as the mesh is observed. - -GRP_TXT packets whose channel key isn't known yet are stored hash-only. After adding the -key to `config.yaml`, the next restart decrypts the matching history; look for -`backfilled N previously-undecrypted channel message(s)` in the log. - ---- - -## Configuration - -### Environment variables (`.env`) - -| Variable | Default | Description | -| ------------------------ | ------------- | ------------------------------------------------------------ | -| `LISTEN_ADDR` | `:8080` | HTTP listen address | -| `POSTGRES_DSN` | — | PostgreSQL connection string | -| `REDIS_ADDR` | — | Redis address (`host:port`). Leave unset to disable caching. | -| `REDIS_PASSWORD` | — | Redis password (optional) | -| `REDIS_DB` | `0` | Redis database index | -| `CONFIG_PATH` | `config.yaml` | Path to YAML config file | -| `MQTT_BROKER_1_URL` | — | Broker 1 WebSocket URL (e.g. `wss://mqtt1.example.com:443`) | -| `MQTT_BROKER_1_USERNAME` | — | Broker 1 username | -| `MQTT_BROKER_1_PASSWORD` | — | Broker 1 password | -| `MQTT_BROKER_2_URL` | — | Broker 2 WebSocket URL | -| `MQTT_BROKER_2_USERNAME` | — | Broker 2 username | -| `MQTT_BROKER_2_PASSWORD` | — | Broker 2 password | -| `BEACON_API_KEY` | — | Admin bearer key; overrides `auth.api_key` when set | -| `LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error`; overrides `log.level` | -| `LOG_FORMAT` | `text` | `text` or `json`; overrides `log.format` | -| `BEACON_CPU_PROFILE_DIR` | — | Private CPU capture directory (see [PROFILING.md](PROFILING.md)) | -| `BEACON_CPU_PROFILE_UNTIL` | — | RFC 3339 deadline for CPU captures, at most 72h ahead | - -### Config file (`config.yaml`) - -```yaml -# MeshMapper integrations (optional, no API key). Import transport scopes and -# IATA border outlines from MeshMapper instead of maintaining scopes: and -# iatas.*.borderFile by hand. Both cover every known IATA that MeshMapper lists; -# no sources to configure. See config.yaml.example for limits and behaviour. -#meshmapper: -# scopes: -# enabled: true -# zones: -# enabled: true -# import_groups: true # MeshMapper zone groups become regions -# channels: -# enabled: true # import each region's public hashtag channels - -# Optional IATA overrides — auto-created on first packet arrival, -# only needed if you want to customise display name or coordinates. -# borderFile points to a GeoJSON Feature (Polygon or MultiPolygon) for the -# region border map feature; relative paths resolve against this config -# file's own directory. Validated at boot — invalid geometry fails startup. -iatas: - YVR: - name: Vancouver International - lat: 49.1967 - lng: -123.1815 - borderFile: borders/yvr.geojson # optional; or use meshmapper.zones - -# Super-regions grouping multiple IATAs. -regions: - - slug: western-canada - name: Western Canada - display_order: 1 - center_lat: 51.0 - center_lng: -114.0 - zoom_level: 5 - iatas: [YVR, YYJ, YYC, YEG] - -# Channel keys for decrypting group messages. -channel_keys: - # Hashtag channels: Beacon derives the PSK from the tag name automatically. - # secret = SHA256("#tag")[:16], channel_hash = SHA256(secret)[0] - # Tag names should be provided without the # prefix. Plain names show under - # every region; {name, region: } limits a channel to one region. - hashtags: - - meshcore - - name: vancouver-mesh - region: western-canada - - # Explicit keys: channel hash (hex) and key (hex), with optional display name. - # The public MeshCore channel key is included in config.yaml.example. - keys: - "11": - key: "8b3387e9c5cdea6ac9e5edbaa115cd72" - name: "Public" - -# Regional transport scopes for matching TRANSPORT_FLOOD packets. -# Plain names have # prepended automatically (e.g. "bc" → "#bc"). -# region (required) is a configured region slug; region-filtered scope lists -# and scope stats show the scope under that region's IATAs. Matching stays global. -# Optional when meshmapper.scopes covers your regions. -scopes: - - name: bc - region: western-canada - - name: "#west" - region: western-canada - -# Observer telemetry storage settings. -telemetry: - retention: 744h # how long to keep telemetry snapshots (default: 31 days) - resolution: 1h # snapshot frequency per observer; duplicates within window are dropped (default: 1h) - -# Packet and observation retention. -packets: - retention: 168h # packets, observations, channel messages (default: 7 days, minimum 24h) - -# Hourly analytics rollups behind the stats endpoints; they outlive raw packets. -analytics: - rollup_retention: 2160h # also the longest /stats/series window (default: 90 days, minimum 24h) - -# Presence write coalescing. -# Observer last_seen and packet last_heard_at bumps are batched in memory and -# flushed on an interval instead of writing one row per observation. An -# unclean shutdown loses at most one interval of presence freshness. -presence: - flush_interval: 30s # how often coalesced bumps are flushed (default: 30s) - packet_ttl: 30s # how long a quiet packet stays coalesced before writing through again (default: 30s) - -# Direct proxy peers allowed to set the client IP through X-Real-IP. -server: - trusted_proxies: [172.30.0.0/24] # default: [] (direct access) - -# Per-client REST limits for /api/v1/*. -ratelimit: - enabled: true # default: true - requests_per_minute: 300 # default: 300 - -# WebSocket settings. -websocket: - max_connections_per_ip: 5 # default: 5 - max_connects_per_minute: 10 # upgrade attempts per IP (default: 10) - allowed_origins: [https://beacon.example.com] # other sites allowed to open /ws (default: same host only) - -# Node staleness, deletion, and clock-drift thresholds. -nodes: - mark_foreign: false # optional indication for repeaters outside configured IATA borders - stale_threshold: 24h # mark a node "stale" in the API after this long unseen (default: 24h) - delete_after: 720h # delete a node entirely after this long unseen (default: 30 days) - clock_drift_threshold: 5m # |device clock - server clock| above which clockOutOfSync=true for a repeater/room server (default: 5m) - -# Optional observer age-out (disabled by default; set e.g. 720h to opt in). -# Enable only when one Beacon ingest process owns the database: presence-cache -# preparation is local to that process, including both of its MQTT workers. -# Deletes at most 1000 observers per background.cleanup interval, only when -# last_seen/last_status_at are old and no observations, telemetry or ownership remain. -# Broker/location/scope metadata cascades; a returning observer receives a new ID. -observers: - delete_after: 0s # omitted or nonpositive disables deletion - -# Redis caching layer (optional). -# Caches read-heavy, slow-changing responses to reduce PostgreSQL load. -# Connection details (address, password, database) are set via environment -# variables. Leave REDIS_ADDR unset to disable caching entirely. -# TTLs are duration strings e.g. "30m", "1h". Per-category TTLs override -# the global ttl. Any unset category inherits ttl. Default: 1h. -cache: - ttl: "1h" - ttls: - stats: "1h" # stats endpoints (hourly rollups; keys change when a new hour is rolled) - reference: "1h" # IATAs, regions, scopes - nodes: "1h" # node detail (also explicitly invalidated on upsert) - observers: "1h" # observer detail (also explicitly invalidated on upsert) - -# Geographic ingest filter (optional). -# Drop packets from observers outside the specified area. -# Country codes are ISO 3166-1 alpha-2. Continent codes: AF AN AS EU NA OC SA. -# If both are set an IATA passes if it matches either (OR semantics). -# Omit entirely to accept all IATAs (default). -ingest: - allow_countries: [CA, US] # only store packets from these countries - allow_continents: [NA] # or: accept all of North America -``` - -IATAs are auto-created on first packet arrival. The config file adds display -names, coordinates, and optional region borders. Regions and channel keys -must be defined here unless MeshMapper imports them (`meshmapper.zones.import_groups`, -`meshmapper.channels`). - -### Admin API - -The `/api/v1/admin` subtree requires `Authorization: Bearer `. Everything else, -including the WebSocket and CORS preflights, is public. - -- Set the key with `BEACON_API_KEY` or `auth.api_key`. A set environment variable wins; an - explicitly empty one disables admin access. No key is generated for you. -- The key must be at least 16 characters with no inner whitespace (surrounding whitespace is - trimmed). An unusable key fails startup. Changing it needs a restart. -- With no key, admin requests return 503. A missing, wrong or duplicated header returns 401 - with `WWW-Authenticate: Bearer`. After auth, unknown paths are 404 and unsupported methods - 405. -- Use a long random key, send it only in the header (never the URL or body), and keep it out - of source control and logs. Terminate HTTPS at the proxy and keep Beacon's listener - private to it. - -Endpoints: - -- **`GET /admin/config`** returns the CORS options with defaults applied, `auth.configured` - and `ingest.broker_count` (configured broker workers, not connection status). No - credentials, broker addresses, channel material or database settings. -- **`PUT /admin/config`** accepts only `{"cors":{"allowed_origins":[...]}}` and replaces the - origin list immediately. It is **runtime-only**: nothing is written, and a restart reloads - the file. The response carries `config`, `persisted: false` and `requires_restart: false`. - Send 1–32 ASCII http(s) origins of at most 512 bytes each, with an optional single - hostname wildcard; a lone `*` allows all. Empty or null lists, paths, queries, credentials, - control characters, unknown fields and bodies over 16 KiB are rejected. Concurrent updates - apply one at a time; requests already in flight may see the old list. -- **`GET/POST /admin/accounts`, `GET/DELETE /admin/accounts/{id}`** manage operator account - records (no login, session or token). POST takes `{"name": "..."}` (body ≤ 4 KiB; name - trimmed, case-sensitive, ≤ 128 characters, no control characters, unique among active - accounts). DELETE deactivates: 204, or 404 if missing, 409 if already inactive. The name - can then be reused. The list returns active and inactive accounts, newest first, - unpaginated. -- **`GET /admin/backup`** is opt-in; see [Backup and export](docs/backup-export.md). - -Browser admin clients need the matching methods in the saved `cors.allowed_methods`. The -default `GET, HEAD, OPTIONS` is read-only. For an admin UI, use -`[GET, HEAD, OPTIONS, POST, PUT, DELETE]`, restrict `cors.allowed_origins` to that UI and -allow the `Authorization` and `Content-Type` headers, or preflight blocks requests that work -from curl. CORS controls browser access, not authentication. - -### Reverse proxies and rate limits - -REST requests under `/api/v1` are rate limited per client IP (300/min by -default; IPv6 clients share a /64). Exhausted clients get `429` with -`rate_limited` and `Retry-After`. WebSocket upgrades are limited separately by -`websocket.max_connects_per_minute`. - -Beacon never trusts `X-Forwarded-For` or `True-Client-IP`. Behind a reverse -proxy, list the proxy's address in `server.trusted_proxies` (CIDR, e.g. -`127.0.0.1/32` or the Docker network subnet) and have the proxy overwrite -`X-Real-IP` with the connecting client's address: - -```nginx -proxy_set_header X-Real-IP $remote_addr; -``` - -```caddy -reverse_proxy app:8080 { - header_up X-Real-IP {remote_host} -} -``` - -Without this every visitor shares the proxy's budget and the site gets 429s -under normal load. Beacon logs a warning at startup when limits are on and no -proxy is trusted, and once at runtime when a request carries forwarding headers -it is ignoring. Full nginx and Caddy examples are in -[beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs). - -### Analytics retention - -Historical stats read hourly rollups kept for `analytics.rollup_retention` (default 90 -days), independently of `packets.retention`. Each UTC hour is rolled about 95 minutes after -it closes, and packet cleanup holds back up to 24h of raw rows until their hours are rolled. -An hour that lost raw rows first is reported as partial, without values. Drill-downs, -sub-hour observer activity and observer comparison still read raw data. Details: -[docs/historical-stats.md](docs/historical-stats.md). - -### Logging - -Logs go to stderr; collect and rotate them with Docker or systemd. Set `log.level` -(`debug`, `info`, `warn`, `error`; default `info`) and `log.format` (`text` or `json`) in -`config.yaml`, or override with `LOG_LEVEL` / `LOG_FORMAT`. Invalid values fail startup. - -Every record has a `component`. Ingest records add the broker name; HTTP completion records -add the validated client address, route, status and duration. Query strings and WebSocket -hello payloads are never logged. Expected ingest skips and routine WebSocket lifecycle -events are debug-level. Reverse-proxy access logs (and fail2ban rules on them) are -unaffected. - -### Foreign repeater indication - -Set `nodes.mark_foreign: true` to add `possiblyForeign` to repeater nodes. It is `true` when -the repeater's reported position is outside the union of all configured -`iatas..borderFile` polygons, with `meshmapper.zones` borders replacing them where -MeshMapper has one. IATAs without a border add no area, and airport coordinates aren't used. -Enabling it with no border source, a missing file or invalid geometry fails startup. Border -changes need a restart. - -- Points on an edge (including hole edges) are inside; hole interiors are outside. -- Other roles and missing or invalid positions get no value. A 0/0 advert clears the stored - position. -- It's a hint from the reported position, not proof of origin. Ingest, heard-in IATAs and - route matching are unaffected. -- Computed at read time, so existing nodes need no backfill. The field is omitted when the - feature is off (the default). -- In `nodeUpdate`, it's a boolean when the advert carries a position, `null` when the - position is 0/0 or invalid or the node stops being a repeater, and omitted to keep the - previous value. `lat`/`lng` follow the same rule. - -GeoJSON uses longitude/latitude order. Split antimeridian-crossing borders into -MultiPolygons per [RFC 7946 §3.1.9](https://www.rfc-editor.org/rfc/rfc7946#section-3.1.9); -edges spanning more than 180° are rejected. - ---- - -## WebSocket API - -Connect to `ws://host:8080/ws`. - -On connect the server sends a `hello`: - -```json -{ "v": 1, "type": "hello", "serverTime": 1234567890000, "connectionId": "uuid" } -``` - -The connection closes after 90 seconds of inactivity. Clients should send a -`ping` every 30 seconds. - -### Client → Server messages - -**Subscribe** — add a filter to this connection. Multiple subscriptions are -unioned (OR semantics): an event matches if it satisfies any active -subscription. The server replies with a `subscriptionId` to use for -unsubscribing. - -```json -{ - "v": 1, - "type": "subscribe", - "id": "sub-1", - "scope": { - "iatas": ["YOW", "YYZ"], - "regionIds": ["1"], - "regionSlugs": ["western-canada"], - "payloadTypes": [4, 5], - "routeTypes": [1, 2], - "channelHashes": ["11"], - "observerIds": [""], - "events": ["packetObservation", "channelMessage"] - } -} -``` - -All scope fields are optional; omitted or empty means no filter on that -dimension. `routeTypes` and `observerIds` filter `packetObservation` events -(`observerIds` also `observerStatus`), and `channelHashes` only `channelMessage`. `regionIds` and -`regionSlugs` are both expanded to their member IATAs server-side. - -**Unsubscribe** — remove a specific subscription by ID. - -```json -{ - "v": 1, - "type": "unsubscribe", - "id": "unsub-1", - "subscriptionId": "" -} -``` - -The server replies `{ "v": 1, "type": "unsubscribed", "id": "unsub-1", "subscriptionId": "..." }`. - -**Configure** — connection-wide flags, all default `false`: - -- `resolvePath` adds per-hop node resolution to `packetObservation` events (see below). -- `includeObserverKey` adds `observation.observerPublicKey`. -- `includeRepeats` also streams later hearings of an already-stored observation - over a new path, as `packetObservation` events with `packet.isRepeat: true` - and `observationCount: 0`. - -Unlike `subscribe`, this is not additive: each `configure` sets all three flags -to exactly the values sent, so an omitted flag turns off. - -```json -{ "v": 1, "type": "configure", "id": "cfg-1", "resolvePath": true, "includeRepeats": true } -``` - -The server replies: - -```json -{ "v": 1, "type": "configured", "id": "cfg-1", "resolvePath": true, "includeObserverKey": false, "includeRepeats": true } -``` - -**Ping** - -```json -{ "v": 1, "type": "ping", "id": "ping-1" } -``` - -### Server → Client events - -| Type | Description | -| ------------------- | --------------------------------------------------- | -| `packetObservation` | New observation written to DB | -| `observerStatus` | Observer status update | -| `nodeUpdate` | Node upserted from advert | -| `channelMessage` | Decrypted channel message (scope must include hash) | - -When `resolvePath` is enabled via `configure`, `packetObservation` events -include a `resolvedPath` array: one entry per hop in the packet's path, each -with a `confidence` (`"high"` for exactly one candidate node, `"ambiguous"` -for multiple, `"none"` for zero) and the matching node(s)' id, name, -lat/lng, and public key. Without `resolvePath` enabled, `resolvedPath` is -`null` — this keeps the default event payload small; only opt in if you're -actually rendering path connections (e.g. drawing hops on a map). - -### Backpressure - -The server write buffer per connection is bounded at 256 events. If a client -falls behind, the server drops the oldest queued events and sends a `lagged` -notice: - -```json -{ "v": 1, "type": "lagged", "droppedCount": 12, "since": 1234567890000 } -``` - -Clients should respond by backfilling over REST, then resume streaming: -`/api/v1/packets/backfill?afterObservationId=` and -`/api/v1/messages/backfill?afterId=`. - -### Reconnection - -Subscriptions are not persisted — they exist only for the lifetime of the -connection. On any disconnect the client should reconnect with backoff, re-issue -all subscriptions, and backfill via the same REST endpoints. - -### Connection limits - -- At most `websocket.max_connections_per_ip` (default 5) concurrent connections - per IP. A connection over the cap is accepted and then closed with code - `1013` (try again later) before `hello`. -- At most `websocket.max_connects_per_minute` (default 10) upgrade attempts per - IP; beyond that the handshake gets `429` with `Retry-After: 60`. -- Browsers may only connect from the same host unless their origin is listed in - `websocket.allowed_origins` (`https://*.example.com` covers every subdomain). - -Behind a reverse proxy both limits need `server.trusted_proxies`, or every -visitor counts as the proxy's IP. - ---- - -## REST API - -Base path: `/api/v1` +Fill in `POSTGRES_DSN` and your MQTT broker credentials in `.env`. Every variable is described +in [Configuration](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/configuration.md), +and the fully annotated `config.yaml` lives in beacon-docs as well; the copy here is a working +starter. Migrations run on startup. The API listens on `LISTEN_ADDR` (default `:8080`) and +Swagger is at `http://localhost:8080/swagger/index.html`. -All list endpoints support cursor-based pagination via `cursor` and `limit` -query params. See the Swagger UI at `http://localhost:8080/swagger/index.html` -for full parameter documentation. +To run the web frontend against it, see +[Running the full stack locally](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/CONTRIBUTING.md#running-the-full-stack-locally). -### Authentication and limits +The published image is `ghcr.io/meshcore-beacon/beacon-server`. `latest` tracks stable +releases, `dev` follows the development branch, and each release is also tagged `X.Y.Z` and +`X.Y`. -Only `/admin/*` needs `Authorization: Bearer `. All `/api/v1` routes are -rate limited per client IP; see [Reverse proxies and rate limits](#reverse-proxies-and-rate-limits). -List `limit` values are clamped to 1–200. +### What you see on an empty database -### Endpoints +Path resolution, capability detection and known routes depend on nodes having advertised to a +local observer. On a fresh database every hop shows `"confidence": "none"` and +`supportsMultibytePaths` is `false` until adverts populate `node_short_ids`. That is expected +and fills in as the mesh is observed. -| Method | Path | Description | -| -------- | --------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| `GET` | `/admin/accounts` | List operator accounts (bearer key) | -| `POST` | `/admin/accounts` | Create an operator account (bearer key) | -| `GET` | `/admin/accounts/{id}` | Get an operator account (bearer key) | -| `DELETE` | `/admin/accounts/{id}` | Deactivate an operator account (bearer key) | -| `GET` | `/admin/backup` | Download a private database and saved-config backup (bearer key, opt-in) | -| `GET` | `/admin/config` | Inspect selected running configuration (bearer key) | -| `PUT` | `/admin/config` | Replace runtime CORS origins (bearer key) | -| `GET` | `/brokers` | List MQTT brokers and connection status | -| `GET` | `/channels` | List channels (`hash`, `iata`/`iatas`, `keyKnown`, `pageCursor`, `limit`) | -| `GET` | `/channels/{channelID}` | Get channel detail by integer ID | -| `GET` | `/channels/{channelID}/messages` | List messages for a channel (`since`, `iatas`, `region`/`regionId`, `scope`, `cursor`, `limit`) | -| `GET` | `/iatas` | List all known IATA codes | -| `GET` | `/iatas/{iata}` | Get a single IATA code | -| `GET` | `/iatas/{iata}/border` | Get an IATA's GeoJSON border, if any (204 if not) | -| `GET` | `/messages` | List channel messages (`channelID`, `channelHash`, `since`, `iatas`, `region`/`regionId`, `scope`, `cursor`, `limit`) | -| `GET` | `/messages/backfill` | Messages after `afterId` | -| `GET` | `/nodes` | List nodes (type, IATA/region, name, scope, `pubkey`/`pubkeyPrefix`, capability flags, `neighbors`) | -| `GET` | `/nodes/{nodeId}` | Get node detail | -| `GET` | `/nodes/{nodeId}/neighbors` | List neighboring nodes observed in the mesh | -| `GET` | `/nodes/{nodeId}/observations` | List observations of packets from a node | -| `GET` | `/observers` | List observers (IATA/region, `type`, `broker`, `status=online\|offline`, `name`, `scope`) | -| `GET` | `/observers/{observerId}` | Get observer detail including broker last-seen timestamps | -| `GET` | `/observers/{observerId}/activity` | Heard-activity history (`range`, `interval`, `until`) | -| `GET` | `/observers/{observerId}/adverts` | Adverts heard by observer | -| `GET` | `/observers/{observerId}/telemetry` | Telemetry history (`range`, `interval=1h\|6h\|24h`, `afterId`) | -| `GET` | `/packets` | List packets (payload/route type, IATA/region, scope, `since`/`until`; plural params take CSV) | -| `GET` | `/packets/backfill` | Packets after `afterObservationId` | -| `GET` | `/packets/{packetHash}` | Get packet with all observations | -| `GET` | `/regions` | List all regions (summary) | -| `GET` | `/regions/{regionId}` | Get a single region with IATA list | -| `GET` | `/routes` | List known routes (`iata`, `hopCount`, `cursor`+`cursorId`) | -| `GET` | `/routes/search` | Search routes by source and destination hash | -| `GET` | `/routes/cross` | Search for routes crossing IATA boundaries | -| `GET` | `/routes/{iata}/{pathKey}/observations` | Retained observations matching a saved route (see below) | -| `GET` | `/scopes` | List transport scope names; IATA/region filters use configured regions and MeshMapper catalogues | -| `GET` | `/scopes/{name}` | Get scope detail | -| `GET` | `/stats/clock-drift` | Repeaters and room servers whose clock drifted past the threshold, worst first | -| `GET` | `/stats/node-types` | Node type breakdown | -| `GET` | `/stats/observations` | Hourly observation counts per IATA (last 7 days by default) | -| `GET` | `/stats/observer-comparison` | Compare flood packets reported by two observers | -| `GET` | `/stats/overview` | Network overview over the last 24 rolled hours | -| `GET` | `/stats/paths` | Path-entry and hash-width distributions | -| `GET` | `/stats/payload-breakdown` | Observation counts by payload type (last 24h by default) | -| `GET` | `/stats/radio-presets` | Radio preset usage by IATA | -| `GET` | `/stats/scopes` | Region scopes with hourly packet, node and observer counts (last 7 days by default) | -| `GET` | `/stats/series` | Hourly card metrics and window summary (`since`/`until` required) | -| `GET` | `/stats/signal` | Reception signal distributions and hourly trends | -| `GET` | `/stats/top-advertisers` | Top N nodes by distinct ADVERT packets, split flood/direct (last 24h by default) | -| `GET` | `/stats/top-nodes` | Top N nodes by advert hearings (last 7 days by default) | -| `GET` | `/stats/top-observers` | Top N observers by observation count (last 24h by default) | -| `GET` | `/stats/top-talkers` | Top N companion names by decrypted channel message count (last 24h by default) | -| `GET` | `/traces` | List trace tags (`type=TRACE\|PING`, IATA/region, scope, `since`/`until`, `cursor`+`cursorTag`) | -| `GET` | `/traces/{tag}` | Get full trace detail with resolved routes | +GRP_TXT packets whose channel key is not known yet are stored hash-only. After adding the key to +`config.yaml`, the next restart decrypts the matching history; look for +`config: backfilled N previously-undecrypted channel message(s)` in the log. -### Observer activity +## Documentation -`GET /observers/{observerId}/activity` returns buckets over `[windowStart, windowEnd)` (live -requests include the current partial bucket), plus `generatedAt`, `source` (`raw` or -`hourly`) and `summary`. +In [beacon-docs](https://github.com/MeshCore-Beacon/beacon-docs): -- `recordedPackets` counts stored observations in the window; repeated broker deliveries of - the same packet count once. Unknown payload types show as `-1`. -- Freshness fields (`lastCompleteHour` with its start and end, `latestRecordedAt`) are - measured at `generatedAt`, even for historical `until` requests. -- `until` (epoch ms, within the last 30 days) lines up two observers' charts. -- Hourly mode reads the rollups plus raw rows for hours not rolled yet (at most 24h). - `rolledUntil` ends the rolled range and `rawFrom` starts the raw tail; hours between them - are uncovered. -- Missing records don't prove downtime. The observer's `observationCount` is a legacy - cumulative presence counter (including status and neighbour events), not a packet total. +- [Deploy with Docker](https://github.com/MeshCore-Beacon/beacon-docs#deploy-the-all-in-one-stack) +- [Getting packets in](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/getting-packets-in.md): brokers, the subscriber account, topics +- [Configuration](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/configuration.md): environment variables and `config.yaml` +- [API contract](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/api-contract.md): REST, WebSocket, admin endpoints +- [Reverse proxy and rate limits](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/reverse-proxy.md) +- [Upgrading](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/upgrading.md) +- [Operations](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/operations.md), [backup and export](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/backup-export.md) and [CPU profiling](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/profiling.md) +- [High level design](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/high-level-design.md) +- [Releases and versioning](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/releases.md) -### Saved-route evidence +In this repo: [Historical stats](docs/historical-stats.md) explains the hourly rollups behind +the stats endpoints, for anyone changing them. `docs/swagger.yaml` is the generated OpenAPI +description. -`GET /routes/{iata}/{pathKey}/observations` returns retained observations matching a saved -route's full path bytes, hash size and hop count within its IATA. `range` defaults to `24h` -(max `720h`) and `limit` to 50 (max 200); page with `nextPageCursor` → `pageCursor`. -Evidence expires with raw packets. +## Contributing ---- +[CONTRIBUTING.md](CONTRIBUTING.md) covers code style, tests, and database and API changes. +Branches, commits and the release flow are shared across the Beacon repos and live in +[beacon-docs/CONTRIBUTING.md](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/CONTRIBUTING.md). +Please read the [Code of Conduct](CODE_OF_CONDUCT.md). Security reports go through +[SECURITY.md](SECURITY.md). ## Acknowledgements diff --git a/SECURITY.md b/SECURITY.md index 397b3071..12a5e817 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -5,7 +5,7 @@ Please do not report security vulnerabilities through public GitHub issues. Instead, contact the maintainers directly via the MeshCore Canada Discord -server: [MeshCore Canada Discord](https://discord.gg/Gz3KvJx2hf) — reach out to +server: [MeshCore Canada Discord](https://discord.gg/Gz3KvJx2hf); reach out to **dedskelly** directly. Include as much detail as possible: the nature of the issue, steps to reproduce, and any potential impact. diff --git a/SHOULDERS.md b/SHOULDERS.md index ad3b0ec4..51a98568 100644 --- a/SHOULDERS.md +++ b/SHOULDERS.md @@ -5,48 +5,48 @@ projects that make this possible: ## Core dependencies -- [meshcore-go](https://github.com/meshcore-go/meshcore-go) — MeshCore packet +- [meshcore-go](https://github.com/meshcore-go/meshcore-go): MeshCore packet decoding -- [paho.mqtt.golang](https://github.com/eclipse/paho.mqtt.golang) — MQTT client -- [pgx](https://github.com/jackc/pgx) — PostgreSQL driver and connection pool -- [sqlc](https://sqlc.dev) — Type-safe SQL code generation -- [chi](https://github.com/go-chi/chi) — HTTP router -- [go-chi/cors](https://github.com/go-chi/cors) — CORS middleware for chi -- [go-chi/httprate](https://github.com/go-chi/httprate) — Per-client HTTP +- [paho.mqtt.golang](https://github.com/eclipse/paho.mqtt.golang): MQTT client +- [pgx](https://github.com/jackc/pgx): PostgreSQL driver and connection pool +- [sqlc](https://sqlc.dev): Type-safe SQL code generation +- [chi](https://github.com/go-chi/chi): HTTP router +- [go-chi/cors](https://github.com/go-chi/cors): CORS middleware for chi +- [go-chi/httprate](https://github.com/go-chi/httprate): Per-client HTTP sliding-window rate limits and retry headers -- [go-redis](https://github.com/redis/go-redis) — Redis client -- [coder/websocket](https://github.com/coder/websocket) — WebSocket +- [go-redis](https://github.com/redis/go-redis): Redis client +- [coder/websocket](https://github.com/coder/websocket): WebSocket implementation -- [swaggo/swag](https://github.com/swaggo/swag) — OpenAPI documentation +- [swaggo/swag](https://github.com/swaggo/swag): OpenAPI documentation generation -- [godotenv](https://github.com/joho/godotenv) — .env file loading -- [yaml.v3](https://github.com/go-yaml/yaml) — YAML config parsing -- [google/uuid](https://github.com/google/uuid) — UUID generation -- [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync) — singleflight for +- [godotenv](https://github.com/joho/godotenv): .env file loading +- [yaml.v3](https://github.com/go-yaml/yaml): YAML config parsing +- [google/uuid](https://github.com/google/uuid): UUID generation +- [golang.org/x/sync](https://pkg.go.dev/golang.org/x/sync): singleflight for collapsing concurrent cache misses -- [paulmach/orb](https://github.com/paulmach/orb) — GeoJSON border geometry and +- [paulmach/orb](https://github.com/paulmach/orb): GeoJSON border geometry and point-in-polygon checks -- [swaggo/http-swagger](https://github.com/swaggo/http-swagger) — Swagger UI +- [swaggo/http-swagger](https://github.com/swaggo/http-swagger): Swagger UI handler ## Testing -- [go.uber.org/mock](https://github.com/uber-go/mock) — gomock mocks for the +- [go.uber.org/mock](https://github.com/uber-go/mock): gomock mocks for the generated queries -- [miniredis](https://github.com/alicebob/miniredis) — In-memory Redis for cache +- [miniredis](https://github.com/alicebob/miniredis): In-memory Redis for cache tests ## Data -- [OurAirports](https://ourairports.com/data/) — IATA airport geographic data +- [OurAirports](https://ourairports.com/data/): IATA airport geographic data (CC0 public domain) ## Infrastructure -- [PostgreSQL](https://www.postgresql.org) — Primary data store -- [Redis](https://redis.io) — Optional caching layer -- [Caddy](https://caddyserver.com) — Reverse proxy +- [PostgreSQL](https://www.postgresql.org): Primary data store +- [Redis](https://redis.io): Optional caching layer +- [Caddy](https://caddyserver.com): Reverse proxy ## Language -- [Go](https://go.dev) — The language that makes this all possible +- [Go](https://go.dev): The language that makes this all possible diff --git a/config.yaml.example b/config.yaml.example index d68edb8a..6044840e 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -1,5 +1,6 @@ -# Beacon configuration file -# Copy to config.yaml and adjust as needed. +# Beacon configuration file: local development starter. Copy to config.yaml and adjust. +# The fully annotated reference, with every key and its default, is +# https://github.com/MeshCore-Beacon/beacon-docs/blob/main/app_config/config.yaml.example # Durations are Go strings (30s, 1h, 168h); negative values prevent startup # (observers.delete_after excepted, where nonpositive disables it). @@ -263,7 +264,7 @@ websocket: # CORS configuration. # Controls which origins, methods and headers are allowed for cross-origin requests. -# Defaults to allowing all origins with GET/HEAD/OPTIONS if omitted — appropriate +# Defaults to allowing all origins with GET/HEAD/OPTIONS if omitted, appropriate # for a public read-only API. Restrict allowed_origins if you expose write endpoints. #cors: # allowed_origins: diff --git a/docs/backup-export.md b/docs/backup-export.md deleted file mode 100644 index d7d24dac..00000000 --- a/docs/backup-export.md +++ /dev/null @@ -1,136 +0,0 @@ -# Backup and export - -Beacon exports its PostgreSQL database and saved `config.yaml` as a private, versioned -`.tar.gz`, either with the standalone `beacon-backup` command or an opt-in admin download. -`beacon-backup -verify` checks an archive offline. There is no import, scheduling or remote -storage. - -## Archive format 1 - -Exactly three regular files: - -- `manifest.json`: format and exporter version, creation time, payload sizes and SHA-256 - hashes, and the exclusions below. -- `database.sql`: one consistent `pg_dump` snapshot (schema, data, migration journal), - without ownership, ACLs or tablespaces so it restores under any role. -- `config.yaml`: the saved file byte-for-byte, read just before the snapshot. Avoid config - edits during an export. - -**It is sensitive and unencrypted** (keys, message data); keep it private. It is not a full -recovery package: `.env` and environment variables, `borderFile` and TLS files, runtime-only -changes, PostgreSQL roles and cluster settings, Redis and deployment files are not included. - -## Requirements and limits - -- `pg_dump` must be in the **same runtime** as the exporter (the CLI's environment, or the - server's PATH for the download), with a major version matching the database. A client on - the Docker host or in another container doesn't count. -- The Docker image ships the PostgreSQL 16 client. Its Alpine 3.19 base has no 17/18 - packages, so a newer database needs a different runtime, not just `POSTGRES_CLIENT_MAJOR`. -- The temp directory (`TMPDIR`) needs about twice the SQL limit free. A small tmpfs won't do. -- The output filesystem must support hard links (used to publish atomically). Existing - destinations, symlinks included, are never replaced. Files are 0600 and staging dirs 0700; - on Windows, restrict the directory ACLs yourself. - -| Limit | Default | -| --- | --- | -| Timeout | 10 min (`-timeout`) | -| Uncompressed SQL | 1 GiB (`-max-bytes`, up to 1 TiB) | -| Saved YAML / manifest | 1 MiB / 64 KiB | -| Compressed archive (verify) | SQL limit + 1% + 2 MiB | -| Lock wait | 5 s | - -Failures, timeouts and interrupts clean up and publish nothing. A SIGKILL or power loss can -leave a private staging dir: `.beacon-backup-*` next to the CLI output, or -`beacon-download-*` in the server's temp dir, which Beacon sweeps on its next start. - -## CLI export - -```sh -go build ./cmd/beacon-backup -beacon-backup -config /private/config.yaml -output /private/beacon-20260913.tar.gz -``` - -Connection comes only from the standard libpq environment (`PGHOST`, `PGPORT`, -`PGDATABASE`, `PGUSER`, TLS settings); `PGDATABASE` is required. Keep the password in a 0600 -`PGPASSFILE` or a service file. The command ignores `.env` and `POSTGRES_DSN`, runs no -migrations, and never prints `pg_dump` stderr since it can leak private data. If it fails, -check the client version, privileges, connection settings and free space. - -## Download API - -`GET /api/v1/admin/backup` streams a fresh archive. Enable it with `backup.enabled: true` and -the admin bearer key, then restart. Send the key only in the `Authorization` header over -HTTPS; the endpoint takes no query or body. Leave it off on public previews. - -At startup Beacon checks `pg_dump --version` against the server. If that or another -prerequisite fails, it logs why and disables only backup; fix it and restart. - -**Connection.** The download uses `POSTGRES_DSN`, which must be a single-host `postgres://` -URL with an explicit user and database. TLS options, `passfile`, `connect_timeout`, -`application_name`, `target_session_attrs` and `options` are passed through; `pool_*` -options are dropped; supported `PG*` env defaults apply under the URL. Keyword DSNs, -services, multiple hosts, unknown or duplicate options, protocol-negotiation settings and -the `PGSERVICE`, `PGSERVICEFILE`, `PGSSLNEGOTIATION`, `PG{MIN,MAX}PROTOCOLVERSION` and `PGTZ` -env vars are rejected. Use the CLI for those setups. Prefer `sslmode=verify-full` for remote -databases. Credentials go to a 0600 service file in staging, never into process arguments, -logs or errors. - -**Behaviour.** One export at a time. Disconnecting cancels it, and the transfer has a -10-minute write deadline. Responses are `no-store`. A failed export sends no partial archive -and logs a sanitized reason. - -| Status | Meaning | -| --- | --- | -| 400 | Query string or body sent | -| 401 | Missing or invalid key | -| 409 | Another export is running | -| 500 | Export failed | -| 503 | Backup unavailable, no admin key, or shutting down | -| 504 | Timed out | -| 507 | SQL size limit exceeded | - -## Verify - -```sh -beacon-backup -verify /private/beacon-20260913.tar.gz -``` - -Reads the file without extracting, running SQL or connecting to anything, so it needs no -config, `pg_dump` or credentials. `-max-bytes` and `-timeout` apply. It requires exactly the -three members as 0600 regular files with USTAR headers (size-only PAX allowed for SQL over -8 GiB). It rejects duplicates, links, devices, extra metadata, truncated or concatenated -streams, and unknown or duplicate manifest keys. Sizes and SHA-256 hashes must match. - -Exit 0 means structure and checksums passed. Errors don't include payload, manifest values -or the path. The manifest is unsigned, so a pass does **not** prove authenticity, safe SQL or -that the dump restores. - -## Restore - -Only restore trusted bundles, because a dump can run arbitrary SQL. Verify, extract -privately, and load into a **new, empty database** on a compatible PostgreSQL with the -needed extensions: - -```sh -psql -X --set ON_ERROR_STOP=on --single-transaction --file database.sql -``` - -Check the migration journal and some records, review the config, and restore external -secrets and files before pointing a new Beacon at it. Never test an export by restoring over -live data. See [pg_dump](https://www.postgresql.org/docs/16/app-pgdump.html) and -[libpq environment](https://www.postgresql.org/docs/16/libpq-envars.html). - -## Testing - -CI runs export, restore round-trip, offline verify and truncated-archive rejection against -PostgreSQL 16. Locally, with `PG*` pointing at a throwaway server whose role can create -databases: - -```sh -go build -o "$PWD/beacon-backup" ./cmd/beacon-backup -BEACON_BACKUP_TEST_POSTGRES=1 BEACON_BACKUP_TEST_BINARY="$PWD/beacon-backup" \ - go test ./internal/backup -run '^TestExportPostgres$' -v -``` - -The test creates, migrates, restores and drops two randomly named databases. diff --git a/docs/packet-summaries.md b/docs/packet-summaries.md deleted file mode 100644 index 34c7ccfd..00000000 --- a/docs/packet-summaries.md +++ /dev/null @@ -1,22 +0,0 @@ -# Packet summaries - -Packet lists, regional lists, reconnect backfill and live `packetObservation` events carry -an optional `summary` string. It comes from the packet's own stored metadata, never from a -node's current name or a fresh decode. - -| Payload | Summary | Source | -|---|---|---| -| ADVERT | Advertised name | Saved `appData.name` / verified decoded advert | -| ACK | `ACK 01020304` | Four-byte acknowledgement checksum | -| TRACE | `TRACE efbeadde` | Trace tag | -| TRACE classified as PING | `PING efbeadde` | Trace tag | - -- References are eight lowercase hex characters in the same byte order as packet detail's - `checksum` / `traceTag`. Zero is valid. Extended ACK retry bytes don't change the checksum. -- They are short references, not unique packet IDs or proof of delivery. No message body, - trace auth code or inferred sender/recipient is included. -- The text is projected from saved JSON in the existing list queries: no extra column, - backfill or per-packet lookup. A stored value of the wrong type or not exactly eight hex - characters omits the field. -- Other payload types have no summary yet. A missing summary doesn't mean the packet is - invalid. diff --git a/env.example b/env.example index f2d930d4..c7a152ef 100644 --- a/env.example +++ b/env.example @@ -1,3 +1,6 @@ +# Local development starter. Every variable is documented at +# https://github.com/MeshCore-Beacon/beacon-docs/blob/main/docs/configuration.md + LISTEN_ADDR=:8080 # Path to the YAML config (default: config.yaml). @@ -13,7 +16,7 @@ LISTEN_ADDR=:8080 POSTGRES_DSN=postgres://beacon:beacon@localhost:5432/beacon?sslmode=disable -# Redis (optional — leave REDIS_ADDR unset to disable caching) +# Redis (optional; leave REDIS_ADDR unset to disable caching) REDIS_ADDR=localhost:6379 REDIS_PASSWORD= REDIS_DB=0 diff --git a/internal/iatadb/gen/airports.csv b/internal/iatadb/gen/airports.csv index 561e33a3..2070812f 100644 --- a/internal/iatadb/gen/airports.csv +++ b/internal/iatadb/gen/airports.csv @@ -29945,7 +29945,7 @@ 599256,"GB-1333","heliport","Clifton Dykes",54.644211,-2.69254,,"EU","GB","GB-U-A",,"no",,,,,,, 599308,"GB-1334","closed","RAF Long Kesh",54.490669,-6.103835,115,"EU","GB","GB-NIR","Maze, County Down","no",,,,,,"https://en.wikipedia.org/wiki/RAF_Long_Kesh", 599309,"GB-1335","heliport","Lisburn Heliport",54.49328,-6.097745,118,"EU","GB","GB-NIR","Lisburn, County Down","no",,,,,,, -599921,"GB-1336","small_airport","(Duplicete)East Kirkby Airfield",53.13889,-0.00056,39,"EU","GB","GB-ENG","East Kirkby","no",,,,,"https://lincsaviation.co.uk","https://en.m.wikipedia.org/wiki/Lincolnshire_Aviation_Heritage_Centre?utm_source=chatgpt.com","RAF East Kirkby; Lincolnshire Aviation Heritage Centre; Bomber Command; Avro Lancaster NX611 “Just Jane”; Mosquito HJ711; East Lindsey; WW2 airfield; museum; heritage." +599921,"GB-1336","small_airport","(Duplicete)East Kirkby Airfield",53.13889,-0.00056,39,"EU","GB","GB-ENG","East Kirkby","no",,,,,"https://lincsaviation.co.uk","https://en.m.wikipedia.org/wiki/Lincolnshire_Aviation_Heritage_Centre","RAF East Kirkby; Lincolnshire Aviation Heritage Centre; Bomber Command; Avro Lancaster NX611 “Just Jane”; Mosquito HJ711; East Lindsey; WW2 airfield; museum; heritage." 599951,"GB-1337","closed","RAF Woodhall Spa",53.131667,-0.18444,33,"EU","GB","GB-ENG","East Linsdey, Lincolnshire","no",,,,,"https://www.lincstrust.org.uk/get-involved/top-reserves/woodhall-spa-airfield","https://en.wikipedia.org/wiki/RAF_Woodhall_Spa","Tattershall Thorpe, No. 617 Squadron, Dambusters, Bristol Bloodhound Missile site, Thorpe Camp Visitor Centre, Lincolnshire Wildlife Trust, RAF Coningsby satellite" 600636,"GB-1338","heliport","Chequers Helicopter Landing Ground",51.742163,-0.781822,,"EU","GB","GB-ENG","Ellesborough, Bucks.","no",,,,,,, 602208,"GB-1339","heliport","Uxbridge Business Park Heliport",51.554853,-0.483221,112,"EU","GB","GB-ENG","Uxbridge, Greater London","no",,,,,,, @@ -30972,7 +30972,7 @@ 30801,"HRYI","small_airport","Butare Airport",-2.59583,29.7367,5801,"AF","RW","RW-05","Butare","no","HRYI","BTQ","HRYI",,,"https://en.wikipedia.org/wiki/Butare_Airport", 31612,"HRYN","small_airport","Nemba Airport",-2.336167,30.206658,4905,"AF","RW","RW-02","Nemba","no","HRYN",,"HRYN",,,"https://en.wikipedia.org/wiki/Nemba_Airport", 31613,"HRYO","small_airport","Gabiro Airport",-1.5499999523162842,30.382999420166016,4806,"AF","RW","RW-02","Gabiro","no",,,"HRYO",,,, -3237,"HRYR","large_airport","Kigali International Airport",-1.96863,30.1395,4859,"AF","RW","RW-01","Kigali","yes","HRYR","KGL","HRYR",,"https://www.rac.co.rw/?utm_source=chatgpt.com","https://en.wikipedia.org/wiki/Kigali_International_Airport","Gregoire Kayibanda International Airport, Kanombe International Airport" +3237,"HRYR","large_airport","Kigali International Airport",-1.96863,30.1395,4859,"AF","RW","RW-01","Kigali","yes","HRYR","KGL","HRYR",,"https://www.rac.co.rw/","https://en.wikipedia.org/wiki/Kigali_International_Airport","Gregoire Kayibanda International Airport, Kanombe International Airport" 32208,"HRYU","small_airport","Ruhengeri Airport",-1.496873,29.631343,6102,"AF","RW","RW-03","Ruhengeri","no","HRYU","RHG","HRYU",,,"https://en.wikipedia.org/wiki/Ruhengeri_Airport", 3238,"HRZA","medium_airport","Kamembe Airport",-2.462239980697632,28.907899856567383,5192,"AF","RW","RW-04","Kamembe","yes","HRZA","KME","HRZA",,,"https://en.wikipedia.org/wiki/Kamembe_Airport", 43976,"HSAK","small_airport","Akobo Airport",7.7779,33.0027,1352,"AF","SS","SS-JG","Akobo","no",,,"HJAK",,,"https://en.wikipedia.org/wiki/Akobo_Airport","HSAK, Ahobo"