From 70c5c389f9886cb8d8c9d9d34462c805cad6fc6b Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 03:46:29 -0400 Subject: [PATCH 1/9] feat(db): retain hourly analytics after raw packet expiry --- .github/workflows/ci.yml | 2 +- README.md | 15 + db/analytics_retention_integration_test.go | 261 +++++++++ db/migrations/039_analytics_retention.sql | 611 +++++++++++++++++++++ db/queries/queries.sql | 15 +- db/retention_integration_test.go | 5 +- db/sqlc/models.go | 158 ++++++ db/sqlc/querier.go | 2 +- db/sqlc/queries.sql.go | 24 +- sqlc.yaml | 8 + 10 files changed, 1067 insertions(+), 34 deletions(-) create mode 100644 db/analytics_retention_integration_test.go create mode 100644 db/migrations/039_analytics_retention.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beb7284..7d37492 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index 5418223..5243068 100644 --- a/README.md +++ b/README.md @@ -533,3 +533,18 @@ log: `LOG_LEVEL` and `LOG_FORMAT` override file settings; empty settings use `info` and `text`. Invalid values prevent startup. Configuration-loading failures can use the bootstrap text logger before file settings are available; failures after initialization retain error severity at every supported level. Records include a component field. Ingest workers also include their broker name, and HTTP completion records include the validated client address, route, status and duration. Query strings and protocol hello payloads are excluded. Expected ingest skips and routine WebSocket lifecycle details are debug-level. Changing the application's format does not change Caddy/Apache access logs or their fail2ban configuration. Collect/rotate stderr through Docker or systemd. + +### Analytics retention + +Hourly traffic, payload, observer activity, talker, advertiser, Signal and Paths +summaries retain 30 days independently of `packets.retention`. Cleanup saves only +aggregates before deleting each packet batch, in the same transaction. Raw +packets, observations and message bodies still expire under packet retention. +The summaries use UTC hourly buckets and appear on the normal view-refresh cycle. + +Migration 039 starts from data still present; previously deleted history cannot +be reconstructed. Telemetry has its own retention setting. Packet drill-down, +sub-hour observer activity, exact observer comparison and current entity/scope +counts continue to describe retained raw data or current entities, rather than +claiming archived packet detail. Archived summaries expire without waiting for +new packet deletions. A failed archive leaves its entire raw batch intact. diff --git a/db/analytics_retention_integration_test.go b/db/analytics_retention_integration_test.go new file mode 100644 index 0000000..2e0466c --- /dev/null +++ b/db/analytics_retention_integration_test.go @@ -0,0 +1,261 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "os" + "testing" + "time" +) + +var retainedViews = []string{"mv_hourly_iata_stats", "mv_payload_breakdown_by_iata", "mv_top_observers_by_iata", "mv_top_talkers_by_iata", "mv_top_advertisers_by_iata", "mv_observer_activity_hourly", "mv_signal_stats_hourly", "mv_path_stats_hourly"} + +func analyticsSnapshot(t *testing.T, ctx context.Context, tx pgx.Tx) map[string]string { + t.Helper() + result := map[string]string{} + for _, view := range retainedViews { + if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW "+view); err != nil { + t.Fatal(err) + } + var rows string + if err := tx.QueryRow(ctx, "SELECT COALESCE(jsonb_agg(r ORDER BY r::text),'[]')::text FROM (SELECT to_jsonb(v) r FROM "+view+" v) s").Scan(&rows); err != nil { + t.Fatal(err) + } + result[view] = rows + } + return result +} + +func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash,last_heard_at) VALUES ('\x01',NOW()-interval '5 days'); INSERT INTO observers(id) VALUES ('00000000-0000-0000-0000-000000000001')`); err != nil { + t.Fatal(err) + } + var schema string + if err := tx.QueryRow(ctx, "SELECT current_schema()").Scan(&schema); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + conn := tx.Conn() + quoted := pgx.Identifier{schema}.Sanitize() + t.Cleanup(func() { + if _, err := conn.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error(err) + } + }) + if _, err := conn.Exec(ctx, "SET search_path TO "+quoted); err != nil { + t.Fatal(err) + } + writer, err := pgx.Connect(ctx, os.Getenv("BEACON_TEST_POSTGRES_DSN")) + if err != nil { + t.Fatal(err) + } + defer writer.Close(context.Background()) + if _, err = writer.Exec(ctx, "SET search_path TO "+quoted); err != nil { + t.Fatal(err) + } + ingest, err := writer.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer ingest.Rollback(context.Background()) + // An in-flight FK insert holds KEY SHARE on the packet until it commits. + if _, err = ingest.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\x01','00000000-0000-0000-0000-000000000001','YVR',NOW()-interval '5 days')`); err != nil { + t.Fatal(err) + } + q := sqlc.New(conn) + params := sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: time.Now().Add(-72 * time.Hour), Valid: true}, BatchSize: 1000} + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("must skip in-flight reception: %d %v", n, err) + } + if err := ingest.Commit(ctx); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 1 { + t.Fatalf("retry after ingestion: %d %v", n, err) + } + var count int + if err := conn.QueryRow(ctx, "SELECT observation_count FROM analytics_hourly_iata_stats").Scan(&count); err != nil || count != 1 { + t.Fatalf("committed reception not archived: %d %v", count, err) + } +} + +// Minimal real tables keep this regression runnable against an empty CI database. +func analyticsTables(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + isolateStatsSchema(t, ctx, tx) + _, err := tx.Exec(ctx, ` +CREATE TABLE packets (packet_hash bytea PRIMARY KEY, payload_type smallint, payload_version smallint, + route_type smallint, raw_payload bytea, raw_header bytea, origin_pubkey bytea, + first_heard_at timestamptz, last_heard_at timestamptz); +CREATE INDEX ON packets(last_heard_at); +CREATE TABLE packet_observations (id bigserial PRIMARY KEY, packet_hash bytea REFERENCES packets ON DELETE CASCADE, + observer_id uuid NOT NULL, iata char(3) NOT NULL, heard_at timestamptz NOT NULL, + path_length_byte smallint, hash_size smallint, hop_count smallint, path_bytes bytea, + snr real, rssi smallint, airtime_ms real, payload_type smallint, UNIQUE(packet_hash,observer_id)); +CREATE TABLE observers (id uuid PRIMARY KEY, public_key bytea, display_name text, observer_type text); +CREATE TABLE nodes (id uuid PRIMARY KEY, public_key bytea UNIQUE, node_type smallint, name text); +CREATE TABLE channel_messages (id bigserial PRIMARY KEY, channel_id integer, packet_hash bytea UNIQUE REFERENCES packets ON DELETE CASCADE, + sender_name text, content text, sent_at timestamptz); +`) + if err != nil { + t.Fatal(err) + } +} + +func TestAnalyticsRetentionPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + _, err := tx.Exec(ctx, ` + INSERT INTO observers (id,public_key,display_name,observer_type) SELECT ('00000000-0000-0000-0000-00000000000'||i)::uuid,int4send(i),CASE WHEN i=1 THEN NULL ELSE 'Observer '||i END,CASE WHEN i=1 THEN NULL ELSE 'test' END FROM generate_series(1,3) i; + INSERT INTO nodes (id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2,'Repeater'); + INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) + SELECT int4send(i),4,0,CASE WHEN i%2=0 THEN 1 ELSE 2 END,'\x00','\x00','\x02',date_trunc('hour',NOW())-interval '5 days', + CASE WHEN i=4 THEN NOW() ELSE date_trunc('hour',NOW())-interval '5 days' END FROM generate_series(1,4) i; + INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,airtime_ms,payload_type) + SELECT packet_hash,('00000000-0000-0000-0000-00000000000'||n)::uuid,iata,first_heard_at+make_interval(secs=>n),0,1,0, + CASE WHEN n=1 THEN 0 ELSE NULL END,CASE WHEN n=1 THEN -100 ELSE NULL END,1.5,4 + FROM packets CROSS JOIN (VALUES ('YVR',1),('YVR',2),('YYZ',3)) v(iata,n); + INSERT INTO channel_messages (channel_id,packet_hash,sender_name,content,sent_at) + SELECT 1,packet_hash,'Sender','body that must expire',first_heard_at FROM packets; + CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS SELECT iata,date_trunc('hour',heard_at) AS hour,count(*) AS observation_count,count(DISTINCT packet_hash) AS unique_packets,count(DISTINCT observer_id) AS active_observers FROM packet_observations GROUP BY 1,2; + `) + if err != nil { + t.Fatal(err) + } + for _, migration := range []string{"016_mv_payload_breakdown.sql", "017_mv_top_observers.sql", "018_mv_top_talkers.sql", "019_mv_top_advertisers.sql", "021_mv_top_advertisers_route_type.sql", "032_mv_observer_activity.sql", "035_mv_signal_stats.sql", "036_mv_path_stats.sql"} { + applyStatsMigration(t, ctx, tx, migration) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + before := analyticsSnapshot(t, ctx, tx) + q := sqlc.New(tx) + cutoff := time.Now().Add(-72 * time.Hour) + // Failure in one archive must roll back every archive write and the raw deletion. + if _, err := tx.Exec(ctx, `SAVEPOINT archive_failure; ALTER TABLE analytics_signal_stats_hourly ADD CONSTRAINT fail_archive CHECK(receptions < 0) NOT VALID`); err != nil { + t.Fatal(err) + } + if _, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}); err == nil { + t.Fatal("expected archive failure") + } + if _, err := tx.Exec(ctx, "ROLLBACK TO archive_failure"); err != nil { + t.Fatal(err) + } + if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 4 { + t.Fatal("archive failure deleted raw packets") + } + for i := 0; i < 3; i++ { + n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}) + if err != nil || n != 1 { + t.Fatalf("delete batch: %d %v", n, err) + } + } + if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 1 || countRows(t, ctx, tx, "SELECT count(*) FROM channel_messages") != 1 { + t.Fatal("raw packets/messages did not expire") + } + after := analyticsSnapshot(t, ctx, tx) + for _, view := range retainedViews { + if after[view] != before[view] { + t.Errorf("%s lost or double-counted history after raw expiry: before=%s after=%s", view, before[view], after[view]) + } + } + // Retry and migration journal retry preserve the exact summaries. + if n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1000}); err != nil || n != 0 { + t.Fatalf("retry: %d %v", n, err) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != before[view] { + t.Errorf("retry changed %s", view) + } + } + // Later receptions on a still-live packet must join its old bucket before expiry. + if _, err := tx.Exec(ctx, `INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000004'); INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,payload_type) + SELECT packet_hash,'00000000-0000-0000-0000-000000000004','YVR',first_heard_at,0,1,0,'NaN',-80,NULL FROM packets; + UPDATE packets SET last_heard_at=first_heard_at`); err != nil { + t.Fatal(err) + } + late := analyticsSnapshot(t, ctx, tx) + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != late[view] { + t.Errorf("late reception changed %s: %s != %s", view, rows, late[view]) + } + } + // Archived labels follow a current rename; no entity FK may erase retained history. + if _, err := tx.Exec(ctx, `UPDATE nodes SET name='Renamed'; UPDATE observers SET display_name=NULL,observer_type=NULL;`); err != nil { + t.Fatal(err) + } + analyticsSnapshot(t, ctx, tx) + if countRows(t, ctx, tx, "SELECT count(*) FROM mv_top_advertisers_by_iata WHERE name='Renamed'") != 2 { + t.Fatal("archive name did not follow rename") + } + if _, err := q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{Column1: pgtype.Interval{Microseconds: int64(30 * 24 * time.Hour / time.Microsecond), Valid: true}, Limit: 10}); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, "DELETE FROM nodes; DELETE FROM observers"); err != nil { + t.Fatal(err) + } + if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_top_advertisers_by_iata") != 2 { + t.Fatal("entity deletion erased archive") + } + // Independent expiry runs even when there are no packets left to delete. + for _, view := range retainedViews { + table := "analytics_" + view[3:] + at := "bucket" + if view == "mv_hourly_iata_stats" || view == "mv_signal_stats_hourly" || view == "mv_path_stats_hourly" { + at = "hour" + } + if _, err := tx.Exec(ctx, "UPDATE "+table+" SET "+at+"="+at+"-interval '31 days'"); err != nil { + t.Fatal(err) + } + } + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + for _, view := range retainedViews { + if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) != 0 { + t.Errorf("%s archive did not expire", view) + } + } + // More than one populated cohort, with shared hourly keys across the boundary. + if _, err := tx.Exec(ctx, ` +INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000001'); +INSERT INTO nodes (id,public_key,node_type) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2); +INSERT INTO packets (packet_hash,payload_type,route_type,origin_pubkey,first_heard_at,last_heard_at,raw_payload) +SELECT int4send(i),4,1,'\x02',date_trunc('hour',NOW())-interval '5 days',date_trunc('hour',NOW())-interval '5 days',decode(repeat('ab',1024),'hex') FROM generate_series(10000,11000) i; +INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) +SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YVR',first_heard_at,0,1,0,4,0,-100 FROM packets; +INSERT INTO channel_messages (packet_hash,sender_name,sent_at) SELECT packet_hash,'Sender',first_heard_at FROM packets; +`); err != nil { + t.Fatal(err) + } + populated := analyticsSnapshot(t, ctx, tx) + started := time.Now() + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + t.Logf("1001 packets with bodies and observations archived/deleted in %s", time.Since(started)) + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != populated[view] { + t.Errorf("batch boundary changed %s", view) + } + } + var archiveRows int + for _, view := range retainedViews { + archiveRows += countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) + } + if archiveRows > 12 { + t.Fatalf("expected compact rollups, got %d archive rows for 1001 packets", archiveRows) + } +} diff --git a/db/migrations/039_analytics_retention.sql b/db/migrations/039_analytics_retention.sql new file mode 100644 index 0000000..75d5007 --- /dev/null +++ b/db/migrations/039_analytics_retention.sql @@ -0,0 +1,611 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Archive only aggregates of deleted packet cohorts, never packet/message bodies +-- or raw paths. Live and archived cohorts are disjoint in every refresh snapshot. +-- Thirty days matches the longest analytics window; raw retention is independent. + + +CREATE TABLE IF NOT EXISTS analytics_hourly_iata_stats ( + iata char(3), + hour timestamptz, + observation_count bigint, + unique_packets bigint, + PRIMARY KEY (iata,hour) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_hourly_iata_stats_time ON analytics_hourly_iata_stats (hour); + +CREATE TABLE IF NOT EXISTS analytics_payload_breakdown_by_iata ( + iata char(3), + payload_type smallint, + bucket timestamptz, + count bigint, + PRIMARY KEY (iata,payload_type,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_payload_breakdown_by_iata_time ON analytics_payload_breakdown_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_observers_by_iata ( + iata char(3), + observer_id uuid, + bucket timestamptz, + observation_count bigint, + display_name text, + observer_type text, + PRIMARY KEY (iata,observer_id,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_observers_by_iata_time ON analytics_top_observers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_talkers_by_iata ( + iata char(3), + sender_name text, + bucket timestamptz, + message_count bigint, + last_sent timestamptz, + PRIMARY KEY (iata,sender_name,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_talkers_by_iata_time ON analytics_top_talkers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_advertisers_by_iata ( + iata char(3), + node_id uuid, + bucket timestamptz, + advert_count bigint, + flood_advert_count bigint, + direct_advert_count bigint, + last_heard timestamptz, + name text, + node_type smallint, + PRIMARY KEY (iata,node_id,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_advertisers_by_iata_time ON analytics_top_advertisers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_observer_activity_hourly ( + observer_id uuid, + payload_type smallint, + bucket timestamptz, + observations bigint, + airtime_ms real, + airtime_n bigint, + snr_sum real, + snr_n bigint, + snr_min real, + rssi_sum bigint, + rssi_n bigint, + PRIMARY KEY (observer_id,payload_type,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_observer_activity_hourly_time ON analytics_observer_activity_hourly (bucket); + +CREATE TABLE IF NOT EXISTS analytics_signal_stats_hourly ( + iata char(3), + hour timestamptz, + kind integer, + snr_bin integer, + rssi_bin integer, + receptions bigint, + snr_samples bigint, + snr_sum double precision, + rssi_samples bigint, + rssi_sum double precision, + PRIMARY KEY (iata,hour,kind,snr_bin,rssi_bin) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_signal_stats_hourly_time ON analytics_signal_stats_hourly (hour); + +CREATE TABLE IF NOT EXISTS analytics_path_stats_hourly ( + iata char(3), + hour timestamptz, + category integer, + hash_bytes integer, + entries integer, + receptions bigint, + PRIMARY KEY (iata,hour,category,hash_bytes,entries) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_path_stats_hourly_time ON analytics_path_stats_hourly (hour); + +DROP MATERIALIZED VIEW IF EXISTS mv_hourly_iata_stats; + +CREATE OR REPLACE VIEW analytics_live_hourly_iata_stats AS +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS +WITH combined AS ( + SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats + UNION ALL + SELECT iata, hour, observation_count, unique_packets FROM analytics_hourly_iata_stats + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observers_by_hour AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, observer_id + FROM packet_observations WHERE heard_at > NOW() - INTERVAL '30 days' + UNION + SELECT iata, bucket AS hour, observer_id FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observer_counts AS (SELECT iata, hour, COUNT(*) AS n FROM observers_by_hour GROUP BY iata,hour) +SELECT combined.iata, combined.hour, SUM(observation_count)::bigint AS observation_count, SUM(unique_packets)::bigint AS unique_packets, MAX(o.n)::bigint AS active_observers +FROM combined JOIN observer_counts o USING (iata,hour) GROUP BY combined.iata,combined.hour; + +CREATE UNIQUE INDEX idx_analytics_hourly_iata_stats_view ON mv_hourly_iata_stats (iata,hour); + + +DROP MATERIALIZED VIEW IF EXISTS mv_payload_breakdown_by_iata; + +CREATE OR REPLACE VIEW analytics_live_payload_breakdown_by_iata AS +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS +WITH combined AS ( + SELECT iata, payload_type, bucket, count FROM analytics_live_payload_breakdown_by_iata + UNION ALL + SELECT iata, payload_type, bucket, count FROM analytics_payload_breakdown_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, payload_type, bucket, SUM(count)::bigint AS count +FROM combined GROUP BY iata,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_payload_breakdown_by_iata_view ON mv_payload_breakdown_by_iata (iata,payload_type,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_observers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_observers_by_iata AS +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS +WITH combined AS ( + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_live_top_observers_by_iata + UNION ALL + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, observer_id, bucket, SUM(observation_count)::bigint AS observation_count, MAX(display_name) AS display_name, MAX(observer_type) AS observer_type +FROM combined GROUP BY iata,observer_id,bucket) +SELECT t.iata, t.observer_id, COALESCE(o.display_name,t.display_name) AS display_name, + COALESCE(o.observer_type,t.observer_type) AS observer_type, t.bucket, t.observation_count +FROM totals t LEFT JOIN observers o ON o.id=t.observer_id; + +CREATE UNIQUE INDEX idx_analytics_top_observers_by_iata_view ON mv_top_observers_by_iata (iata,observer_id,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_talkers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_talkers_by_iata AS +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN packet_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS +WITH combined AS ( + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_live_top_talkers_by_iata + UNION ALL + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_top_talkers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, sender_name, bucket, SUM(message_count)::bigint AS message_count, MAX(last_sent) AS last_sent +FROM combined GROUP BY iata,sender_name,bucket; + +CREATE UNIQUE INDEX idx_analytics_top_talkers_by_iata_view ON mv_top_talkers_by_iata (iata,sender_name,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_advertisers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_advertisers_by_iata AS +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN packet_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS +WITH combined AS ( + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_live_top_advertisers_by_iata + UNION ALL + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_top_advertisers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, node_id, bucket, SUM(advert_count)::bigint AS advert_count, SUM(flood_advert_count)::bigint AS flood_advert_count, SUM(direct_advert_count)::bigint AS direct_advert_count, MAX(last_heard) AS last_heard, MAX(name) AS name, MAX(node_type) AS node_type +FROM combined GROUP BY iata,node_id,bucket) +SELECT t.iata, t.node_id, COALESCE(n.name,t.name) AS name, COALESCE(n.node_type,t.node_type) AS node_type, + t.bucket, t.advert_count, t.flood_advert_count, t.direct_advert_count, t.last_heard +FROM totals t LEFT JOIN nodes n ON n.id=t.node_id; + +CREATE UNIQUE INDEX idx_analytics_top_advertisers_by_iata_view ON mv_top_advertisers_by_iata (iata,node_id,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_observer_activity_hourly; + +CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS +SELECT + observer_id, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY observer_id, payload_type, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS +WITH combined AS ( + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly + UNION ALL + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n +FROM combined GROUP BY observer_id,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_signal_stats_hourly; + +CREATE OR REPLACE VIEW analytics_live_signal_stats_hourly AS +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); + +CREATE MATERIALIZED VIEW mv_signal_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_live_signal_stats_hourly + UNION ALL + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_signal_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, kind, snr_bin, rssi_bin, SUM(receptions)::bigint AS receptions, SUM(snr_samples)::bigint AS snr_samples, SUM(snr_sum)::double precision AS snr_sum, SUM(rssi_samples)::bigint AS rssi_samples, SUM(rssi_sum)::double precision AS rssi_sum +FROM combined GROUP BY iata,hour,kind,snr_bin,rssi_bin; + +CREATE UNIQUE INDEX idx_analytics_signal_stats_hourly_view ON mv_signal_stats_hourly (iata,hour,kind,snr_bin,rssi_bin); + + +DROP MATERIALIZED VIEW IF EXISTS mv_path_stats_hourly; + +CREATE OR REPLACE VIEW analytics_live_path_stats_hourly AS +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; + +CREATE MATERIALIZED VIEW mv_path_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_live_path_stats_hourly + UNION ALL + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_path_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, category, hash_bytes, entries, SUM(receptions)::bigint AS receptions +FROM combined GROUP BY iata,hour,category,hash_bytes,entries; + +CREATE UNIQUE INDEX idx_analytics_path_stats_hourly_view ON mv_path_stats_hourly (iata,hour,category,hash_bytes,entries); + + +-- VOLATILE gives the archive statement a new READ COMMITTED snapshot after +-- acquiring parent locks. FK inserts cannot race between that snapshot and the +-- cascade. SKIP LOCKED leaves packets being ingested for a later cleanup tick. +CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) +RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ +DECLARE hashes bytea[]; deleted bigint; +BEGIN + IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; + SELECT array_agg(p.packet_hash) INTO hashes FROM ( + SELECT ep.packet_hash FROM packets ep + WHERE ep.last_heard_at < cutoff + ORDER BY ep.last_heard_at, ep.packet_hash + LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED + ) p; + + WITH expired_observations AS MATERIALIZED ( + SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) + ), +archived_hourly_iata_stats AS ( + INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) + SELECT iata, hour, observation_count, unique_packets FROM ( +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,hour) DO UPDATE SET + observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, + unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets +), +archived_payload_breakdown_by_iata AS ( + INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) + SELECT iata, payload_type, bucket, count FROM ( +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET + count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count +), +archived_top_observers_by_iata AS ( + INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM expired_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET + observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, + display_name = EXCLUDED.display_name, + observer_type = EXCLUDED.observer_type +), +archived_top_talkers_by_iata AS ( + INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) + SELECT iata, sender_name, bucket, message_count, last_sent FROM ( +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN expired_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') + ) batch + ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET + message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, + last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) +), +archived_top_advertisers_by_iata AS ( + INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN expired_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,node_id,bucket) DO UPDATE SET + advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, + flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, + direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, + last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), + name = EXCLUDED.name, + node_type = EXCLUDED.node_type +), +archived_observer_activity_hourly AS ( + INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( +SELECT + observer_id, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY observer_id, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET + observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, + airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, + airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, + snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, + snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, + snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), + rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, + rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n +), +archived_signal_stats_hourly AS ( + INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) + ) batch + ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET + receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, + snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, + snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, + rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, + rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum +), +archived_path_stats_hourly AS ( + INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) + SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries + ) batch + ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET + receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions +) + DELETE FROM packets WHERE packet_hash = ANY(hashes); + GET DIAGNOSTICS deleted = ROW_COUNT; + + -- Once per completed cleanup, including when there were no raw packets to delete. + IF deleted < batch_size THEN + DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + END IF; + RETURN deleted; +END $$; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index ccc8492..29690a1 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -661,18 +661,9 @@ ORDER BY po.id ASC LIMIT $6; --- name: DeleteOldPackets :execrows --- One batch of expired packets; observations and channel messages cascade. -WITH expired AS ( - SELECT ep.packet_hash - FROM packets ep - WHERE ep.last_heard_at < @cutoff - ORDER BY ep.last_heard_at - LIMIT @batch_size - FOR UPDATE OF ep SKIP LOCKED -) -DELETE FROM packets p USING expired e -WHERE p.packet_hash = e.packet_hash; +-- name: DeleteOldPackets :one +-- Locks, archives and cascades one bounded packet cohort in a single transaction. +SELECT archive_delete_packets(@cutoff::timestamptz, @batch_size::integer)::bigint; -- name: DeleteOldNodes :exec -- Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- diff --git a/db/retention_integration_test.go b/db/retention_integration_test.go index 5e40dbd..d630717 100644 --- a/db/retention_integration_test.go +++ b/db/retention_integration_test.go @@ -46,10 +46,9 @@ func countRows(t *testing.T, ctx context.Context, tx pgx.Tx, sql string) int { // More than one batch of old packets; observations go with them. func TestDeleteOldPacketsBatchesPostgres(t *testing.T) { ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") _, err := tx.Exec(ctx, ` -CREATE TEMP TABLE packets (LIKE public.packets INCLUDING ALL) ON COMMIT DROP; -CREATE TEMP TABLE packet_observations (LIKE public.packet_observations INCLUDING ALL) ON COMMIT DROP; -ALTER TABLE packet_observations ADD FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) SELECT int4send(i), 4, 0, 1, '\x00', '\x00', CASE WHEN i <= 2500 THEN '2026-01-01'::timestamptz ELSE '2026-02-01'::timestamptz END, diff --git a/db/sqlc/models.go b/db/sqlc/models.go index b5ffb05..2c6e20f 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -16,6 +16,164 @@ type Account struct { DeactivatedAt pgtype.Timestamptz `json:"deactivated_at"` } +type AnalyticsHourlyIataStat struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount *int64 `json:"observation_count"` + UniquePackets *int64 `json:"unique_packets"` +} + +type AnalyticsLiveHourlyIataStat struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount int64 `json:"observation_count"` + UniquePackets int64 `json:"unique_packets"` +} + +type AnalyticsLiveObserverActivityHourly struct { + ObserverID uuid.UUID `json:"observer_id"` + PayloadType *int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin float32 `json:"snr_min"` + RssiSum int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + +type AnalyticsLivePathStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +type AnalyticsLivePayloadBreakdownByIatum struct { + Iata string `json:"iata"` + PayloadType *int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Count int64 `json:"count"` +} + +type AnalyticsLiveSignalStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrSum float64 `json:"snr_sum"` + RssiSamples int64 `json:"rssi_samples"` + RssiSum float64 `json:"rssi_sum"` +} + +type AnalyticsLiveTopAdvertisersByIatum struct { + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + NodeType int16 `json:"node_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` + LastHeard interface{} `json:"last_heard"` +} + +type AnalyticsLiveTopObserversByIatum struct { + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + ObservationCount int64 `json:"observation_count"` +} + +type AnalyticsLiveTopTalkersByIatum struct { + Iata string `json:"iata"` + SenderName *string `json:"sender_name"` + Bucket pgtype.Timestamptz `json:"bucket"` + MessageCount int64 `json:"message_count"` + LastSent interface{} `json:"last_sent"` +} + +type AnalyticsObserverActivityHourly struct { + ObserverID uuid.UUID `json:"observer_id"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Observations *int64 `json:"observations"` + AirtimeMs *float32 `json:"airtime_ms"` + AirtimeN *int64 `json:"airtime_n"` + SnrSum *float32 `json:"snr_sum"` + SnrN *int64 `json:"snr_n"` + SnrMin *float32 `json:"snr_min"` + RssiSum *int64 `json:"rssi_sum"` + RssiN *int64 `json:"rssi_n"` +} + +type AnalyticsPathStatsHourly struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions *int64 `json:"receptions"` +} + +type AnalyticsPayloadBreakdownByIatum struct { + Iata string `json:"iata"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Count *int64 `json:"count"` +} + +type AnalyticsSignalStatsHourly struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions *int64 `json:"receptions"` + SnrSamples *int64 `json:"snr_samples"` + SnrSum *float64 `json:"snr_sum"` + RssiSamples *int64 `json:"rssi_samples"` + RssiSum *float64 `json:"rssi_sum"` +} + +type AnalyticsTopAdvertisersByIatum struct { + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Bucket pgtype.Timestamptz `json:"bucket"` + AdvertCount *int64 `json:"advert_count"` + FloodAdvertCount *int64 `json:"flood_advert_count"` + DirectAdvertCount *int64 `json:"direct_advert_count"` + LastHeard pgtype.Timestamptz `json:"last_heard"` + Name *string `json:"name"` + NodeType *int16 `json:"node_type"` +} + +type AnalyticsTopObserversByIatum struct { + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + Bucket pgtype.Timestamptz `json:"bucket"` + ObservationCount *int64 `json:"observation_count"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` +} + +type AnalyticsTopTalkersByIatum struct { + Iata string `json:"iata"` + SenderName string `json:"sender_name"` + Bucket pgtype.Timestamptz `json:"bucket"` + MessageCount *int64 `json:"message_count"` + LastSent pgtype.Timestamptz `json:"last_sent"` +} + type Channel struct { ID int32 `json:"id"` ChannelHash []byte `json:"channel_hash"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 1547c1c..38e6c97 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -29,7 +29,7 @@ type Querier interface { // Opt-in age-out: preserve retained history and manually recorded ownership. // Bound deletions per cleanup tick and skip observers being updated by ingest. DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) - // One batch of expired packets; observations and channel messages cascade. + // Locks, archives and cascades one bounded packet cohort in a single transaction. DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) // One batch of routes past retention, or past grace with too few observations. // GREATEST keeps the scan on idx_known_routes_last_seen. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index c047b44..6a0eee1 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -124,17 +124,8 @@ func (q *Queries) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timest return items, nil } -const deleteOldPackets = `-- name: DeleteOldPackets :execrows -WITH expired AS ( - SELECT ep.packet_hash - FROM packets ep - WHERE ep.last_heard_at < $1 - ORDER BY ep.last_heard_at - LIMIT $2 - FOR UPDATE OF ep SKIP LOCKED -) -DELETE FROM packets p USING expired e -WHERE p.packet_hash = e.packet_hash +const deleteOldPackets = `-- name: DeleteOldPackets :one +SELECT archive_delete_packets($1::timestamptz, $2::integer)::bigint ` type DeleteOldPacketsParams struct { @@ -142,13 +133,12 @@ type DeleteOldPacketsParams struct { BatchSize int32 `json:"batch_size"` } -// One batch of expired packets; observations and channel messages cascade. +// Locks, archives and cascades one bounded packet cohort in a single transaction. func (q *Queries) DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) { - result, err := q.db.Exec(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) - if err != nil { - return 0, err - } - return result.RowsAffected(), nil + row := q.db.QueryRow(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err } const deleteOldRoutes = `-- name: DeleteOldRoutes :execrows diff --git a/sqlc.yaml b/sqlc.yaml index 7943109..032c991 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -22,3 +22,11 @@ sql: go_type: "encoding/json.RawMessage" - db_type: "bpchar" go_type: "string" + + # Coalesced archive labels can still be NULL; preserve the API pointers. + - column: "mv_top_observers_by_iata.display_name" + go_type: {type: "string", pointer: true} + - column: "mv_top_observers_by_iata.observer_type" + go_type: {type: "string", pointer: true} + - column: "mv_top_advertisers_by_iata.name" + go_type: {type: "string", pointer: true} From fd47529b06cc47b4dfe7899546e5a3ec60184369 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 21:59:48 -0400 Subject: [PATCH 2/9] fix(observers): separate packet metrics from presence updates --- .github/workflows/ci.yml | 2 +- README.md | 18 ++ .../040_observer_unknown_activity.sql | 279 ++++++++++++++++++ db/observer_metrics_integration_test.go | 87 ++++++ db/observer_retention_integration_test.go | 4 +- db/observers.go | 62 ++-- db/observers_test.go | 35 ++- db/presence.go | 11 +- db/queries/observer_activity_summary.sql | 13 + db/queries/queries.sql | 19 +- db/sqlc/mock/querier.go | 17 +- db/sqlc/models.go | 2 +- db/sqlc/observer_activity_summary.sql.go | 52 ++++ db/sqlc/querier.go | 2 + db/sqlc/queries.sql.go | 52 +++- docs/docs.go | 46 +++ docs/swagger.json | 46 +++ docs/swagger.yaml | 33 +++ internal/api/handlers/observers.go | 13 +- internal/api/handlers/observers_test.go | 20 +- internal/api/handlers/stub_reader_test.go | 6 +- internal/api/observers.go | 19 +- internal/api/reader.go | 2 +- internal/cache/cache_test.go | 2 +- internal/cache/reader.go | 6 +- internal/ingest/ingest.go | 2 +- internal/ingest/ingest_test.go | 6 +- internal/ingest/neighbors.go | 2 +- internal/ingest/packet.go | 2 +- internal/ingest/status.go | 2 +- internal/presence/broker_freshness_test.go | 50 ++++ internal/presence/coalescer.go | 27 +- sqlc.yaml | 3 + 33 files changed, 848 insertions(+), 94 deletions(-) create mode 100644 db/migrations/040_observer_unknown_activity.sql create mode 100644 db/observer_metrics_integration_test.go create mode 100644 db/queries/observer_activity_summary.sql create mode 100644 db/sqlc/observer_activity_summary.sql.go create mode 100644 internal/presence/broker_freshness_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d37492..c9d71ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index 5243068..33ae4e1 100644 --- a/README.md +++ b/README.md @@ -548,3 +548,21 @@ sub-hour observer activity, exact observer comparison and current entity/scope counts continue to describe retained raw data or current entities, rather than claiming archived packet detail. Archived summaries expire without waiting for new packet deletions. A failed archive leaves its entire raw batch intact. + +### Observer monitoring metrics + +Observer activity returns complete buckets in `[windowStart, windowEnd)`, plus +`generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the +sum of stored observations in those buckets; repeated broker delivery of the +same retained packet/observer pair counts once. Unknown payload types appear as +`-1` rather than disappearing. `lastCompleteHour` uses the previous complete UTC +hour and includes its own start/end; `latestRecordedAt` is the latest retained +reception timestamp. Missing records do not prove downtime. The optional `until` +(epoch milliseconds, within the last 30 days) aligns two observers' charts. + +The existing observer `observationCount` remains a legacy cumulative presence +counter for compatibility, including status/neighbour events. It is not a +period packet total. Broker presence and packet-arrival timestamps are now +updated separately; this cannot reconstruct previously overwritten timestamps. +Migration 040 repairs archived unknown-type counts from the all-payload observer +rollup; radio samples already discarded for those legacy rows remain unknown. diff --git a/db/migrations/040_observer_unknown_activity.sql b/db/migrations/040_observer_unknown_activity.sql new file mode 100644 index 0000000..6afab7e --- /dev/null +++ b/db/migrations/040_observer_unknown_activity.sql @@ -0,0 +1,279 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Unknown legacy payloads are still recorded packets. Restore their archived +-- counts from the all-payload observer rollup; their discarded radio samples +-- cannot be recovered. Subtract all existing activity (including prior repairs) +-- so migration-journal retries do not double-count or erase newer samples. +INSERT INTO analytics_observer_activity_hourly +(observer_id,payload_type,bucket,observations,airtime_n,snr_n,rssi_n) +SELECT totals.observer_id, -1, totals.bucket, totals.n-COALESCE(activity.n,0),0,0,0 +FROM (SELECT observer_id,bucket,SUM(observation_count)::bigint AS n + FROM analytics_top_observers_by_iata GROUP BY observer_id,bucket) totals +LEFT JOIN (SELECT observer_id,bucket,SUM(observations)::bigint AS n + FROM analytics_observer_activity_hourly GROUP BY observer_id,bucket) activity +USING (observer_id,bucket) +WHERE totals.n>COALESCE(activity.n,0) +ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE +SET observations=analytics_observer_activity_hourly.observations+EXCLUDED.observations; + +DROP MATERIALIZED VIEW mv_observer_activity_hourly; +CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS +WITH combined AS ( + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly + UNION ALL + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n +FROM combined GROUP BY observer_id,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); + + + +-- Preserve the bounded atomic archive, including unknown payload activity. +CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) +RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ +DECLARE hashes bytea[]; deleted bigint; +BEGIN + IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; + SELECT array_agg(p.packet_hash) INTO hashes FROM ( + SELECT ep.packet_hash FROM packets ep + WHERE ep.last_heard_at < cutoff + ORDER BY ep.last_heard_at, ep.packet_hash + LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED + ) p; + + WITH expired_observations AS MATERIALIZED ( + SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) + ), +archived_hourly_iata_stats AS ( + INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) + SELECT iata, hour, observation_count, unique_packets FROM ( +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,hour) DO UPDATE SET + observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, + unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets +), +archived_payload_breakdown_by_iata AS ( + INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) + SELECT iata, payload_type, bucket, count FROM ( +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET + count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count +), +archived_top_observers_by_iata AS ( + INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM expired_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET + observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, + display_name = EXCLUDED.display_name, + observer_type = EXCLUDED.observer_type +), +archived_top_talkers_by_iata AS ( + INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) + SELECT iata, sender_name, bucket, message_count, last_sent FROM ( +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN expired_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') + ) batch + ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET + message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, + last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) +), +archived_top_advertisers_by_iata AS ( + INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN expired_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,node_id,bucket) DO UPDATE SET + advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, + flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, + direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, + last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), + name = EXCLUDED.name, + node_type = EXCLUDED.node_type +), +archived_observer_activity_hourly AS ( + INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET + observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, + airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, + airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, + snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, + snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, + snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), + rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, + rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n +), +archived_signal_stats_hourly AS ( + INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) + ) batch + ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET + receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, + snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, + snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, + rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, + rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum +), +archived_path_stats_hourly AS ( + INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) + SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries + ) batch + ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET + receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions +) + DELETE FROM packets WHERE packet_hash = ANY(hashes); + GET DIAGNOSTICS deleted = ROW_COUNT; + + -- Once per completed cleanup, including when there were no raw packets to delete. + IF deleted < batch_size THEN + DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + END IF; + RETURN deleted; +END $$; diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go new file mode 100644 index 0000000..868ce6c --- /dev/null +++ b/db/observer_metrics_integration_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "strings" + "testing" + "time" +) + +func TestObserverMetricsPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + // Use the real observer DDL without depending on a preinstalled public schema. + initial, err := migrationFiles.ReadFile("migrations/001_initial_schema.sql") + if err != nil { + t.Fatal(err) + } + ddl := string(initial) + start := strings.Index(ddl, "CREATE TABLE observers (") + end := strings.Index(ddl[start:], "\n);") + start + 3 + if _, err := tx.Exec(ctx, "DROP TABLE observers; "+ddl[start:end]+" ALTER TABLE observers ADD COLUMN region_scope TEXT;"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, ` + CREATE TABLE observer_brokers (observer_id uuid,broker_name text,last_seen timestamptz,last_packet_at timestamptz,PRIMARY KEY(observer_id,broker_name)); + INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); + INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,0,-100 FROM packets; + `); err != nil { + t.Fatal(err) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + store := &Store{q: sqlc.New(tx)} + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { + t.Fatal(err) + } + // Migration repairs the two unknown counts that 039 did not archive. + applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") + applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") + until := time.Now().UTC().Truncate(time.Hour) + activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until) + if err != nil { + t.Fatal(err) + } + if activity.Summary.RecordedPackets != 3 || len(activity.PayloadTypes) != 2 || activity.Summary.LatestRecordedAt != nil || activity.Summary.LastCompleteHour != 0 { + t.Fatalf("archive summary: %+v types=%+v", activity.Summary, activity.PayloadTypes) + } + if activity.WindowEnd != until.UnixMilli() || activity.WindowStart != until.Add(-7*24*time.Hour).UnixMilli() || activity.Source != "hourly" { + t.Fatalf("window: %+v", activity) + } + if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,last_heard_at) VALUES ('\xaa',NOW()); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,0,-100); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW()) ON CONFLICT DO NOTHING;`); err != nil { + t.Fatal(err) + } + raw, err := store.GetObserverActivity(ctx, id, time.Hour, 15*time.Minute, until) + if err != nil { + t.Fatal(err) + } + if raw.Summary.RecordedPackets != 1 || raw.Summary.LastCompleteHour != 1 || raw.Summary.LatestRecordedAt == nil || raw.PayloadTypes[0].PayloadType != -1 { + t.Fatalf("raw/duplicate: %+v", raw) + } + if err := store.UpsertObserverBroker(ctx, id, "one", false); err != nil { + t.Fatal(err) + } + var packet pgtype.Timestamptz + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if packet.Valid { + t.Fatal("status created a packet timestamp") + } + if err := store.UpsertObserverBroker(ctx, id, "one", true); err != nil { + t.Fatal(err) + } + if err := store.TouchObserverBrokers(ctx, []uuid.UUID{id}, []string{"one"}, []time.Time{until.Add(time.Hour)}, []time.Time{{}}); err != nil { + t.Fatal(err) + } + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if !packet.Valid { + t.Fatal("status erased packet timestamp") + } +} diff --git a/db/observer_retention_integration_test.go b/db/observer_retention_integration_test.go index 8672ea4..8e64ea9 100644 --- a/db/observer_retention_integration_test.go +++ b/db/observer_retention_integration_test.go @@ -100,7 +100,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil { t.Fatal(err) } - if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture"); err != nil { + if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture", true); err != nil { t.Fatal(err) } // A future cutoff simulates this cached observer having aged past retention. @@ -112,7 +112,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil || newID == oldID { t.Fatalf("returning observer reused a deleted ID: %v", err) } - if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture"); err != nil { + if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture", true); err != nil { t.Fatal(err) } var brokers int diff --git a/db/observers.go b/db/observers.go index a3448cf..1735042 100644 --- a/db/observers.go +++ b/db/observers.go @@ -230,12 +230,37 @@ func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uui // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. // Buckets of an hour or coarser come from the hourly rollup; anything finer reads observations directly. // Range and Interval are left empty for the handler to fill. -func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { +func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { obs, err := s.q.GetObserverByID(ctx, observerID) if err != nil { return nil, err } - activity := &api.ObserverActivity{} + now := time.Now().UTC() + if until.IsZero() { + until = now + } + until = until.UTC().Truncate(interval) + since := until.Add(-window) + if !since.Equal(since.Truncate(interval)) { + since = since.Truncate(interval).Add(interval) + } + activity := &api.ObserverActivity{WindowStart: since.UnixMilli(), WindowEnd: until.UnixMilli(), GeneratedAt: now.UnixMilli(), Source: "raw", Summary: &api.ObserverActivitySummary{}} + if interval >= time.Hour { + activity.Source = "hourly" + } + live, err := s.q.GetObserverActivityLiveSummary(ctx, sqlc.GetObserverActivityLiveSummaryParams{ + ObserverID: observerID, GeneratedAt: pgtype.Timestamptz{Time: now, Valid: true}, + HourStart: pgtype.Timestamptz{Time: now.Truncate(time.Hour).Add(-time.Hour), Valid: true}, HourEnd: pgtype.Timestamptz{Time: now.Truncate(time.Hour), Valid: true}}) + if err != nil { + return nil, err + } + activity.Summary.LastCompleteHour = live.LastCompleteHour + if live.LatestRecordedAt.Valid { + v := live.LatestRecordedAt.Time.UnixMilli() + activity.Summary.LatestRecordedAt = &v + } + activity.Summary.LastCompleteHourStart = now.Truncate(time.Hour).Add(-time.Hour).UnixMilli() + activity.Summary.LastCompleteHourEnd = now.Truncate(time.Hour).UnixMilli() // radio is non-nil only when airtime is actually costable, so radio != null implies costed buckets if obs.RadioSf != nil && obs.RadioBwKhz != nil && obs.RadioCr != nil && *obs.RadioSf >= 7 && *obs.RadioSf <= 12 && *obs.RadioBwKhz > 0 && *obs.RadioCr > 0 { @@ -247,12 +272,6 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w PreambleSymbols: lora.PreambleSymbols(int(*obs.RadioSf)), } } - // round the window start up to a bucket boundary so the first bucket is never a partial one - start := time.Now().Add(-window).UTC() - since := start.Truncate(interval) - if since.Before(start) { - since = since.Add(interval) - } sinceTS := pgtype.Timestamptz{Time: since, Valid: true} binWidth := pgtype.Interval{Microseconds: interval.Microseconds(), Valid: true} @@ -261,6 +280,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w ObserverID: observerID, Column2: sinceTS, Column3: binWidth, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err @@ -268,6 +288,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) for _, r := range rows { p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations if r.AirtimeN > 0 { airtime := r.AirtimeMs p.AirtimeMs = &airtime @@ -286,18 +307,20 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityHourlyPayloadTypes(ctx, sqlc.GetObserverActivityHourlyPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - if v.PayloadType == nil { - continue + payloadType := int16(-1) + if v.PayloadType != nil { + payloadType = *v.PayloadType } activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), Count: v.Count, }) } @@ -308,6 +331,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w ObserverID: observerID, Column2: sinceTS, Column3: binWidth, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err @@ -315,6 +339,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) for _, r := range rows { p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations if r.AirtimeN > 0 { airtime := r.AirtimeMs p.AirtimeMs = &airtime @@ -332,18 +357,20 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityRawPayloadTypes(ctx, sqlc.GetObserverActivityRawPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - if v.PayloadType == nil { - continue + payloadType := int16(-1) + if v.PayloadType != nil { + payloadType = *v.PayloadType } activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), Count: v.Count, }) } @@ -424,10 +451,11 @@ func (s *Store) GetObserverRadio(ctx context.Context, observerID uuid.UUID) (ing return settings, nil } -func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { params := sqlc.UpsertObserverBrokerParams{ ObserverID: observerID, BrokerName: brokerName, + IsPacket: isPacket, } return s.q.UpsertObserverBroker(ctx, params) } diff --git a/db/observers_test.go b/db/observers_test.go index 76ccf54..31b562e 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -569,6 +569,7 @@ func f32(v float32) *float32 { return &v } func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -592,7 +593,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { }, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -615,8 +616,8 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { if p.RSSIAvg != nil { t.Errorf("expected nil RSSIAvg, got %v", *p.RSSIAvg) } - if len(got.PayloadTypes) != 1 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 { - t.Fatalf("expected one payload type 4 with count 9, got %+v", got.PayloadTypes) + if len(got.PayloadTypes) != 2 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 || got.PayloadTypes[1].PayloadType != -1 || got.PayloadTypes[1].Count != 3 { + t.Fatalf("expected known and unknown payloads to reconcile, got %+v", got.PayloadTypes) } if got.PayloadTypes[0].PayloadTypeName != api.PayloadTypeName(4) { t.Errorf("expected payload type name %q, got %q", api.PayloadTypeName(4), got.PayloadTypes[0].PayloadTypeName) @@ -638,6 +639,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -658,7 +660,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -689,6 +691,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { func TestGetObserverActivity_RawPath(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -709,7 +712,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{{PayloadType: i16(1), Count: 5}}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -737,6 +740,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) { func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -757,7 +761,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -779,6 +783,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { func TestGetObserverActivity_RawPathEmpty(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -789,7 +794,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -804,6 +809,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) { func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -814,7 +820,7 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -826,13 +832,14 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { func TestGetObserverActivity_UnknownObserver(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000002") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). Return(sqlc.Observer{}, pgx.ErrNoRows) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour, time.Time{}) if !errors.Is(err, pgx.ErrNoRows) { t.Fatalf("expected pgx.ErrNoRows, got %v", err) } @@ -858,6 +865,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { t.Run(tc.name, func(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -868,7 +876,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -882,6 +890,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") window, interval := 24*time.Hour, 6*time.Hour @@ -897,12 +906,12 @@ func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) - before := time.Now().Add(-window) + before := time.Now().UTC().Truncate(interval).Add(-window) store := &Store{q: mock} - if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval); err != nil { + if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval, time.Time{}); err != nil { t.Fatalf("unexpected error: %v", err) } - after := time.Now().Add(-window) + after := time.Now().UTC().Truncate(interval).Add(-window) if gotSince.UnixNano()%int64(interval) != 0 { t.Errorf("since %s is not aligned to %s", gotSince, interval) diff --git a/db/presence.go b/db/presence.go index ea2d6fe..4122266 100644 --- a/db/presence.go +++ b/db/presence.go @@ -20,11 +20,12 @@ func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time }) } -func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error { +func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error { return s.q.TouchObserverBrokers(ctx, sqlc.TouchObserverBrokersParams{ Column1: ids, Column2: brokers, Column3: toTimestamptzs(seen), + Column4: nullableTimes(packets), }) } @@ -42,3 +43,11 @@ func toTimestamptzs(ts []time.Time) []pgtype.Timestamptz { } return out } + +func nullableTimes(ts []time.Time) []pgtype.Timestamptz { + out := make([]pgtype.Timestamptz, len(ts)) + for i, t := range ts { + out[i] = pgtype.Timestamptz{Time: t, Valid: !t.IsZero()} + } + return out +} diff --git a/db/queries/observer_activity_summary.sql b/db/queries/observer_activity_summary.sql new file mode 100644 index 0000000..55b5aed --- /dev/null +++ b/db/queries/observer_activity_summary.sql @@ -0,0 +1,13 @@ +-- name: GetObserverActivityLiveSummary :one +-- Two indexed ranges, bounded to one observer; no legacy presence counters. +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = @observer_id::uuid + AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 29690a1..aaf0be8 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -275,14 +275,14 @@ SELECT COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz GROUP BY bucket ORDER BY bucket; -- name: GetObserverActivityRawPayloadTypes :many SELECT payload_type, COUNT(*)::bigint AS count FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -299,14 +299,14 @@ SELECT COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz GROUP BY 1 ORDER BY 1; -- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -363,19 +363,20 @@ RETURNING o.id; -- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN @is_packet::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW(); + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at); -- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index aeae62b..cf80ee9 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -source=db/sqlc/querier.go -destination=db/sqlc/mock/querier.go -package=mockdb +// mockgen -source db/sqlc/querier.go -destination db/sqlc/mock/querier.go -package mockdb // // Package mockdb is a generated GoMock package. @@ -384,6 +384,21 @@ func (mr *MockQuerierMockRecorder) GetObserverActivityHourlyPayloadTypes(ctx, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityHourlyPayloadTypes", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityHourlyPayloadTypes), ctx, arg) } +// GetObserverActivityLiveSummary mocks base method. +func (m *MockQuerier) GetObserverActivityLiveSummary(ctx context.Context, arg db.GetObserverActivityLiveSummaryParams) (db.GetObserverActivityLiveSummaryRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityLiveSummary", ctx, arg) + ret0, _ := ret[0].(db.GetObserverActivityLiveSummaryRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityLiveSummary indicates an expected call of GetObserverActivityLiveSummary. +func (mr *MockQuerierMockRecorder) GetObserverActivityLiveSummary(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityLiveSummary", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityLiveSummary), ctx, arg) +} + // GetObserverActivityRaw mocks base method. func (m *MockQuerier) GetObserverActivityRaw(ctx context.Context, arg db.GetObserverActivityRawParams) ([]db.GetObserverActivityRawRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 2c6e20f..ebf9293 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -32,7 +32,7 @@ type AnalyticsLiveHourlyIataStat struct { type AnalyticsLiveObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` - PayloadType *int16 `json:"payload_type"` + PayloadType int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs float32 `json:"airtime_ms"` diff --git a/db/sqlc/observer_activity_summary.sql.go b/db/sqlc/observer_activity_summary.sql.go new file mode 100644 index 0000000..c6c97ac --- /dev/null +++ b/db/sqlc/observer_activity_summary.sql.go @@ -0,0 +1,52 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: observer_activity_summary.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = $1::uuid + AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly +` + +type GetObserverActivityLiveSummaryParams struct { + ObserverID uuid.UUID `json:"observer_id"` + GeneratedAt pgtype.Timestamptz `json:"generated_at"` + HourStart pgtype.Timestamptz `json:"hour_start"` + HourEnd pgtype.Timestamptz `json:"hour_end"` +} + +type GetObserverActivityLiveSummaryRow struct { + LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"` + LastCompleteHour int64 `json:"last_complete_hour"` +} + +// Two indexed ranges, bounded to one observer; no legacy presence counters. +func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) { + row := q.db.QueryRow(ctx, getObserverActivityLiveSummary, + arg.ObserverID, + arg.GeneratedAt, + arg.HourStart, + arg.HourEnd, + ) + var i GetObserverActivityLiveSummaryRow + err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour) + return i, err +} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 38e6c97..8f96cbe 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -58,6 +58,8 @@ type Querier interface { // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) + // Two indexed ranges, bounded to one observer; no legacy presence counters. + GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) // Sub-hour activity buckets straight off idx_observations_observer; no join to packets. // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 6a0eee1..3dd6de8 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -650,7 +650,7 @@ SELECT COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $4::timestamptz GROUP BY 1 ORDER BY 1 ` @@ -659,6 +659,7 @@ type GetObserverActivityHourlyParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityHourlyRow struct { @@ -675,7 +676,12 @@ type GetObserverActivityHourlyRow struct { // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityHourly, arg.ObserverID, arg.Column2, arg.Column3) + rows, err := q.db.Query(ctx, getObserverActivityHourly, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + ) if err != nil { return nil, err } @@ -707,7 +713,7 @@ func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserver const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -715,6 +721,7 @@ ORDER BY count DESC type GetObserverActivityHourlyPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityHourlyPayloadTypesRow struct { @@ -723,7 +730,7 @@ type GetObserverActivityHourlyPayloadTypesRow struct { } func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2) + rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) if err != nil { return nil, err } @@ -754,7 +761,7 @@ SELECT COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $4::timestamptz GROUP BY bucket ORDER BY bucket ` @@ -763,6 +770,7 @@ type GetObserverActivityRawParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityRawRow struct { @@ -781,7 +789,12 @@ type GetObserverActivityRawRow struct { // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverActivityRawParams) ([]GetObserverActivityRawRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityRaw, arg.ObserverID, arg.Column2, arg.Column3) + rows, err := q.db.Query(ctx, getObserverActivityRaw, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + ) if err != nil { return nil, err } @@ -813,7 +826,7 @@ func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverAct const getObserverActivityRawPayloadTypes = `-- name: GetObserverActivityRawPayloadTypes :many SELECT payload_type, COUNT(*)::bigint AS count FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -821,6 +834,7 @@ ORDER BY count DESC type GetObserverActivityRawPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityRawPayloadTypesRow struct { @@ -829,7 +843,7 @@ type GetObserverActivityRawPayloadTypesRow struct { } func (q *Queries) GetObserverActivityRawPayloadTypes(ctx context.Context, arg GetObserverActivityRawPayloadTypesParams) ([]GetObserverActivityRawPayloadTypesRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2) + rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) if err != nil { return nil, err } @@ -4422,11 +4436,12 @@ func (q *Queries) SetPacketDecrypted(ctx context.Context, packetHash []byte) err const touchObserverBrokers = `-- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name ` @@ -4435,10 +4450,16 @@ type TouchObserverBrokersParams struct { Column1 []uuid.UUID `json:"column_1"` Column2 []string `json:"column_2"` Column3 []pgtype.Timestamptz `json:"column_3"` + Column4 []pgtype.Timestamptz `json:"column_4"` } func (q *Queries) TouchObserverBrokers(ctx context.Context, arg TouchObserverBrokersParams) error { - _, err := q.db.Exec(ctx, touchObserverBrokers, arg.Column1, arg.Column2, arg.Column3) + _, err := q.db.Exec(ctx, touchObserverBrokers, + arg.Column1, + arg.Column2, + arg.Column3, + arg.Column4, + ) return err } @@ -4943,22 +4964,23 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe const upsertObserverBroker = `-- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN $3::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW() + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at) ` type UpsertObserverBrokerParams struct { ObserverID uuid.UUID `json:"observer_id"` BrokerName string `json:"broker_name"` + IsPacket bool `json:"is_packet"` } // ============================================================ // OBSERVER BROKERS // ============================================================ func (q *Queries) UpsertObserverBroker(ctx context.Context, arg UpsertObserverBrokerParams) error { - _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName) + _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName, arg.IsPacket) return err } diff --git a/docs/docs.go b/docs/docs.go index 43ed843..bc88fbf 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1468,6 +1468,12 @@ const docTemplate = `{ "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", "name": "interval", "in": "query" + }, + { + "type": "integer", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "name": "until", + "in": "query" } ], "responses": { @@ -3924,6 +3930,7 @@ const docTemplate = `{ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -3979,6 +3986,10 @@ const docTemplate = `{ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { "type": "object", "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, "interval": { "type": "string" }, @@ -3999,6 +4010,21 @@ const docTemplate = `{ }, "range": { "type": "string" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive complete-bucket end, epoch ms", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" } } }, @@ -4046,6 +4072,26 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 8a83f4c..75a8e5a 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1466,6 +1466,12 @@ "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", "name": "interval", "in": "query" + }, + { + "type": "integer", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "name": "until", + "in": "query" } ], "responses": { @@ -3922,6 +3928,7 @@ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -3977,6 +3984,10 @@ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { "type": "object", "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, "interval": { "type": "string" }, @@ -3997,6 +4008,21 @@ }, "range": { "type": "string" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive complete-bucket end, epoch ms", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" } } }, @@ -4044,6 +4070,26 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f03b978..cab6df7 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -521,6 +521,7 @@ definitions: description: epoch ms type: integer observationCount: + description: legacy cumulative presence counter; includes non-packet events type: integer observerType: description: e.g. "meshcoretomqtt", "meshcoreha" @@ -560,6 +561,9 @@ definitions: type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity: properties: + generatedAt: + description: response computation time, not proof of continuous coverage + type: integer interval: type: string payloadTypes: @@ -574,6 +578,17 @@ definitions: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio' range: type: string + source: + description: raw or hourly; missing records do not prove an outage + type: string + summary: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary' + windowEnd: + description: exclusive complete-bucket end, epoch ms + type: integer + windowStart: + description: inclusive complete-bucket start, epoch ms + type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint: properties: @@ -604,6 +619,19 @@ definitions: sf: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary: + properties: + lastCompleteHour: + type: integer + lastCompleteHourEnd: + type: integer + lastCompleteHourStart: + type: integer + latestRecordedAt: + type: integer + recordedPackets: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker: properties: lastPacketAt: @@ -2401,6 +2429,11 @@ paths: in: query name: interval type: string + - description: Optional exclusive end in epoch milliseconds; aligned down to + a complete bucket, at most 30 days old + in: query + name: until + type: integer produces: - application/json responses: diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 43147b5..15d52d6 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -256,6 +256,7 @@ var activityIntervals = map[string]time.Duration{ // @Param observerId path string true "Observer UUID" // @Param range query string false "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h" // @Param interval query string false "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)" +// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old" // @Success 200 {object} api.ObserverActivity // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError @@ -295,7 +296,17 @@ func getObserverActivity(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "range/interval exceeds 1000 buckets") return } - activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval) + until := time.Time{} + if values, ok := r.URL.Query()["until"]; ok { + n, err := strconv.ParseInt(values[0], 10, 64) + now := time.Now() + if len(values) != 1 || err != nil || n < 0 || n > now.UnixMilli() || n < now.Add(-720*time.Hour).UnixMilli() { + respondError(w, http.StatusBadRequest, "until must be one past epoch-millisecond timestamp within 30 days") + return + } + until = time.UnixMilli(n) + } + activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval, until) if err != nil { if errors.Is(err, pgx.ErrNoRows) { respondError(w, http.StatusNotFound, "observer not found") diff --git a/internal/api/handlers/observers_test.go b/internal/api/handlers/observers_test.go index 1bf865f..cfef1c9 100644 --- a/internal/api/handlers/observers_test.go +++ b/internal/api/handlers/observers_test.go @@ -200,7 +200,7 @@ func TestGetObserverActivity_Defaults(t *testing.T) { var gotWindow, gotInterval time.Duration r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -232,7 +232,7 @@ func TestGetObserverActivity_CustomRangeAndInterval(t *testing.T) { var gotWindow, gotInterval time.Duration r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -264,7 +264,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { t.Fatal("reader should not be called") return nil, nil }, @@ -286,7 +286,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { var gotWindow, gotInterval time.Duration ok := chi.NewRouter() ok.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -309,6 +309,10 @@ func TestGetObserverActivity_BadRequests(t *testing.T) { query string }{ {"invalid uuid", "not-a-uuid", ""}, + {"bad end", "00000000-0000-0000-0000-000000000001", "?until=banana"}, + {"negative end", "00000000-0000-0000-0000-000000000001", "?until=-1"}, + {"duplicate end", "00000000-0000-0000-0000-000000000001", "?until=1&until=2"}, + {"future end", "00000000-0000-0000-0000-000000000001", "?until=253402300799999"}, {"unparseable range", "00000000-0000-0000-0000-000000000001", "?range=banana"}, {"range too long", "00000000-0000-0000-0000-000000000001", "?range=721h"}, {"zero range", "00000000-0000-0000-0000-000000000001", "?range=0"}, @@ -319,7 +323,7 @@ func TestGetObserverActivity_BadRequests(t *testing.T) { t.Run(tt.name, func(t *testing.T) { r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { t.Fatal("reader should not be called") return nil, nil }, @@ -338,7 +342,7 @@ func TestGetObserverActivity_NotFound(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, pgx.ErrNoRows }, })) @@ -364,7 +368,7 @@ func TestGetObserverActivity_ReaderError(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, errors.New("boom") }, })) @@ -381,7 +385,7 @@ func TestGetObserverActivity_NullsSerialise(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return &api.ObserverActivity{ PayloadTypes: []api.PayloadBreakdownItem{}, Points: []api.ObserverActivityPoint{{T: 1757376000000, Observations: 3}}, diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 42200c1..004d94c 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -32,7 +32,7 @@ type stubReader struct { getObserver func(ctx context.Context, observerID uuid.UUID) (*api.Observer, error) getObserverTelemetry func(ctx context.Context, observerID uuid.UUID, since, until time.Time, afterID int64) (*api.ObserverTelemetry, error) getObserverTelemetryBucketed func(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]api.ObserverTelemetryPoint, error) - getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) + getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) getObserverScopes func(ctx context.Context, observerID uuid.UUID) ([]string, error) listObserverAdverts func(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) listNodes func(ctx context.Context, nodeType int16, iatas []string, supportsMultibytePaths, supportsMultibyteTraces *bool, pubkey []byte, pubkeyPrefix, name, scope string, cursor int64, limit int32, includeNeighbors bool) (api.Page[api.NodeSummary], error) @@ -186,9 +186,9 @@ func (s stubReader) GetObserverTelemetryBucketed(ctx context.Context, observerID return nil, nil } -func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { +func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { if s.getObserverActivity != nil { - return s.getObserverActivity(ctx, observerID, window, interval) + return s.getObserverActivity(ctx, observerID, window, interval, until) } return nil, nil } diff --git a/internal/api/observers.go b/internal/api/observers.go index 206387b..d16fb13 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -44,8 +44,8 @@ type Observer struct { LastStatusAt *int64 `json:"lastStatusAt,omitempty"` // epoch ms FirstSeen int64 `json:"firstSeen"` // epoch ms LastSeen int64 `json:"lastSeen"` // epoch ms - ObservationCount int64 `json:"observationCount"` - Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on + ObservationCount int64 `json:"observationCount"` // legacy cumulative presence counter; includes non-packet events + Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on } // ObserverTelemetryPoint is a single telemetry snapshot for an observer. @@ -89,7 +89,22 @@ type ObserverActivityPoint struct { } // ObserverActivity is the per-observer heard-activity response. +// ObserverActivitySummary describes stored packet records, never MQTT presence events. +type ObserverActivitySummary struct { + RecordedPackets int64 `json:"recordedPackets"` + LastCompleteHour int64 `json:"lastCompleteHour"` + LastCompleteHourStart int64 `json:"lastCompleteHourStart"` + LastCompleteHourEnd int64 `json:"lastCompleteHourEnd"` + LatestRecordedAt *int64 `json:"latestRecordedAt"` +} + type ObserverActivity struct { + WindowStart int64 `json:"windowStart"` // inclusive complete-bucket start, epoch ms + WindowEnd int64 `json:"windowEnd"` // exclusive complete-bucket end, epoch ms + GeneratedAt int64 `json:"generatedAt"` // response computation time, not proof of continuous coverage + Source string `json:"source"` // raw or hourly; missing records do not prove an outage + Summary *ObserverActivitySummary `json:"summary,omitempty"` + Range string `json:"range"` Interval string `json:"interval"` Radio *ObserverActivityRadio `json:"radio"` diff --git a/internal/api/reader.go b/internal/api/reader.go index 8aa3f1a..c9a424d 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -100,7 +100,7 @@ type Reader interface { // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. // interval >= 1h is served from the hourly rollup. Returns pgx.ErrNoRows for an unknown observer. // Range and Interval on the result are left empty for the handler to fill. - GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*ObserverActivity, error) + GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*ObserverActivity, error) // GetObserverScopes returns the names of all transport scopes an observer has // been seen forwarding packets for, ordered alphabetically. diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index e58f054..21d6f3d 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -148,7 +148,7 @@ func (s *stubReader) GetObserverTelemetryBucketed(_ context.Context, _ uuid.UUID return nil, nil } -func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { +func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, nil } diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 65551da..2d31ba4 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -372,10 +372,10 @@ func (cr *CachedReader) GetObserverTelemetryBucketed(ctx context.Context, observ } // GetObserverActivity implements [api.Reader]. -func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { - key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() +func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { + key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano) return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) { - return cr.inner.GetObserverActivity(ctx, observerID, window, interval) + return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until) }) } diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 998c205..0c4d7f8 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -89,7 +89,7 @@ type DB interface { UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) // UpsertObserverBroker records that this observer was seen on brokerName. - UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error + UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error // UpsertIATA auto-creates an iata_codes row if it doesn't exist yet. UpsertIATA(ctx context.Context, iata string) error diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index ea206c5..2090f0e 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -201,8 +201,10 @@ func (s *stubDB) SetNodeCapability(_ context.Context, nodeID uuid.UUID, paths, t func (s *stubDB) UpsertObserver(_ context.Context, _ []byte) (uuid.UUID, string, error) { return uuid.Nil, "", nil } -func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string) error { return nil } -func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } +func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, _ bool) error { + return nil +} +func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } func (s *stubDB) UpsertPacket(_ context.Context, _ UpsertPacketParams) (bool, error) { return false, nil } diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go index f96a32a..3def0c9 100644 --- a/internal/ingest/neighbors.go +++ b/internal/ingest/neighbors.go @@ -48,7 +48,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra w.log.Error(fmt.Sprintf("db: upsert observer failed in neighbors from %s", pubkeyHex), "error", err) return } - if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName); err != nil { + if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName, false); err != nil { w.log.Error(fmt.Sprintf("db: upsert observer broker failed in neighbors from %s", pubkeyHex), "error", err) } diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 92f3da7..bcc9ca5 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -324,7 +324,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ w.log.Error(fmt.Sprintf("db: upsert observer failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } - err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName) + err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, true) if err != nil { w.log.Error(fmt.Sprintf("db: update observer broker failed with packet from %s/%s", iata, pubkeyHex), "error", err) return diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 31c8064..3c693db 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -87,7 +87,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) if w.onObserverUpsert != nil { w.onObserverUpsert(ctx, id) } - if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName); err != nil { + if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, false); err != nil { w.log.Error(fmt.Sprintf("db: upsert observer broker failed in status from %s", pubkeyHex), "error", err) } params := UpdateObserverStatusParams{ diff --git a/internal/presence/broker_freshness_test.go b/internal/presence/broker_freshness_test.go new file mode 100644 index 0000000..887a6e7 --- /dev/null +++ b/internal/presence/broker_freshness_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package presence + +import ( + "context" + "github.com/google/uuid" + "testing" + "time" +) + +type brokerTestStore struct { + Store + initial []bool + seen, packets []time.Time +} + +func (s *brokerTestStore) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, packet bool) error { + s.initial = append(s.initial, packet) + return nil +} +func (s *brokerTestStore) TouchObserverBrokers(_ context.Context, _ []uuid.UUID, _ []string, seen, packets []time.Time) error { + s.seen = seen + s.packets = packets + return nil +} + +func TestBrokerPresenceDoesNotInventPacketFreshness(t *testing.T) { + ctx := context.Background() + store := &brokerTestStore{} + c := New(store, time.Second, time.Minute) + now := time.Now() + c.now = func() time.Time { return now } + id := uuid.New() + c.UpsertObserverBroker(ctx, id, "one", false) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if len(store.initial) != 1 || store.initial[0] || !store.packets[0].IsZero() { + t.Fatal("status-only activity invented a packet") + } + now = now.Add(time.Minute) + packetTime := now + c.UpsertObserverBroker(ctx, id, "one", true) + now = now.Add(time.Minute) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if !store.seen[0].Equal(now) || !store.packets[0].Equal(packetTime) { + t.Fatal("later status changed packet freshness") + } +} diff --git a/internal/presence/coalescer.go b/internal/presence/coalescer.go index 9315f43..517f121 100644 --- a/internal/presence/coalescer.go +++ b/internal/presence/coalescer.go @@ -31,7 +31,7 @@ type Store interface { // TouchObserverBrokers applies coalesced last_seen/last_packet_at bumps // for the given (observer, broker) pairs in one statement. - TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error + TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error // TouchPackets applies coalesced last_heard_at bumps for the given packet // hashes in one statement. @@ -51,6 +51,8 @@ type observerBump struct { count int32 } +type brokerBump struct{ seen, packet time.Time } + type brokerKey struct { id uuid.UUID broker string @@ -68,7 +70,7 @@ type Coalescer struct { identities map[string]identity // pubkey -> observer row dirtyObservers map[uuid.UUID]observerBump knownBrokers map[brokerKey]struct{} - dirtyBrokers map[brokerKey]time.Time + dirtyBrokers map[brokerKey]brokerBump knownIATAs map[string]struct{} packetsSeen map[string]time.Time // hash -> last observation dirtyPackets map[string]time.Time @@ -86,7 +88,7 @@ func New(store Store, flushInterval, packetTTL time.Duration) *Coalescer { identities: make(map[string]identity), dirtyObservers: make(map[uuid.UUID]observerBump), knownBrokers: make(map[brokerKey]struct{}), - dirtyBrokers: make(map[brokerKey]time.Time), + dirtyBrokers: make(map[brokerKey]brokerBump), knownIATAs: make(map[string]struct{}), packetsSeen: make(map[string]time.Time), dirtyPackets: make(map[string]time.Time), @@ -122,17 +124,22 @@ func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUI // UpsertObserverBroker writes through the first time a pair is seen and // records a bump afterwards. -func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { key := brokerKey{id: observerID, broker: brokerName} c.mu.Lock() if _, ok := c.knownBrokers[key]; ok { - c.dirtyBrokers[key] = c.now() + bump := c.dirtyBrokers[key] + bump.seen = c.now() + if isPacket { + bump.packet = bump.seen + } + c.dirtyBrokers[key] = bump c.mu.Unlock() return nil } c.mu.Unlock() - if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName); err != nil { + if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName, isPacket); err != nil { return err } c.mu.Lock() @@ -248,7 +255,7 @@ func (c *Coalescer) Flush(ctx context.Context) { brokers := c.dirtyBrokers packets := c.dirtyPackets c.dirtyObservers = make(map[uuid.UUID]observerBump) - c.dirtyBrokers = make(map[brokerKey]time.Time) + c.dirtyBrokers = make(map[brokerKey]brokerBump) c.dirtyPackets = make(map[string]time.Time) cutoff := c.now().Add(-c.packetTTL) for key, seen := range c.packetsSeen { @@ -266,12 +273,14 @@ func (c *Coalescer) Flush(ctx context.Context) { ids := make([]uuid.UUID, 0, len(brokers)) names := make([]string, 0, len(brokers)) seen := make([]time.Time, 0, len(brokers)) + packetTimes := make([]time.Time, 0, len(brokers)) for key, ts := range brokers { ids = append(ids, key.id) names = append(names, key.broker) - seen = append(seen, ts) + seen = append(seen, ts.seen) + packetTimes = append(packetTimes, ts.packet) } - if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen); err != nil { + if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen, packetTimes); err != nil { slog.Error(fmt.Sprintf("presence: flush observer brokers failed (%d rows dropped)", len(ids)), "component", "presence", "error", err) } } diff --git a/sqlc.yaml b/sqlc.yaml index 032c991..b5e86b8 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -30,3 +30,6 @@ sql: go_type: {type: "string", pointer: true} - column: "mv_top_advertisers_by_iata.name" go_type: {type: "string", pointer: true} + + - column: "mv_observer_activity_hourly.payload_type" + go_type: {type: "int16", pointer: true} From 8b66a534051f2bf7d2c25f16aa042239f34d420b Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 22:34:58 -0400 Subject: [PATCH 3/9] fix(observers): keep non-finite signal samples out of JSON --- db/observer_metrics_integration_test.go | 6 +++++- db/observers.go | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 868ce6c..f4e114e 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -3,6 +3,7 @@ package db import ( + "encoding/json" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -30,7 +31,7 @@ func TestObserverMetricsPostgres(t *testing.T) { INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) - SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,0,-100 FROM packets; + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash=int4send(2) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; `); err != nil { t.Fatal(err) } @@ -48,6 +49,9 @@ func TestObserverMetricsPostgres(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := json.Marshal(activity); err != nil { + t.Fatalf("non-finite sample leaked into JSON: %v", err) + } if activity.Summary.RecordedPackets != 3 || len(activity.PayloadTypes) != 2 || activity.Summary.LatestRecordedAt != nil || activity.Summary.LastCompleteHour != 0 { t.Fatalf("archive summary: %+v types=%+v", activity.Summary, activity.PayloadTypes) } diff --git a/db/observers.go b/db/observers.go index 1735042..09a377a 100644 --- a/db/observers.go +++ b/db/observers.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "fmt" "log/slog" + "math" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -296,7 +297,12 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w if r.SnrN > 0 { avg := r.SnrSum / float32(r.SnrN) min := r.SnrMin - p.SNRAvg, p.SNRMin = &avg, &min + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } } if r.RssiN > 0 { avg := float32(r.RssiSum) / float32(r.RssiN) @@ -346,7 +352,12 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w } if r.SnrN > 0 { avg, min := r.SnrAvg, r.SnrMin - p.SNRAvg, p.SNRMin = &avg, &min + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } } if r.RssiN > 0 { avg := r.RssiAvg From 39c9d185087dcf6bc5aede8f8d1e86c61c1365c9 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 22:39:21 -0400 Subject: [PATCH 4/9] test(observers): exercise non-finite raw and archived samples --- db/observer_metrics_integration_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index f4e114e..6266dff 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -31,7 +31,7 @@ func TestObserverMetricsPostgres(t *testing.T) { INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) - SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash=int4send(2) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash IN (int4send(1),int4send(2)) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; `); err != nil { t.Fatal(err) } @@ -59,7 +59,7 @@ func TestObserverMetricsPostgres(t *testing.T) { t.Fatalf("window: %+v", activity) } if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,last_heard_at) VALUES ('\xaa',NOW()); - INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,0,-100); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,'NaN'::real,-100); INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW()) ON CONFLICT DO NOTHING;`); err != nil { t.Fatal(err) } @@ -67,6 +67,9 @@ func TestObserverMetricsPostgres(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := json.Marshal(raw); err != nil { + t.Fatalf("raw non-finite sample leaked into JSON: %v", err) + } if raw.Summary.RecordedPackets != 1 || raw.Summary.LastCompleteHour != 1 || raw.Summary.LatestRecordedAt == nil || raw.PayloadTypes[0].PayloadType != -1 { t.Fatalf("raw/duplicate: %+v", raw) } From e78010b94e52d3a7506312f7ee53c520df1aeea6 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 14:40:01 -0400 Subject: [PATCH 5/9] feat(routes): add bounded retained observation evidence --- .github/workflows/ci.yml | 2 +- README.md | 10 + db/migrations/041_route_evidence_index.sql | 9 + db/queries/route_evidence.sql | 22 ++ db/route_evidence.go | 93 +++++++++ db/route_evidence_integration_test.go | 205 +++++++++++++++++++ db/route_evidence_test.go | 23 +++ db/routes.go | 6 +- db/sqlc/mock/querier.go | 30 +++ db/sqlc/querier.go | 5 + db/sqlc/route_evidence.sql.go | 125 +++++++++++ docs/docs.go | 166 +++++++++++++++ docs/swagger.json | 166 +++++++++++++++ docs/swagger.yaml | 116 +++++++++++ internal/api/handlers/route_evidence.go | 118 +++++++++++ internal/api/handlers/route_evidence_test.go | 69 +++++++ internal/api/handlers/routes.go | 1 + internal/api/handlers/stub_reader_test.go | 8 + internal/api/reader.go | 2 + internal/api/route_evidence.go | 110 ++++++++++ internal/api/route_evidence_test.go | 25 +++ internal/api/routes.go | 1 + internal/cache/cache_test.go | 6 +- internal/cache/reader.go | 5 + 24 files changed, 1319 insertions(+), 4 deletions(-) create mode 100644 db/migrations/041_route_evidence_index.sql create mode 100644 db/queries/route_evidence.sql create mode 100644 db/route_evidence.go create mode 100644 db/route_evidence_integration_test.go create mode 100644 db/route_evidence_test.go create mode 100644 db/sqlc/route_evidence.sql.go create mode 100644 internal/api/handlers/route_evidence.go create mode 100644 internal/api/handlers/route_evidence_test.go create mode 100644 internal/api/route_evidence.go create mode 100644 internal/api/route_evidence_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9d71ab..da907fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index 33ae4e1..3d123cc 100644 --- a/README.md +++ b/README.md @@ -566,3 +566,13 @@ period packet total. Broker presence and packet-arrival timestamps are now updated separately; this cannot reconstruct previously overwritten timestamps. Migration 040 repairs archived unknown-type counts from the all-payload observer rollup; radio samples already discarded for those legacy rows remain unknown. + +### Saved-route observation evidence + +Known-route responses include `pathKey`, a stable identity within the route's IATA. Use `GET /api/v1/routes/{iata}/{pathKey}/observations` to fetch the **full saved route** and retained report references. Search results can contain a subsegment while sharing the full route's key; the evidence response always describes the complete saved sequence. + +The match requires the complete saved `pathBytes`, `hashSize`, hop count and IATA. A compact digest index narrows candidates, but full bytes are still compared. Other hash widths, TRACE readings/intended routes and unclassified legacy observations are excluded. Matching short prefixes does not confirm historical node identities, forwarding or delivery. The stored route counter can include repeated processing and outlive raw reports; it is not a retained-result total. + +`range` defaults to `24h` and accepts durations up to `720h`, anchored on the server. Alternatively supply both `since` and exclusive `until` in epoch milliseconds, with a maximum 30-day span and no future end. `limit` defaults to 50 and is capped at 200. Follow `nextPageCursor` as `pageCursor`; its route, window and microsecond/ID boundary are pinned. Do not combine it with another range, or change its explicit window. Numeric legacy `cursor` is unsupported. Responses include effective window bounds, `matchAvailable`, an empty `items` array when no matching raw evidence remains, and `hasMore`; no total-count scan or packet/message body is added. Malformed saved path metadata is explicitly unavailable, and missing routes return 404. + +Migration 041 builds the compact observation index concurrently. Keep it as a single statement outside a transaction; the existing runner handles an interrupted or already-built index before recording completion. It does not alter retained rows or expiry configuration. Native PostgreSQL tests cover ties below millisecond precision, cursor scope, different widths/sites, TRACE/unknown exclusions, raw expiry, index retry and custom/generic indexed plans. diff --git a/db/migrations/041_route_evidence_index.sql b/db/migrations/041_route_evidence_index.sql new file mode 100644 index 0000000..5b614da --- /dev/null +++ b/db/migrations/041_route_evidence_index.sql @@ -0,0 +1,9 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- One statement outside a transaction. RunMigrations recovers interrupted builds; +-- avoid IF NOT EXISTS, which would silently accept an invalid index. +-- A compact digest narrows candidates; queries still compare the complete bytes. +CREATE INDEX CONCURRENTLY idx_observations_route_evidence +ON packet_observations (iata, hash_size, (decode(md5(path_bytes), 'hex')), heard_at DESC, id DESC) +WHERE path_bytes IS NOT NULL AND payload_type IS NOT NULL AND payload_type <> 9; diff --git a/db/queries/route_evidence.sql b/db/queries/route_evidence.sql new file mode 100644 index 0000000..960b95e --- /dev/null +++ b/db/queries/route_evidence.sql @@ -0,0 +1,22 @@ +-- name: GetRouteEvidenceRoute :one +SELECT * FROM known_routes +WHERE iata = @iata::bpchar AND path_key = @path_key::bytea; + +-- name: ListRouteEvidence :many +-- Leading index equalities and the time/ID boundary bound both custom and generic plans. +-- Full-byte equality is required even when the compact digest matches. TRACE path bytes +-- carry readings; unclassified legacy observations cannot be safely called ordinary paths. +SELECT po.id, po.packet_hash, po.observer_id, o.display_name AS observer_name, + po.heard_at, po.payload_type, po.rssi, po.snr +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.iata = @iata::bpchar + AND po.hash_size = @hash_size::smallint + AND decode(md5(po.path_bytes), 'hex') = @path_digest::bytea + AND po.path_bytes = @path_bytes::bytea + AND po.hop_count = @hop_count::smallint + AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 + AND po.heard_at >= @since::timestamptz AND po.heard_at < @until::timestamptz + AND (po.heard_at, po.id) < (@before_at::timestamptz, @before_id::bigint) +ORDER BY po.heard_at DESC, po.id DESC +LIMIT @page_limit::integer; diff --git a/db/route_evidence.go b/db/route_evidence.go new file mode 100644 index 0000000..fc7209e --- /dev/null +++ b/db/route_evidence.go @@ -0,0 +1,93 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "math" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgtype" +) + +func savedRoutePath(prefixes [][]byte, count int32) (int16, []byte, bool) { + if count < 2 || count > 63 || len(prefixes) != int(count) { + return 0, nil, false + } + width := len(prefixes[0]) + if width < 1 || width > 3 { + return 0, nil, false + } + for _, p := range prefixes { + if len(p) != width { + return 0, nil, false + } + } + return int16(width), bytes.Join(prefixes, nil), true +} + +func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + if !api.ValidRouteEvidenceKey(iata, key) || !api.ValidRouteEvidenceWindow(query.Since, query.Until) || query.Limit < 1 || query.Limit > 200 { + return nil, api.ErrRouteEvidenceInput + } + if c := query.Cursor; c != nil { + if c.IATA != iata || c.PathKey != key || !c.Since.Equal(query.Since) || !c.Until.Equal(query.Until) || c.ID <= 0 || c.HeardAt.Before(query.Since) || !c.HeardAt.Before(query.Until) { + return nil, api.ErrRouteEvidenceInput + } + } + keyBytes, _ := hex.DecodeString(key) + row, err := s.q.GetRouteEvidenceRoute(ctx, sqlc.GetRouteEvidenceRouteParams{Iata: iata, PathKey: keyBytes}) + if err != nil { + return nil, err + } + nodes, err := s.GetNodesByIDs(ctx, row.NodeIds) + if err != nil { + return nil, err + } + route := toKnownRoutes([]knownRouteRow{{ID: row.ID, NodeIds: row.NodeIds, HashPrefix: row.HashPrefix, Iata: row.Iata, HopCount: row.HopCount, FirstSeen: row.FirstSeen, LastSeen: row.LastSeen, ObservationCount: row.ObservationCount}}, nodes)[0] + route.PathKey = hex.EncodeToString(row.PathKey) + out := &api.RouteEvidence{Page: api.Page[api.RouteObservation]{Items: []api.RouteObservation{}}, Route: route, WindowStart: query.Since.UnixMilli(), WindowEnd: query.Until.UnixMilli(), GeneratedAt: time.Now().UnixMilli(), MatchType: "saved_path_prefixes"} + width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) + if !valid { + return out, nil + } + out.MatchAvailable = true + out.HashSize = width + out.PathBytes = hex.EncodeToString(path) + beforeAt, beforeID := query.Until, int64(math.MaxInt64) + if query.Cursor != nil { + beforeAt, beforeID = query.Cursor.HeardAt, query.Cursor.ID + } + ts := func(t time.Time) pgtype.Timestamptz { return pgtype.Timestamptz{Time: t, Valid: true} } + digest := md5.Sum(path) + rows, err := s.q.ListRouteEvidence(ctx, sqlc.ListRouteEvidenceParams{Iata: iata, HashSize: width, HopCount: int16(row.HopCount), PathDigest: digest[:], PathBytes: path, Since: ts(query.Since), Until: ts(query.Until), BeforeAt: ts(beforeAt), BeforeID: beforeID, PageLimit: query.Limit + 1}) + if err != nil { + return nil, err + } + out.HasMore = len(rows) > int(query.Limit) + if out.HasMore { + rows = rows[:query.Limit] + } + for _, r := range rows { + payload := int16(-1) + if r.PayloadType != nil { + payload = *r.PayloadType + } + snr := r.Snr + if snr != nil && (math.IsNaN(float64(*snr)) || math.IsInf(float64(*snr), 0)) { + snr = nil + } + out.Items = append(out.Items, api.RouteObservation{ID: r.ID, PacketHash: hex.EncodeToString(r.PacketHash), ObserverID: r.ObserverID, ObserverName: r.ObserverName, HeardAt: r.HeardAt.Time.UnixMilli(), PayloadType: payload, PayloadTypeName: api.PayloadTypeName(payload), RSSI: r.Rssi, SNR: snr}) + } + if out.HasMore { + last := rows[len(rows)-1] + cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID}).String() + out.NextPageCursor = &cursor + } + return out, nil +} diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go new file mode 100644 index 0000000..b49224e --- /dev/null +++ b/db/route_evidence_integration_test.go @@ -0,0 +1,205 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +type evidenceCapture struct { + pgx.Tx + statement string + args []any +} + +func (c *evidenceCapture) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + if strings.Contains(sql, "-- name: ListRouteEvidence") { + c.statement = sql + c.args = args + } + return c.Tx.Query(ctx, sql, args...) +} + +func TestRouteEvidencePostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + ddl, err := migrationFiles.ReadFile("migrations/024_known_routes_pathkey.sql") + if err != nil { + t.Fatal(err) + } + text := string(ddl) + start := strings.Index(text, "CREATE TABLE known_routes_new (") + end := start + strings.Index(text[start:], "\n);") + 3 + table := strings.ReplaceAll(text[start:end], "known_routes_new", "known_routes") + _, err = tx.Exec(ctx, `CREATE TABLE iata_codes (iata char(3) PRIMARY KEY); INSERT INTO iata_codes VALUES ('YOW'),('YVR'); + ALTER TABLE nodes ADD COLUMN latitude double precision, ADD COLUMN longitude double precision;`+table+` + INSERT INTO observers(id,display_name) VALUES ('00000000-0000-0000-0000-000000000001','One'),('00000000-0000-0000-0000-000000000002','Two'); + INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa','A'),('00000000-0000-0000-0000-000000000002','\xbb','B'); + INSERT INTO packets(packet_hash) SELECT int4send(i) FROM generate_series(1,10) i;`) + if err != nil { + t.Fatal(err) + } + index, err := migrationFiles.ReadFile("migrations/041_route_evidence_index.sql") + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, strings.Replace(string(index), "CREATE INDEX CONCURRENTLY", "CREATE INDEX", 1)); err != nil { + t.Fatal(err) + } + capture := &evidenceCapture{Tx: tx} + store := &Store{q: sqlc.New(capture)} + ids := []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000001"), uuid.MustParse("00000000-0000-0000-0000-000000000002")} + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } + key := hex.EncodeToString(routePathKey(ids)) + since := time.Now().UTC().Truncate(time.Millisecond).Add(-time.Hour) + until := since.Add(time.Hour) + at := since.Add(time.Minute + 123456*time.Microsecond) + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,snr) + SELECT i,int4send(i), '00000000-0000-0000-0000-000000000001',CASE WHEN i=5 THEN 'YVR' ELSE 'YOW' END, + CASE WHEN i=9 THEN $2::timestamptz WHEN i=1 THEN $1::timestamptz-interval '2 microseconds' ELSE $1::timestamptz END, + CASE WHEN i=4 THEN 2 ELSE 1 END,CASE WHEN i=4 THEN 1 WHEN i=10 THEN 3 ELSE 2 END, + CASE WHEN i=6 THEN '\xabbb'::bytea WHEN i=10 THEN '\xaabbcc'::bytea ELSE '\xaabb'::bytea END, + CASE WHEN i=7 THEN 9 WHEN i=8 THEN NULL ELSE 4 END,CASE WHEN i=1 THEN 'NaN'::real ELSE 0 END FROM generate_series(1,10) i; + `, at, until) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) VALUES (11,int4send(1),'00000000-0000-0000-0000-000000000002','YOW',$1,1,2,'\xaabb',4)`, at.Add(-time.Microsecond)); err != nil { + t.Fatal(err) + } + query := api.RouteEvidenceQuery{Since: since, Until: until, Limit: 2} + first, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil { + t.Fatal(err) + } + if !first.HasMore || first.NextPageCursor == nil || len(first.Items) != 2 || first.Items[0].ID != 3 || first.Items[1].ID != 2 || first.Route.PathKey != key || first.HashSize != 1 || first.PathBytes != "aabb" { + t.Fatalf("first page: %+v", first) + } + query.Cursor, err = api.ParseRouteEvidenceCursor(*first.NextPageCursor) + if err != nil { + t.Fatal(err) + } + second, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil { + t.Fatal(err) + } + if second.HasMore || len(second.Items) != 2 || second.Items[0].ID != 11 || second.Items[1].ID != 1 || second.Items[1].SNR != nil { + t.Fatalf("precise second page: %+v", second) + } + if _, err = json.Marshal(second); err != nil { + t.Fatalf("non-finite JSON: %v", err) + } + query.Cursor.IATA = "YVR" + if _, err = store.GetRouteEvidence(ctx, "YOW", key, query); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatal("cross-route cursor accepted") + } + query.Cursor = nil + if _, err = store.GetRouteEvidence(ctx, "YVR", key, query); !errors.Is(err, pgx.ErrNoRows) { + t.Fatal("unknown route did not return not-found") + } + if _, err = tx.Exec(ctx, "DELETE FROM packets"); err != nil { + t.Fatal(err) + } + empty, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || !empty.MatchAvailable || len(empty.Items) != 0 || empty.HasMore { + t.Fatalf("expired evidence: %+v %v", empty, err) + } + + // Many unrelated paths must not turn this into a raw-table or route-ID scan. + started := time.Now() + _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash) SELECT int4send(i) FROM generate_series(100,200099) i;`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',$1,1,2, + CASE WHEN get_byte(packet_hash,3)=0 THEN '\xaabb'::bytea ELSE packet_hash END,4 FROM packets;`, at) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, "ANALYZE packet_observations; ANALYZE observers;"); err != nil { + t.Fatal(err) + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err = tx.Exec(ctx, "SET LOCAL plan_cache_mode = "+mode); err != nil { + t.Fatal(err) + } + result, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || len(result.Items) != 2 { + t.Fatalf("indexed read: %v", err) + } + var plan string + if err = tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+capture.statement, capture.args...).Scan(&plan); err != nil { + t.Fatal(err) + } + if !strings.Contains(plan, "idx_observations_route_evidence") { + t.Fatalf("missing evidence index: %s", plan) + } + t.Logf("%s plan: %s", mode, plan) + } + t.Logf("200000-row fixture insert/analyze/read: %s", time.Since(started)) + if _, err = tx.Exec(ctx, `UPDATE known_routes SET hash_prefix=ARRAY['\xaa'::bytea]`); err != nil { + t.Fatal(err) + } + unavailable, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || unavailable.MatchAvailable || len(unavailable.Items) != 0 { + t.Fatalf("malformed stored path: %+v %v", unavailable, err) + } +} + +func TestRouteEvidenceIndexPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + schema := fmt.Sprintf("route_evidence_%d", time.Now().UnixNano()) + ident := pgx.Identifier{schema}.Sanitize() + if _, err = conn.Exec(ctx, "CREATE SCHEMA "+ident+"; SET search_path TO "+ident); err != nil { + t.Fatal(err) + } + defer func() { _, _ = conn.Exec(context.Background(), "DROP SCHEMA "+ident+" CASCADE") }() + if _, err = conn.Exec(ctx, `CREATE TABLE packet_observations (id bigint,iata char(3),hash_size smallint,path_bytes bytea,heard_at timestamptz,payload_type smallint)`); err != nil { + t.Fatal(err) + } + migration, err := migrationFiles.ReadFile("migrations/041_route_evidence_index.sql") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err = applyMigration(ctx, conn, string(migration)); err != nil { + t.Fatalf("migration/retry: %v", err) + } + } + if _, err = conn.Exec(ctx, `UPDATE pg_index SET indisvalid=false WHERE indexrelid='idx_observations_route_evidence'::regclass`); err != nil { + t.Fatal(err) + } + if err = applyMigration(ctx, conn, string(migration)); err != nil { + t.Fatal(err) + } + var valid bool + if err = conn.QueryRow(ctx, `SELECT indisvalid FROM pg_index WHERE indexrelid='idx_observations_route_evidence'::regclass`).Scan(&valid); err != nil || !valid { + t.Fatalf("invalid index after recovery: %v", err) + } +} diff --git a/db/route_evidence_test.go b/db/route_evidence_test.go new file mode 100644 index 0000000..868b565 --- /dev/null +++ b/db/route_evidence_test.go @@ -0,0 +1,23 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "bytes" + "testing" +) + +func TestSavedRoutePath(t *testing.T) { + size, path, ok := savedRoutePath([][]byte{{0xaa}, {0xbb}}, 2) + if !ok || size != 1 || !bytes.Equal(path, []byte{0xaa, 0xbb}) { + t.Fatal("valid path lost") + } + for _, prefixes := range [][][]byte{nil, {{1}}, {{1}, {2, 3}}, {{}, {2}}, {{1, 2, 3, 4}, {5, 6, 7, 8}}} { + if _, _, ok := savedRoutePath(prefixes, int32(len(prefixes))); ok { + t.Fatalf("accepted malformed path %x", prefixes) + } + } + if _, _, ok := savedRoutePath([][]byte{{1}, {2}}, 3); ok { + t.Fatal("accepted inconsistent hop count") + } +} diff --git a/db/routes.go b/db/routes.go index 187e07e..71820ed 100644 --- a/db/routes.go +++ b/db/routes.go @@ -118,6 +118,7 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st hops = append(hops, hop) } items = append(items, api.KnownRoute{ + PathKey: hex.EncodeToString(routePathKey(r.NodeIds)), ID: r.ID, IATA: r.Iata, HopCount: int32(len(hops)), @@ -223,7 +224,7 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t return nil, nil } - // 5. find cross-IATA links — nodes at the boundary of source routes + // 5. find cross-IATA links — nodes at the boundary of source routes // that have neighbors in the target IATA at the start of target routes var results []api.CrossIATARoute @@ -249,7 +250,7 @@ func (s *Store) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, t continue } if targetHops, ok := targetNodeSet[neighbor.ID]; ok { - // found a cross-IATA link — build the route + // found a cross-IATA link — build the route sourceSegment := sr.Hops[:i+1] targetSegment := extractFromNode(targetHops, neighbor.ID) @@ -348,6 +349,7 @@ func toKnownRoutes(rows []knownRouteRow, nodes map[uuid.UUID]*api.ResolvedNode) hops = append(hops, hop) } items = append(items, api.KnownRoute{ + PathKey: hex.EncodeToString(routePathKey(r.NodeIds)), ID: r.ID, IATA: r.Iata, HopCount: r.HopCount, diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index cf80ee9..9d3fb18 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -684,6 +684,21 @@ func (mr *MockQuerierMockRecorder) GetRegionIATAs(ctx, regionID any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegionIATAs", reflect.TypeOf((*MockQuerier)(nil).GetRegionIATAs), ctx, regionID) } +// GetRouteEvidenceRoute mocks base method. +func (m *MockQuerier) GetRouteEvidenceRoute(ctx context.Context, arg db.GetRouteEvidenceRouteParams) (db.KnownRoute, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRouteEvidenceRoute", ctx, arg) + ret0, _ := ret[0].(db.KnownRoute) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRouteEvidenceRoute indicates an expected call of GetRouteEvidenceRoute. +func (mr *MockQuerierMockRecorder) GetRouteEvidenceRoute(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRouteEvidenceRoute", reflect.TypeOf((*MockQuerier)(nil).GetRouteEvidenceRoute), ctx, arg) +} + // GetScopeByName mocks base method. func (m *MockQuerier) GetScopeByName(ctx context.Context, name string) (db.GetScopeByNameRow, error) { m.ctrl.T.Helper() @@ -1223,6 +1238,21 @@ func (mr *MockQuerierMockRecorder) ListRegions(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegions", reflect.TypeOf((*MockQuerier)(nil).ListRegions), ctx) } +// ListRouteEvidence mocks base method. +func (m *MockQuerier) ListRouteEvidence(ctx context.Context, arg db.ListRouteEvidenceParams) ([]db.ListRouteEvidenceRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRouteEvidence", ctx, arg) + ret0, _ := ret[0].([]db.ListRouteEvidenceRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRouteEvidence indicates an expected call of ListRouteEvidence. +func (mr *MockQuerierMockRecorder) ListRouteEvidence(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRouteEvidence", reflect.TypeOf((*MockQuerier)(nil).ListRouteEvidence), ctx, arg) +} + // ListTraceTags mocks base method. func (m *MockQuerier) ListTraceTags(ctx context.Context, arg db.ListTraceTagsParams) ([]db.ListTraceTagsRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 8f96cbe..a577064 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -87,6 +87,7 @@ type Querier interface { GetRegion(ctx context.Context, id int32) (GetRegionRow, error) GetRegionBySlug(ctx context.Context, slug string) (GetRegionBySlugRow, error) GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) + GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) GetScopeNames(ctx context.Context) ([]string, error) // Aggregate matching observations once, separately from node memberships to avoid @@ -198,6 +199,10 @@ type Querier interface { // REGIONS // ============================================================ ListRegions(ctx context.Context) ([]ListRegionsRow, error) + // Leading index equalities and the time/ID boundary bound both custom and generic plans. + // Full-byte equality is required even when the compact digest matches. TRACE path bytes + // carry readings; unclassified legacy observations cannot be safely called ordinary paths. + ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) // ============================================================ // TRACES // ============================================================ diff --git a/db/sqlc/route_evidence.sql.go b/db/sqlc/route_evidence.sql.go new file mode 100644 index 0000000..ab7f1ee --- /dev/null +++ b/db/sqlc/route_evidence.sql.go @@ -0,0 +1,125 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: route_evidence.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const getRouteEvidenceRoute = `-- name: GetRouteEvidenceRoute :one +SELECT id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count, last_reconfirmed_at FROM known_routes +WHERE iata = $1::bpchar AND path_key = $2::bytea +` + +type GetRouteEvidenceRouteParams struct { + Iata string `json:"iata"` + PathKey []byte `json:"path_key"` +} + +func (q *Queries) GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) { + row := q.db.QueryRow(ctx, getRouteEvidenceRoute, arg.Iata, arg.PathKey) + var i KnownRoute + err := row.Scan( + &i.ID, + &i.PathKey, + &i.NodeIds, + &i.HashPrefix, + &i.Iata, + &i.HopCount, + &i.FirstSeen, + &i.LastSeen, + &i.ObservationCount, + &i.LastReconfirmedAt, + ) + return i, err +} + +const listRouteEvidence = `-- name: ListRouteEvidence :many +SELECT po.id, po.packet_hash, po.observer_id, o.display_name AS observer_name, + po.heard_at, po.payload_type, po.rssi, po.snr +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.iata = $1::bpchar + AND po.hash_size = $2::smallint + AND decode(md5(po.path_bytes), 'hex') = $3::bytea + AND po.path_bytes = $4::bytea + AND po.hop_count = $5::smallint + AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 + AND po.heard_at >= $6::timestamptz AND po.heard_at < $7::timestamptz + AND (po.heard_at, po.id) < ($8::timestamptz, $9::bigint) +ORDER BY po.heard_at DESC, po.id DESC +LIMIT $10::integer +` + +type ListRouteEvidenceParams struct { + Iata string `json:"iata"` + HashSize int16 `json:"hash_size"` + PathDigest []byte `json:"path_digest"` + PathBytes []byte `json:"path_bytes"` + HopCount int16 `json:"hop_count"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + BeforeAt pgtype.Timestamptz `json:"before_at"` + BeforeID int64 `json:"before_id"` + PageLimit int32 `json:"page_limit"` +} + +type ListRouteEvidenceRow struct { + ID int64 `json:"id"` + PacketHash []byte `json:"packet_hash"` + ObserverID uuid.UUID `json:"observer_id"` + ObserverName *string `json:"observer_name"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + PayloadType *int16 `json:"payload_type"` + Rssi *int16 `json:"rssi"` + Snr *float32 `json:"snr"` +} + +// Leading index equalities and the time/ID boundary bound both custom and generic plans. +// Full-byte equality is required even when the compact digest matches. TRACE path bytes +// carry readings; unclassified legacy observations cannot be safely called ordinary paths. +func (q *Queries) ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) { + rows, err := q.db.Query(ctx, listRouteEvidence, + arg.Iata, + arg.HashSize, + arg.PathDigest, + arg.PathBytes, + arg.HopCount, + arg.Since, + arg.Until, + arg.BeforeAt, + arg.BeforeID, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListRouteEvidenceRow{} + for rows.Next() { + var i ListRouteEvidenceRow + if err := rows.Scan( + &i.ID, + &i.PacketHash, + &i.ObserverID, + &i.ObserverName, + &i.HeardAt, + &i.PayloadType, + &i.Rssi, + &i.Snr, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/docs/docs.go b/docs/docs.go index bc88fbf..ae84dab 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2131,6 +2131,92 @@ const docTemplate = `{ } } }, + "/routes/{iata}/{pathKey}/observations": { + "get": { + "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Get retained reports matching a full saved route prefix sequence", + "parameters": [ + { + "type": "string", + "description": "Three-character IATA code", + "name": "iata", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Stable 32-hex pathKey from a known-route response", + "name": "pathKey", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Window start epoch ms; provide with until (default last 24h)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Exclusive window end epoch ms; maximum span 30d, no future end", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Default 50; positive, capped at 200", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/scopes": { "get": { "produces": [ @@ -3601,6 +3687,10 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" + }, + "pathKey": { + "description": "stable identity within this IATA; use for route evidence links", + "type": "string" } } }, @@ -4844,6 +4934,50 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence": { + "type": "object", + "properties": { + "generatedAt": { + "type": "integer" + }, + "hasMore": { + "type": "boolean" + }, + "hashSize": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation" + } + }, + "matchAvailable": { + "type": "boolean" + }, + "matchType": { + "type": "string" + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + }, + "pathBytes": { + "type": "string" + }, + "route": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + }, + "windowEnd": { + "type": "integer" + }, + "windowStart": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop": { "type": "object", "properties": { @@ -4864,6 +4998,38 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation": { + "type": "object", + "properties": { + "heardAt": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "observerId": { + "type": "string" + }, + "observerName": { + "type": "string" + }, + "packetHash": { + "type": "string" + }, + "payloadType": { + "type": "integer" + }, + "payloadTypeName": { + "type": "string" + }, + "rssi": { + "type": "integer" + }, + "snr": { + "type": "number" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 75a8e5a..482bef9 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2129,6 +2129,92 @@ } } }, + "/routes/{iata}/{pathKey}/observations": { + "get": { + "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Get retained reports matching a full saved route prefix sequence", + "parameters": [ + { + "type": "string", + "description": "Three-character IATA code", + "name": "iata", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Stable 32-hex pathKey from a known-route response", + "name": "pathKey", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Window start epoch ms; provide with until (default last 24h)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Exclusive window end epoch ms; maximum span 30d, no future end", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Default 50; positive, capped at 200", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/scopes": { "get": { "produces": [ @@ -3599,6 +3685,10 @@ }, "observationCount": { "type": "integer" + }, + "pathKey": { + "description": "stable identity within this IATA; use for route evidence links", + "type": "string" } } }, @@ -4842,6 +4932,50 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence": { + "type": "object", + "properties": { + "generatedAt": { + "type": "integer" + }, + "hasMore": { + "type": "boolean" + }, + "hashSize": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation" + } + }, + "matchAvailable": { + "type": "boolean" + }, + "matchType": { + "type": "string" + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + }, + "pathBytes": { + "type": "string" + }, + "route": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + }, + "windowEnd": { + "type": "integer" + }, + "windowStart": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop": { "type": "object", "properties": { @@ -4862,6 +4996,38 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation": { + "type": "object", + "properties": { + "heardAt": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "observerId": { + "type": "string" + }, + "observerName": { + "type": "string" + }, + "packetHash": { + "type": "string" + }, + "payloadType": { + "type": "integer" + }, + "payloadTypeName": { + "type": "string" + }, + "rssi": { + "type": "integer" + }, + "snr": { + "type": "number" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index cab6df7..a7f70de 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -272,6 +272,9 @@ definitions: type: integer observationCount: type: integer + pathKey: + description: stable identity within this IATA; use for route evidence links + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.Node: properties: @@ -1151,6 +1154,35 @@ definitions: description: hex-encoded prefix used for resolution type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence: + properties: + generatedAt: + type: integer + hasMore: + type: boolean + hashSize: + type: integer + items: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation' + type: array + matchAvailable: + type: boolean + matchType: + type: string + nextCursor: + type: integer + nextPageCursor: + type: string + pathBytes: + type: string + route: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute' + windowEnd: + type: integer + windowStart: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop: properties: hashBytes: @@ -1163,6 +1195,27 @@ definitions: nodeId: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation: + properties: + heardAt: + type: integer + id: + type: integer + observerId: + type: string + observerName: + type: string + packetHash: + type: string + payloadType: + type: integer + payloadTypeName: + type: string + rssi: + type: integer + snr: + type: number + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail: properties: iataCount: @@ -2799,6 +2852,69 @@ paths: summary: List known routes tags: - Routes + /routes/{iata}/{pathKey}/observations: + get: + description: Matches the saved IATA, complete path bytes and hash width. Excludes + TRACE and unclassified reports; other widths and search-result subsegments + are not included. Byte matches do not prove hop identities or delivery. The + route and its historical counter can outlive raw evidence. Cursors retain + full timestamp precision and pin route/window scope. + parameters: + - description: Three-character IATA code + in: path + name: iata + required: true + type: string + - description: Stable 32-hex pathKey from a known-route response + in: path + name: pathKey + required: true + type: string + - description: Server-anchored duration (default 24h, max 720h); exclusive with + since/until/pageCursor + in: query + name: range + type: string + - description: Window start epoch ms; provide with until (default last 24h) + in: query + name: since + type: integer + - description: Exclusive window end epoch ms; maximum span 30d, no future end + in: query + name: until + type: integer + - description: Opaque precise cursor from nextPageCursor; window is pinned + in: query + name: pageCursor + type: string + - description: Default 50; positive, capped at 200 + in: query + maximum: 200 + minimum: 1 + name: limit + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Get retained reports matching a full saved route prefix sequence + tags: + - Routes /routes/cross: get: parameters: diff --git a/internal/api/handlers/route_evidence.go b/internal/api/handlers/route_evidence.go new file mode 100644 index 0000000..6c156fe --- /dev/null +++ b/internal/api/handlers/route_evidence.go @@ -0,0 +1,118 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" +) + +func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.RouteEvidenceQuery, error) { + q := api.RouteEvidenceQuery{} + values := r.URL.Query() + for _, name := range []string{"pageCursor", "since", "until", "range", "limit"} { + if len(values[name]) > 1 { + return q, api.ErrRouteEvidenceInput + } + } + if values.Has("cursor") { + return q, api.ErrRouteEvidenceInput + } + limit, err := parseLimit(r, 50) + if err != nil { + return q, err + } + q.Limit = limit + if values.Has("pageCursor") { + c, err := api.ParseRouteEvidenceCursor(values.Get("pageCursor")) + if err != nil || c.IATA != iata || c.PathKey != key { + return q, api.ErrRouteEvidenceInput + } + q.Cursor = c + q.Since = c.Since + q.Until = c.Until + } else { + q.Until = now.UTC().Truncate(time.Millisecond) + q.Since = q.Until.Add(-24 * time.Hour) + } + if values.Has("range") { + if values.Has("pageCursor") || values.Has("since") || values.Has("until") { + return q, api.ErrRouteEvidenceInput + } + duration, err := time.ParseDuration(values.Get("range")) + if err != nil || duration < time.Millisecond || duration > api.MaxRouteEvidenceWindow || duration%time.Millisecond != 0 { + return q, api.ErrRouteEvidenceInput + } + q.Since = q.Until.Add(-duration) + } + if values.Has("since") || values.Has("until") { + if !values.Has("since") || !values.Has("until") { + return q, api.ErrRouteEvidenceInput + } + since, e1 := strconv.ParseInt(values.Get("since"), 10, 64) + until, e2 := strconv.ParseInt(values.Get("until"), 10, 64) + if e1 != nil || e2 != nil || since < 0 || until < 0 || since > 253402300799999 || until > 253402300799999 { + return q, api.ErrRouteEvidenceInput + } + q.Since = time.UnixMilli(since).UTC() + q.Until = time.UnixMilli(until).UTC() + if q.Cursor != nil && (!q.Since.Equal(q.Cursor.Since) || !q.Until.Equal(q.Cursor.Until)) { + return q, api.ErrRouteEvidenceInput + } + } + if !api.ValidRouteEvidenceWindow(q.Since, q.Until) || q.Until.After(now) { + return q, api.ErrRouteEvidenceInput + } + return q, nil +} + +// getRouteEvidence godoc +// +// @Summary Get retained reports matching a full saved route prefix sequence +// @Description Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope. +// @Tags Routes +// @Produce json +// @Param iata path string true "Three-character IATA code" +// @Param pathKey path string true "Stable 32-hex pathKey from a known-route response" +// @Param range query string false "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor" +// @Param since query int false "Window start epoch ms; provide with until (default last 24h)" +// @Param until query int false "Exclusive window end epoch ms; maximum span 30d, no future end" +// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; window is pinned" +// @Param limit query int false "Default 50; positive, capped at 200" minimum(1) maximum(200) +// @Success 200 {object} api.RouteEvidence +// @Failure 400 {object} handlers.APIError +// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Router /routes/{iata}/{pathKey}/observations [get] +func getRouteEvidence(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + iata, key := strings.ToUpper(chi.URLParam(r, "iata")), strings.ToLower(chi.URLParam(r, "pathKey")) + if !api.ValidRouteEvidenceKey(iata, key) { + respondError(w, 400, "invalid IATA or route pathKey") + return + } + query, err := routeEvidenceQuery(r, iata, key, time.Now()) + if err != nil { + respondError(w, 400, "invalid route evidence window, limit or page cursor") + return + } + result, err := reader.GetRouteEvidence(r.Context(), iata, key, query) + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && result == nil: + respondError(w, 404, "saved route not found") + case errors.Is(err, api.ErrRouteEvidenceInput): + respondError(w, 400, "invalid route evidence request") + case err != nil: + respondError(w, 500, "internal server error") + default: + respond(w, 200, result) + } + } +} diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go new file mode 100644 index 0000000..382ea0f --- /dev/null +++ b/internal/api/handlers/route_evidence_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "context" + "errors" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +const evidenceKey = "f097439148601d9f3291c474f82fa64c" + +func TestRouteEvidenceQuery(t *testing.T) { + now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC) + q, err := routeEvidenceQuery(httptest.NewRequest("GET", "/", nil), "YOW", evidenceKey, now) + if err != nil || q.Limit != 50 || q.Until != now || q.Until.Sub(q.Since) != 24*time.Hour { + t.Fatalf("defaults: %+v %v", q, err) + } + c := api.RouteEvidenceCursor{IATA: "YOW", PathKey: evidenceKey, Since: q.Since, Until: q.Until, HeardAt: q.Since.Add(time.Hour + 123*time.Microsecond), ID: 3} + next, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String())+"&limit=999", nil), "YOW", evidenceKey, now.Add(time.Hour)) + if err != nil || next.Limit != 200 || !next.Until.Equal(now) || !next.Cursor.HeardAt.Equal(c.HeardAt) { + t.Fatalf("pinned cursor: %+v %v", next, err) + } + for _, query := range []string{"?limit=0", "?limit=-1", "?limit=a", "?limit=1&limit=2", "?since=0", "?until=10", "?since=2&until=1", "?since=0&until=2678400000", "?since=0&until=99999999999999999", "?pageCursor=bad", "?pageCursor=", "?cursor=123", "?pageCursor=" + url.QueryEscape(c.String()) + "&since=0&until=10"} { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String()), nil), "YVR", evidenceKey, now); err == nil { + t.Fatal("cursor crossed IATA") + } +} + +func TestRouteEvidenceHandler(t *testing.T) { + for _, tc := range []struct { + path string + err error + status int + }{ + {"/yow/" + evidenceKey + "/observations", nil, 200}, + {"/YOW/bad/observations", nil, 400}, + {"/LONG/" + evidenceKey + "/observations", nil, 400}, + {"/YOW/" + evidenceKey + "/observations", pgx.ErrNoRows, 404}, + {"/YOW/" + evidenceKey + "/observations", errors.New("database unavailable"), 500}, + } { + called := false + router := RoutesRouter(stubReader{getRouteEvidence: func(_ context.Context, iata, key string, q api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + called = true + if iata != "YOW" || key != evidenceKey || q.Limit != 50 { + t.Fatal("wrong reader scope") + } + return &api.RouteEvidence{Page: api.Page[api.RouteObservation]{Items: []api.RouteObservation{}}}, tc.err + }}) + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", tc.path, nil)) + if w.Code != tc.status { + t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String()) + } + if tc.status == 400 && called { + t.Fatal("invalid request reached database") + } + } +} diff --git a/internal/api/handlers/routes.go b/internal/api/handlers/routes.go index a5bdd76..77f1117 100644 --- a/internal/api/handlers/routes.go +++ b/internal/api/handlers/routes.go @@ -22,6 +22,7 @@ func RoutesRouter(reader api.Reader) http.Handler { r.Get("/", listKnownRoutes(reader)) r.Get("/cross", searchCrossIATARoutes(reader)) r.Get("/search", searchKnownRoutes(reader)) + r.Get("/{iata}/{pathKey}/observations", getRouteEvidence(reader)) return r } diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 004d94c..f9f885a 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -16,6 +16,7 @@ import ( // Unset fields return zero values. Use it for both validation tests // (leave all fields nil) and happy path tests (set only what you need). type stubReader struct { + getRouteEvidence func(context.Context, string, string, api.RouteEvidenceQuery) (*api.RouteEvidence, error) getObserverComparison func(context.Context, uuid.UUID, uuid.UUID, time.Time, time.Time, []string) (*api.ObserverComparison, error) listIATAs func(ctx context.Context) ([]api.IATA, error) getIATA func(ctx context.Context, iata string) (*api.IATA, error) @@ -409,3 +410,10 @@ func (s stubReader) GetNodesByIDs(ctx context.Context, ids []uuid.UUID) (map[uui } return nil, nil } + +func (s stubReader) GetRouteEvidence(ctx context.Context, iata, key string, q api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + if s.getRouteEvidence != nil { + return s.getRouteEvidence(ctx, iata, key, q) + } + return nil, nil +} diff --git a/internal/api/reader.go b/internal/api/reader.go index c9a424d..d46ec2d 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -23,6 +23,8 @@ type Page[T any] struct { } type Reader interface { + // GetRouteEvidence returns the full saved route and exact-prefix retained ordinary reports. + GetRouteEvidence(ctx context.Context, iata, pathKey string, query RouteEvidenceQuery) (*RouteEvidence, error) // ListIATAs returns all known IATA codes with display name and coordinates. // IATAs are auto-created on first packet arrival from that location. ListIATAs(ctx context.Context) ([]IATA, error) diff --git a/internal/api/route_evidence.go b/internal/api/route_evidence.go new file mode 100644 index 0000000..cadc056 --- /dev/null +++ b/internal/api/route_evidence.go @@ -0,0 +1,110 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import ( + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +const MaxRouteEvidenceWindow = 30 * 24 * time.Hour + +var ErrRouteEvidenceInput = errors.New("invalid route evidence request") + +// RouteObservation is a retained report reference, not a packet body or proof of hop identity. +type RouteObservation struct { + ID int64 `json:"id"` + PacketHash string `json:"packetHash"` + ObserverID uuid.UUID `json:"observerId"` + ObserverName *string `json:"observerName,omitempty"` + HeardAt int64 `json:"heardAt"` + PayloadType int16 `json:"payloadType"` + PayloadTypeName string `json:"payloadTypeName"` + RSSI *int16 `json:"rssi,omitempty"` + SNR *float32 `json:"snr,omitempty"` +} + +// RouteEvidence uses nextPageCursor rather than the lossy legacy numeric cursor. +type RouteEvidence struct { + Page[RouteObservation] + Route KnownRoute `json:"route"` + WindowStart int64 `json:"windowStart"` + WindowEnd int64 `json:"windowEnd"` + GeneratedAt int64 `json:"generatedAt"` + MatchType string `json:"matchType"` + MatchAvailable bool `json:"matchAvailable"` + HashSize int16 `json:"hashSize"` + PathBytes string `json:"pathBytes"` + NextPageCursor *string `json:"nextPageCursor,omitempty"` +} + +type RouteEvidenceQuery struct { + Since time.Time + Until time.Time + Cursor *RouteEvidenceCursor + Limit int32 +} + +// RouteEvidenceCursor binds the precise (heard_at DESC,id DESC) boundary to its route/window. +type RouteEvidenceCursor struct { + IATA string + PathKey string + Since time.Time + Until time.Time + HeardAt time.Time + ID int64 +} + +func ValidRouteEvidenceKey(iata, key string) bool { + if len(iata) != 3 || len(key) != 32 { + return false + } + for _, c := range iata { + if !(c >= 'A' && c <= 'Z') && !(c >= '0' && c <= '9') { + return false + } + } + _, err := hex.DecodeString(key) + return err == nil && key == strings.ToLower(key) +} + +func ValidRouteEvidenceWindow(since, until time.Time) bool { + return !since.Before(time.Unix(0, 0)) && until.After(since) && until.Sub(since) <= MaxRouteEvidenceWindow +} + +func (c RouteEvidenceCursor) String() string { + return fmt.Sprintf("v1:%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) +} + +func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { + if len(raw) > 200 { + return nil, ErrRouteEvidenceInput + } + p := strings.Split(raw, ":") + if len(p) != 7 || p[0] != "v1" || !ValidRouteEvidenceKey(p[1], p[2]) { + return nil, ErrRouteEvidenceInput + } + values := [4]int64{} + for i, s := range p[3:] { + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || n < 0 || strconv.FormatInt(n, 10) != s { + return nil, ErrRouteEvidenceInput + } + values[i] = n + } + // Bound before converting; this also keeps millisecond/microsecond comparisons safe. + if values[0] > 253402300799999 || values[1] > 253402300799999 || values[2] > 253402300799999999 || values[3] <= 0 { + return nil, ErrRouteEvidenceInput + } + c := &RouteEvidenceCursor{IATA: p[1], PathKey: p[2], Since: time.UnixMilli(values[0]).UTC(), Until: time.UnixMilli(values[1]).UTC(), HeardAt: time.UnixMicro(values[2]).UTC(), ID: values[3]} + if !ValidRouteEvidenceWindow(c.Since, c.Until) || c.HeardAt.Before(c.Since) || !c.HeardAt.Before(c.Until) { + return nil, ErrRouteEvidenceInput + } + return c, nil +} diff --git a/internal/api/route_evidence_test.go b/internal/api/route_evidence_test.go new file mode 100644 index 0000000..42c9a73 --- /dev/null +++ b/internal/api/route_evidence_test.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import ( + "strings" + "testing" + "time" +) + +func TestRouteEvidenceCursor(t *testing.T) { + c := RouteEvidenceCursor{IATA: "YOW", PathKey: "f097439148601d9f3291c474f82fa64c", Since: time.UnixMilli(1000).UTC(), Until: time.UnixMilli(3000).UTC(), HeardAt: time.UnixMicro(2000123).UTC(), ID: 9223372036854775806} + got, err := ParseRouteEvidenceCursor(c.String()) + if err != nil || *got != c { + t.Fatalf("cursor round-trip: %+v %v", got, err) + } + if got.HeardAt.Nanosecond() != 123000 { + t.Fatal("lost microsecond precision") + } + for _, bad := range []string{"", strings.Repeat("x", 300), "v2:YOW:x:1:2:3:4", strings.Replace(c.String(), "YOW", "TOOLONG", 1), strings.Replace(c.String(), c.PathKey, "bad", 1), strings.Replace(c.String(), "2000123", "4000000", 1), strings.TrimSuffix(c.String(), "9223372036854775806") + "0"} { + if _, err := ParseRouteEvidenceCursor(bad); err == nil { + t.Errorf("accepted %q", bad) + } + } +} diff --git a/internal/api/routes.go b/internal/api/routes.go index 3bc7ece..3f1bee1 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -16,6 +16,7 @@ type RouteHop struct { // have been confirmed as high confidence. type KnownRoute struct { ID int64 `json:"id"` + PathKey string `json:"pathKey,omitempty"` // stable identity within this IATA; use for route evidence links IATA string `json:"iata"` HopCount int32 `json:"hopCount"` Hops []RouteHop `json:"hops"` diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 21d6f3d..58467ce 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -366,7 +366,7 @@ func TestCachedReader_IATASortingForStableKey(t *testing.T) { return &api.StatsOverview{TotalPackets: 42}, nil }) - // call CachedReader with reversed order — should hit same key + // call CachedReader with reversed order — should hit same key _ = cr result, err := getOrSet(context.Background(), c, "beacon:stats:overview:YVR,YYJ", time.Minute, func() (*api.StatsOverview, error) { calls++ @@ -418,3 +418,7 @@ func TestCachedReader_InvalidateObserver(t *testing.T) { t.Error("expected observer scopes key to be deleted") } } + +func (s *stubReader) GetRouteEvidence(_ context.Context, _, _ string, _ api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + return nil, nil +} diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 2d31ba4..13f71fa 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -464,6 +464,11 @@ func (cr *CachedReader) SearchKnownRoutes(ctx context.Context, iata, fromHash, t return cr.inner.SearchKnownRoutes(ctx, iata, fromHash, toHash) } +// Precise cursor/window combinations are intentionally passed through, like route lists. +func (cr *CachedReader) GetRouteEvidence(ctx context.Context, iata, key string, query api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + return cr.inner.GetRouteEvidence(ctx, iata, key, query) +} + // SearchCrossIATARoutes implements [api.Reader]. func (cr *CachedReader) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, toHash, toIATA string) ([]api.CrossIATARoute, error) { return cr.inner.SearchCrossIATARoutes(ctx, fromHash, fromIATA, toHash, toIATA) From 0fc1398eecaa24f1dbdd0fae28bc259f15a439f2 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 14:56:08 -0400 Subject: [PATCH 6/9] test(routes): cover server-anchored evidence windows --- internal/api/handlers/route_evidence_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go index 382ea0f..8340d85 100644 --- a/internal/api/handlers/route_evidence_test.go +++ b/internal/api/handlers/route_evidence_test.go @@ -22,6 +22,10 @@ func TestRouteEvidenceQuery(t *testing.T) { if err != nil || q.Limit != 50 || q.Until != now || q.Until.Sub(q.Since) != 24*time.Hour { t.Fatalf("defaults: %+v %v", q, err) } + week, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?range=168h", nil), "YOW", evidenceKey, now) + if err != nil || !week.Until.Equal(now) || week.Until.Sub(week.Since) != 7*24*time.Hour { + t.Fatalf("server anchored range: %+v %v", week, err) + } c := api.RouteEvidenceCursor{IATA: "YOW", PathKey: evidenceKey, Since: q.Since, Until: q.Until, HeardAt: q.Since.Add(time.Hour + 123*time.Microsecond), ID: 3} next, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String())+"&limit=999", nil), "YOW", evidenceKey, now.Add(time.Hour)) if err != nil || next.Limit != 200 || !next.Until.Equal(now) || !next.Cursor.HeardAt.Equal(c.HeardAt) { @@ -35,6 +39,11 @@ func TestRouteEvidenceQuery(t *testing.T) { if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String()), nil), "YVR", evidenceKey, now); err == nil { t.Fatal("cursor crossed IATA") } + for _, query := range []string{"range=0h", "range=-1h", "range=721h", "range=1ns", "range=7d", "range=24h&range=168h", "range=24h&since=0&until=10", "range=24h&pageCursor=" + url.QueryEscape(c.String())} { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } } func TestRouteEvidenceHandler(t *testing.T) { From 8f646d91ce1b404670a9bcdec04ca5b8102fd599 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 17:59:04 -0400 Subject: [PATCH 7/9] feat(scopes): import cached regional MeshMapper catalogues --- .github/workflows/ci.yml | 2 +- cmd/beacon/main.go | 17 ++ config.yaml.example | 15 ++ db/meshmapper.go | 49 ++++ db/meshmapper_integration_test.go | 139 +++++++++++ db/migrations/042_meshmapper_scopes.sql | 17 ++ db/queries/meshmapper.sql | 27 +++ db/queries/queries.sql | 5 +- db/sqlc/meshmapper.sql.go | 94 ++++++++ db/sqlc/mock/querier.go | 29 +++ db/sqlc/models.go | 12 + db/sqlc/querier.go | 7 + db/sqlc/queries.sql.go | 5 +- docs/meshmapper-scopes.md | 26 +++ internal/cache/reader.go | 5 + internal/config/config.go | 4 + internal/config/meshmapper.go | 64 ++++++ internal/config/meshmapper_test.go | 49 ++++ internal/config/seed.go | 27 +-- internal/config/seed_test.go | 21 +- internal/ingest/packet.go | 31 ++- internal/ingest/scope_import_test.go | 95 ++++++++ internal/meshmapper/scopes.go | 294 ++++++++++++++++++++++++ internal/meshmapper/scopes_test.go | 253 ++++++++++++++++++++ internal/scopestore/scopestore.go | 22 +- 25 files changed, 1260 insertions(+), 49 deletions(-) create mode 100644 db/meshmapper.go create mode 100644 db/meshmapper_integration_test.go create mode 100644 db/migrations/042_meshmapper_scopes.sql create mode 100644 db/queries/meshmapper.sql create mode 100644 db/sqlc/meshmapper.sql.go create mode 100644 docs/meshmapper-scopes.md create mode 100644 internal/config/meshmapper.go create mode 100644 internal/config/meshmapper_test.go create mode 100644 internal/ingest/scope_import_test.go create mode 100644 internal/meshmapper/scopes.go create mode 100644 internal/meshmapper/scopes_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da907fe..6be1def 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 8e322e6..1fa2ea3 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -29,6 +29,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/MeshCore-Beacon/beacon-server/internal/keystore" "github.com/MeshCore-Beacon/beacon-server/internal/logging" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" "github.com/MeshCore-Beacon/beacon-server/internal/presence" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" @@ -189,6 +190,16 @@ func main() { } scopes.Load(scopeEntries) slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") + var scopeImporter *meshmapper.Importer + if cfg.MeshMapper.Scopes.Enabled { + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, scopes, scopeEntries) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper scope catalogues", "component", "startup", "error", err) + os.Exit(1) + } + } // ── Build channel keystore ────────────────────────────────────────────── entries := make(map[string][]keystore.Entry) @@ -281,6 +292,9 @@ func main() { ) if cr, ok := reader.(*cache.CachedReader); ok { + if scopeImporter != nil { + scopeImporter.SetCacheInvalidator(cr.InvalidateScopeNames) + } broker1.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver) broker2.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver) } @@ -301,6 +315,9 @@ func main() { } tasks = append(tasks, background.ObserverCleanupTask(coalescer, resolved.ObserverDeleteAfter, resolved.CleanupInterval, onDelete)) } + if scopeImporter != nil { + tasks = append(tasks, background.Task{Name: "meshmapper.scopes", Interval: meshmapper.PollInterval, Run: scopeImporter.Refresh}) + } scheduler := background.New(tasks) go scheduler.Start(ctx) diff --git a/config.yaml.example b/config.yaml.example index a0a86cc..3e2a66f 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -214,6 +214,21 @@ cache: # allow_countries: [CA, US] # allow_continents: [NA] +# Optional public MeshMapper scope-name discovery. Manual scopes remain authoritative. +# No API key is needed. Sources must belong to a configured region's IATAs. +# Group endpoints are not supported: their names cannot be attributed to each IATA. +# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping. +# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. +# Refresh success/failure/freshness is logged under component=meshmapper.scopes. +# Removing/turning off a source deactivates its imported matching keys on restart; +# historical scope identities and recorded evidence are retained. +#meshmapper: +# scopes: +# enabled: false +# refresh_interval: 1h # 5m-24h; one source checked per 15s tick +# sources: +# YOW: https://yow.meshmapper.net/get_scopes.php + # Background task intervals. # Shorter intervals are useful during initial deployment to confirm data is # flowing. Back off to 1h or more once stable. diff --git a/db/meshmapper.go b/db/meshmapper.go new file mode 100644 index 0000000..3ddbb28 --- /dev/null +++ b/db/meshmapper.go @@ -0,0 +1,49 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetScopeCatalogue(ctx context.Context, iata, url string) (*meshmapper.Cache, error) { + row, err := s.q.GetScopeCatalogue(ctx, sqlc.GetScopeCatalogueParams{Iata: iata, Url: url}) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + cache := &meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError} + if row.Etag != nil { + cache.ETag = *row.Etag + } + return cache, nil +} + +func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, entries []scopestore.Entry) error { + params := sqlc.SaveScopeCatalogueParams{ + Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError, + CheckedAt: pgtype.Timestamptz{Time: cache.CheckedAt, Valid: !cache.CheckedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: cache.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: cache.NextAttempt, Valid: true}, + Names: []string{}, Keys: [][]byte{}, Fingerprints: [][]byte{}, + } + if !cache.CheckedAt.IsZero() { + params.Etag = &cache.ETag + } + for _, entry := range entries { + params.Names = append(params.Names, entry.Name) + params.Keys = append(params.Keys, entry.TransportKey) + params.Fingerprints = append(params.Fingerprints, entry.KeyFingerprint) + } + return s.q.SaveScopeCatalogue(ctx, params) +} diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go new file mode 100644 index 0000000..06d7109 --- /dev/null +++ b/db/meshmapper_integration_test.go @@ -0,0 +1,139 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestMeshMapperCataloguePostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"transport_scopes", "meshmapper_scope_catalogues"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + manual := scopestore.FromName("manual") + manual.TransportKey[0] ^= 1 + if err := store.UpsertTransportScope(ctx, manual.Name, "Operator label", manual.TransportKey, manual.KeyFingerprint); err != nil { + t.Fatal(err) + } + url := "https://yow.meshmapper.net/get_scopes.php" + if row, err := store.GetScopeCatalogue(ctx, "YOW", url); err != nil || row != nil { + t.Fatal(row, err) + } + now := time.Now().UTC().Truncate(time.Second) + payload := []byte(`{"generated_at":"2026-09-27T16:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":3,"scopes":[{"name":"yow","repeaters":0,"default":0,"monitored":true,"wardriving":false}]}`) + update := meshmapper.Cache{Payload: payload, ETag: `"v1"`, CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(time.Hour)} + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("manual"), scopestore.FromName("yow")}); err != nil { + t.Fatal(err) + } + manualRows, err := store.GetTransportScopes(ctx) + if err != nil || len(manualRows) != 1 || manualRows[0].TransportKey[0] != manual.TransportKey[0] { + t.Fatal("manual key changed", manualRows, err) + } + var label string + var imported bool + if err := tx.QueryRow(ctx, "SELECT display_name,imported_only FROM transport_scopes WHERE name='#manual'").Scan(&label, &imported); err != nil || label != "Operator label" || imported { + t.Fatal(label, imported, err) + } + row, err := store.GetScopeCatalogue(ctx, "YOW", url) + if err != nil || row.ETag != `"v1"` || !row.CheckedAt.Equal(now) { + t.Fatal(row, err) + } + scopes := scopestore.New() + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + if err != nil || len(scopes.Entries()) != 2 { + t.Fatal("restart did not restore imported membership", err) + } + // A conditional success preserves the document, advances checked time and clears a failure. + checked := now.Add(time.Hour) + if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{ETag: `"v1"`, CheckedAt: checked, AttemptedAt: checked, NextAttempt: checked.Add(time.Hour)}, nil); err != nil { + t.Fatal(err) + } + if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{AttemptedAt: checked.Add(time.Minute), NextAttempt: checked.Add(2 * time.Hour), LastError: "HTTP 429"}, nil); err != nil { + t.Fatal(err) + } + row, err = store.GetScopeCatalogue(ctx, "YOW", url) + if err != nil || !row.CheckedAt.Equal(checked) || row.ETag != `"v1"` || row.LastError != "HTTP 429" || len(row.Payload) == 0 { + t.Fatal("failed refresh lost durable snapshot", row, err) + } + // Empty membership deactivates matching but retains the old scope identity for historical rows. + empty := []byte(`{"generated_at":"2026-09-27T17:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":0,"scopes":[]}`) + update.Payload = empty + update.ETag = "" + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, nil); err != nil { + t.Fatal(err) + } + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + if err != nil || len(scopes.Entries()) != 1 { + t.Fatal("removed imported membership remained active", err) + } + if _, err := store.GetTransportScopeByName(ctx, "#yow"); err != nil { + t.Fatal("deleted historical identity", err) + } + row, _ = store.GetScopeCatalogue(ctx, "YOW", url) + if row.ETag != "" { + t.Fatal("200 without ETag retained obsolete validator") + } + // Reconfiguring a known imported name as manual promotes it without losing its identity. + yow := scopestore.FromName("yow") + if err := store.UpsertTransportScope(ctx, yow.Name, "Local", yow.TransportKey, yow.KeyFingerprint); err != nil { + t.Fatal(err) + } + manualRows, err = store.GetTransportScopes(ctx) + if err != nil || len(manualRows) != 2 { + t.Fatal("manual promotion failed", err) + } + // A failed cache write also rolls back the identity insertion in the same statement. + if _, err := tx.Exec(ctx, "ALTER TABLE meshmapper_scope_catalogues ADD CHECK(last_error <> 'force-failure'); SAVEPOINT rejected"); err != nil { + t.Fatal(err) + } + update.LastError = "force-failure" + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("uncommitted")}); err == nil { + t.Fatal("expected constraint failure") + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT rejected"); err != nil { + t.Fatal(err) + } + var count int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM transport_scopes WHERE name='#uncommitted'").Scan(&count); err != nil || count != 0 { + t.Fatal("partially committed catalogue", count, err) + } +} diff --git a/db/migrations/042_meshmapper_scopes.sql b/db/migrations/042_meshmapper_scopes.sql new file mode 100644 index 0000000..43b48b6 --- /dev/null +++ b/db/migrations/042_meshmapper_scopes.sql @@ -0,0 +1,17 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Preserve historical identities while distinguishing import-only matching keys. +ALTER TABLE transport_scopes ADD COLUMN IF NOT EXISTS imported_only BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE TABLE IF NOT EXISTS meshmapper_scope_catalogues ( + iata TEXT NOT NULL, + url TEXT NOT NULL, + payload JSONB, + etag TEXT, + checked_at TIMESTAMPTZ, + attempted_at TIMESTAMPTZ NOT NULL, + next_attempt TIMESTAMPTZ NOT NULL, + last_error TEXT NOT NULL DEFAULT '', + PRIMARY KEY (iata, url) +); diff --git a/db/queries/meshmapper.sql b/db/queries/meshmapper.sql new file mode 100644 index 0000000..16e8243 --- /dev/null +++ b/db/queries/meshmapper.sql @@ -0,0 +1,27 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- name: GetScopeCatalogue :one +SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; + +-- name: SaveScopeCatalogue :exec +-- One statement commits the validated snapshot and its lookup identities together. +-- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +-- after an error or 304. Imported names never replace existing manual metadata. +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest(@names::text[]) AS name, unnest(@keys::bytea[]) AS key, + unnest(@fingerprints::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index aaf0be8..7863de0 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -49,10 +49,11 @@ VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO UPDATE SET display_name = EXCLUDED.display_name, transport_key = EXCLUDED.transport_key, - key_fingerprint = EXCLUDED.key_fingerprint; + key_fingerprint = EXCLUDED.key_fingerprint, + imported_only = FALSE; -- name: GetTransportScopes :many -SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name; +SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name; -- name: GetTransportScopeByName :one SELECT id FROM transport_scopes WHERE name = $1; diff --git a/db/sqlc/meshmapper.sql.go b/db/sqlc/meshmapper.sql.go new file mode 100644 index 0000000..b8ba12d --- /dev/null +++ b/db/sqlc/meshmapper.sql.go @@ -0,0 +1,94 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: meshmapper.sql + +package db + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getScopeCatalogue = `-- name: GetScopeCatalogue :one + +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2 +` + +type GetScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` +} + +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +func (q *Queries) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) { + row := q.db.QueryRow(ctx, getScopeCatalogue, arg.Iata, arg.Url) + var i MeshmapperScopeCatalogue + err := row.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ) + return i, err +} + +const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest($9::text[]) AS name, unnest($10::bytea[]) AS key, + unnest($11::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` + Names []string `json:"names"` + Keys [][]byte `json:"keys"` + Fingerprints [][]byte `json:"fingerprints"` +} + +// One statement commits the validated snapshot and its lookup identities together. +// Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +// after an error or 304. Imported names never replace existing manual metadata. +func (q *Queries) SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error { + _, err := q.db.Exec(ctx, saveScopeCatalogue, + arg.Iata, + arg.Url, + arg.Payload, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + arg.Names, + arg.Keys, + arg.Fingerprints, + ) + return err +} diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index 9d3fb18..16ea073 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -714,6 +714,21 @@ func (mr *MockQuerierMockRecorder) GetScopeByName(ctx, name any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeByName", reflect.TypeOf((*MockQuerier)(nil).GetScopeByName), ctx, name) } +// GetScopeCatalogue mocks base method. +func (m *MockQuerier) GetScopeCatalogue(ctx context.Context, arg db.GetScopeCatalogueParams) (db.MeshmapperScopeCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetScopeCatalogue", ctx, arg) + ret0, _ := ret[0].(db.MeshmapperScopeCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetScopeCatalogue indicates an expected call of GetScopeCatalogue. +func (mr *MockQuerierMockRecorder) GetScopeCatalogue(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).GetScopeCatalogue), ctx, arg) +} + // GetScopeNames mocks base method. func (m *MockQuerier) GetScopeNames(ctx context.Context) ([]string, error) { m.ctrl.T.Helper() @@ -1542,6 +1557,20 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } +// SaveScopeCatalogue mocks base method. +func (m *MockQuerier) SaveScopeCatalogue(ctx context.Context, arg db.SaveScopeCatalogueParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveScopeCatalogue", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveScopeCatalogue indicates an expected call of SaveScopeCatalogue. +func (mr *MockQuerierMockRecorder) SaveScopeCatalogue(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveScopeCatalogue), ctx, arg) +} + // SearchKnownRoutes mocks base method. func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index ebf9293..87d9978 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -234,6 +234,17 @@ type KnownRoute struct { LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"` } +type MeshmapperScopeCatalogue struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + type MvHourlyIataStat struct { Iata string `json:"iata"` Hour pgtype.Timestamptz `json:"hour"` @@ -528,4 +539,5 @@ type TransportScope struct { TransportKey []byte `json:"transport_key"` KeyFingerprint []byte `json:"key_fingerprint"` CreatedAt pgtype.Timestamptz `json:"created_at"` + ImportedOnly bool `json:"imported_only"` } diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index a577064..f96b9a3 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -89,6 +89,9 @@ type Querier interface { GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) + // Copyright 2026 Beacon Contributors + // SPDX-License-Identifier: AGPL-3.0-or-later + GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) GetScopeNames(ctx context.Context) ([]string, error) // Aggregate matching observations once, separately from node memberships to avoid // a cross-join. Empty IATAs keep the original global counts, including associations @@ -250,6 +253,10 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + // One statement commits the validated snapshot and its lookup identities together. + // Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data + // after an error or 304. Imported names never replace existing manual metadata. + SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 3dd6de8..12f6934 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -2012,7 +2012,7 @@ func (q *Queries) GetTransportScopeByName(ctx context.Context, name string) (int } const getTransportScopes = `-- name: GetTransportScopes :many -SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name +SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name ` type GetTransportScopesRow struct { @@ -5189,7 +5189,8 @@ VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO UPDATE SET display_name = EXCLUDED.display_name, transport_key = EXCLUDED.transport_key, - key_fingerprint = EXCLUDED.key_fingerprint + key_fingerprint = EXCLUDED.key_fingerprint, + imported_only = FALSE ` type UpsertTransportScopeParams struct { diff --git a/docs/meshmapper-scopes.md b/docs/meshmapper-scopes.md new file mode 100644 index 0000000..7c9a391 --- /dev/null +++ b/docs/meshmapper-scopes.md @@ -0,0 +1,26 @@ +# Optional MeshMapper scope discovery + +Beacon can periodically read the [public Scopes API](https://wiki.meshmapper.net/scopes-api/) to discover additional transport scope names for configured regions. Manual `scopes` remain authoritative. This does not supply channel decryption keys or per-repeater records. + +```yaml +meshmapper: + scopes: + enabled: true + refresh_interval: 1h + sources: + YOW: https://yow.meshmapper.net/get_scopes.php +``` + +The IATA must already occur in `regions[].iatas`. Set the exact published regional HTTPS URL; credentials, query strings, other hosts and redirects are rejected. The response must identify the same region and exactly one zone. Group catalogues are rejected because they cannot assign each scope/count to individual member regions. No API key is required. + +The default is disabled. Refresh defaults to one hour and accepts 5 minutes through 24 hours. One source is considered every 15 seconds, keeping requests below four per minute per process. Multiple deployments share MeshMapper's per-IP allowance. ETags reduce transfer, HTTP 304 advances the last successful check, and 429 pauses all sources until the later of the normal retry time and `Retry-After`. The pause survives restarts. HTTP 503 also honors that source's `Retry-After`. + +Responses are limited to 64 KiB and 64 names per source, with at most 16 sources. Names are case-sensitive and at most 128 bytes before Beacon's usual `#` normalization; existing `#`/`$` prefixes are retained. Zero-count monitored names are kept. Imported matching candidates are limited to their source IATA; manual keys keep their existing global applicability. A short transport code that matches multiple candidate names remains unresolved. + +Migration 042 stores the last validated response, validator and check/retry timestamps in PostgreSQL. Saving a response and its scope identities is atomic. Timeouts, invalid responses and HTTP errors retain the prior catalogue; startup restores it without HTTP. A successful empty response removes that source's active imported membership. Removing or disabling a source takes effect on restart. Scope identities referenced by historical records are retained. Adding an imported name to the manual list promotes it to manual ownership. + +Operator logs under `component=meshmapper.scopes` show the IATA, source, active name count, source generation time, last successful check, next attempt and last error. An empty last error means the latest request succeeded; a zero checked time means no successful response has been stored. A 304 retains the original source generation time because MeshMapper excludes that timestamp from its ETag. Cached names may be old during an outage; use the check/error fields to assess freshness. + +Catalogue counts remain source metadata. They never create Beacon packet counts, observer associations or node defaults. Actual packet-code matching supplies those associations through the existing ingest path. New names affect subsequent packets; no historical scan is started. Channel message tags and import-status UI are separate follow-ups. Source corruption or an unavailable catalogue database at startup is reported as a startup failure; disabling the optional importer restores manual-only behavior without deleting saved data. + +Validate and back up PostgreSQL before applying the migration. Rollback to an older server must restore its matching pre-import database and configuration as well as its binary; an older matcher does not understand imported-only ownership or regional candidate limits. diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 13f71fa..b28e7bc 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -80,6 +80,11 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { } } +// InvalidateScopeNames makes newly committed catalogue names available to filters. +func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { + cr.c.del(ctx, keyScopeNames) +} + // InvalidateNode removes the cached entries for a node by UUID. // Should be called from the ingest path after a node upsert. func (cr *CachedReader) InvalidateNode(ctx context.Context, nodeID uuid.UUID) { diff --git a/internal/config/config.go b/internal/config/config.go index b3c6cd3..b8b8fff 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -33,6 +33,7 @@ type Config struct { Routes RoutesConfig `yaml:"routes"` Ingest IngestFilterConfig `yaml:"ingest"` Scopes []ScopeConfig `yaml:"scopes"` + MeshMapper MeshMapperConfig `yaml:"meshmapper"` Cache CacheConfig `yaml:"cache"` CORS CORSConfig `yaml:"cors"` RateLimit RateLimitConfig `yaml:"ratelimit"` @@ -408,6 +409,9 @@ func Load(path string) (*Config, error) { } } configDir := filepath.Dir(path) + if err := cfg.validateMeshMapper(); err != nil { + return nil, err + } for iata, details := range cfg.IATAs { if details.BorderFile != "" && !filepath.IsAbs(details.BorderFile) { details.BorderFile = filepath.Join(configDir, details.BorderFile) diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go new file mode 100644 index 0000000..5f36774 --- /dev/null +++ b/internal/config/meshmapper.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "fmt" + "net/url" + "regexp" + "time" +) + +// MaxScopeSources bounds both network refresh work and imported matching keys. +const MaxScopeSources = 16 + +type MeshMapperConfig struct { + Scopes MeshMapperScopesConfig `yaml:"scopes"` +} + +// MeshMapperScopesConfig augments, but never replaces, the manual scopes list. +type MeshMapperScopesConfig struct { + Enabled bool `yaml:"enabled"` + Sources map[string]string `yaml:"sources"` // configured IATA -> published regional endpoint + RefreshInterval duration `yaml:"refresh_interval"` +} + +func (c MeshMapperScopesConfig) Interval() time.Duration { + if c.RefreshInterval.Duration == 0 { + return time.Hour + } + return c.RefreshInterval.Duration +} + +var meshMapperHost = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) + +func (c *Config) validateMeshMapper() error { + s := c.MeshMapper.Scopes + if !s.Enabled { + return nil + } + if len(s.Sources) == 0 || len(s.Sources) > MaxScopeSources { + return fmt.Errorf("meshmapper.scopes.sources must contain 1-%d regional endpoints", MaxScopeSources) + } + if s.Interval() < 5*time.Minute || s.Interval() > 24*time.Hour { + return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 5m and 24h") + } + configured := map[string]bool{} + for _, region := range c.Regions { + for _, iata := range region.IATAs { + configured[iata] = true + } + } + for iata, endpoint := range s.Sources { + if !configured[iata] { + return fmt.Errorf("meshmapper.scopes source %q must belong to a configured region", iata) + } + u, err := url.Parse(endpoint) + if err != nil || u.Scheme != "https" || u.User != nil || !meshMapperHost.MatchString(u.Host) || + u.Path != "/get_scopes.php" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return fmt.Errorf("meshmapper.scopes source %q must be a published https://.meshmapper.net/get_scopes.php endpoint without credentials or parameters", iata) + } + } + return nil +} diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go new file mode 100644 index 0000000..56a5634 --- /dev/null +++ b/internal/config/meshmapper_test.go @@ -0,0 +1,49 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestMeshMapperConfig(t *testing.T) { + valid := "regions:\n - slug: ottawa\n iatas: [YOW]\nmeshmapper:\n scopes:\n enabled: true\n sources:\n YOW: https://yow.meshmapper.net/get_scopes.php\n" + for _, tc := range []struct { + name, text string + bad bool + }{ + {"valid", valid, false}, + {"unknown region", strings.Replace(valid, "iatas: [YOW]", "iatas: [YVR]", 1), true}, + {"http", strings.Replace(valid, "https:", "http:", 1), true}, + {"private host", strings.Replace(valid, "yow.meshmapper.net", "127.0.0.1", 1), true}, + {"credentials", strings.Replace(valid, "https://", "https://secret@", 1), true}, + {"query", strings.Replace(valid, "get_scopes.php", "get_scopes.php?key=secret", 1), true}, + {"non API", strings.Replace(valid, "get_scopes.php", "index.php", 1), true}, + {"fast", valid + " refresh_interval: 1s\n", true}, + {"negative", valid + " refresh_interval: -1h\n", true}, + {"disabled", strings.Replace(valid, "enabled: true", "enabled: false", 1), false}, + } { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(p) + if (err != nil) != tc.bad { + t.Fatalf("error=%v", err) + } + if err == nil && cfg.MeshMapper.Scopes.Interval() != time.Hour { + t.Fatal("wrong default") + } + }) + } + cfg := Config{MeshMapper: MeshMapperConfig{Scopes: MeshMapperScopesConfig{Enabled: true}}} + if err := cfg.validateMeshMapper(); err == nil { + t.Fatal("empty sources accepted") + } +} diff --git a/internal/config/seed.go b/internal/config/seed.go index 53ea371..a419573 100644 --- a/internal/config/seed.go +++ b/internal/config/seed.go @@ -5,12 +5,12 @@ package config import ( "context" - "crypto/sha256" "encoding/json" "fmt" "log/slog" "os" - "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) // Seeder is the database interface required to seed config data on startup. @@ -63,29 +63,10 @@ func Seed(ctx context.Context, cfg *Config, db Seeder) error { } // Transport Codes for _, s := range cfg.Scopes { - name := normalizeScopeName(s.Name) - key := deriveScopeKey(name) - h := sha256.Sum256(key) - fingerprint := h[:8] - if err := db.UpsertTransportScope(ctx, name, "", key, fingerprint); err != nil { + entry := scopestore.FromName(s.Name) + if err := db.UpsertTransportScope(ctx, entry.Name, "", entry.TransportKey, entry.KeyFingerprint); err != nil { return err } } return nil } - -// normalizeScopeName ensures the scope name has a # or $ prefix. -// Plain names get # prepended: "bc" → "#bc". -func normalizeScopeName(name string) string { - if strings.HasPrefix(name, "#") || strings.HasPrefix(name, "$") { - return name - } - return "#" + name -} - -// deriveScopeKey derives the 16-byte transport key from a normalized scope name. -// key = SHA256(name)[:16] -func deriveScopeKey(name string) []byte { - h := sha256.Sum256([]byte(name)) - return h[:16] -} diff --git a/internal/config/seed_test.go b/internal/config/seed_test.go index 4f2fa0c..069bbe3 100644 --- a/internal/config/seed_test.go +++ b/internal/config/seed_test.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" "os" "testing" ) @@ -198,39 +199,39 @@ func TestSeed_DBError(t *testing.T) { } func TestNormalizeScopeName_WithHash(t *testing.T) { - if normalizeScopeName("#bc") != "#bc" { + if scopestore.FromName("#bc").Name != "#bc" { t.Error("expected #bc unchanged") } } func TestNormalizeScopeName_WithDollar(t *testing.T) { - if normalizeScopeName("$bc") != "$bc" { + if scopestore.FromName("$bc").Name != "$bc" { t.Error("expected $bc unchanged") } } func TestNormalizeScopeName_WithoutPrefix(t *testing.T) { - if normalizeScopeName("bc") != "#bc" { + if scopestore.FromName("bc").Name != "#bc" { t.Error("expected bc to become #bc") } } func TestNormalizeScopeName_Empty(t *testing.T) { - if normalizeScopeName("") != "#" { + if scopestore.FromName("").Name != "#" { t.Error("expected empty string to become #") } } func TestDeriveScopeKey_Length(t *testing.T) { - key := deriveScopeKey("#bc") + key := scopestore.FromName("#bc").TransportKey if len(key) != 16 { t.Errorf("expected 16 bytes, got %d", len(key)) } } func TestDeriveScopeKey_Deterministic(t *testing.T) { - a := deriveScopeKey("#bc") - b := deriveScopeKey("#bc") + a := scopestore.FromName("#bc").TransportKey + b := scopestore.FromName("#bc").TransportKey if hex.EncodeToString(a) != hex.EncodeToString(b) { t.Error("expected same key for same input") } @@ -238,7 +239,7 @@ func TestDeriveScopeKey_Deterministic(t *testing.T) { func TestDeriveScopeKey_KnownValue(t *testing.T) { // SHA256("#bc")[:16] — pin the exact derivation so changes are caught - key := deriveScopeKey("#bc") + key := scopestore.FromName("#bc").TransportKey got := hex.EncodeToString(key) // generate this once: echo -n "#bc" | sha256sum | cut -c1-32 const want = "84509cfe73d94f7f6a8299e6bcdb8a3c" @@ -248,8 +249,8 @@ func TestDeriveScopeKey_KnownValue(t *testing.T) { } func TestDeriveScopeKey_DifferentInputs(t *testing.T) { - a := deriveScopeKey("#bc") - b := deriveScopeKey("#other") + a := scopestore.FromName("#bc").TransportKey + b := scopestore.FromName("#other").TransportKey if hex.EncodeToString(a) == hex.EncodeToString(b) { t.Error("expected different keys for different inputs") } diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index bcc9ca5..5dfe7d7 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -11,11 +11,13 @@ import ( "encoding/hex" "encoding/json" "fmt" + "slices" "strings" "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/lora" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" "github.com/google/uuid" "github.com/meshcore-go/meshcore-go" ) @@ -694,14 +696,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ var matchedScope *string if packet.RouteType() == meshcore.RouteTypeTransportFlood || packet.RouteType() == meshcore.RouteTypeTransportDirect { - for _, entry := range w.scopes.Entries() { - code := computeTransportCode(entry.TransportKey, packet.PayloadType(), packet.Payload) - if code == packet.TransportCode1 { - s := entry.Name - matchedScope = &s - break - } - } + matchedScope = matchTransportScope(w.scopes.Entries(), iata, packet.PayloadType(), packet.Payload, packet.TransportCode1) } var scopeID *int32 if matchedScope != nil { @@ -930,6 +925,26 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } } +// matchTransportScope never treats catalogue membership as forwarding evidence. +// Multiple candidate names matching the short code are ambiguous, not first-wins. +func matchTransportScope(entries []scopestore.Entry, iata string, payloadType uint8, payload []byte, code uint16) *string { + var matched *string + for _, entry := range entries { + if entry.IATAs != nil && !slices.Contains(entry.IATAs, iata) { + continue + } + if computeTransportCode(entry.TransportKey, payloadType, payload) != code { + continue + } + if matched != nil && *matched != entry.Name { + return nil + } + name := entry.Name + matched = &name + } + return matched +} + // computeTransportCode derives transport_code_1 from a transport key and packet payload. // code = HMAC-SHA256(key, payload_type_byte || payload)[0:2] as little-endian uint16. // Coerces reserved values 0x0000 → 0x0001 and 0xFFFF → 0xFFFE per §2.4. diff --git a/internal/ingest/scope_import_test.go b/internal/ingest/scope_import_test.go new file mode 100644 index 0000000..3c085e4 --- /dev/null +++ b/internal/ingest/scope_import_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "fmt" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/meshcore-go/meshcore-go" +) + +func TestTransportScopeRegionalAndAmbiguousMatches(t *testing.T) { + entry := scopestore.FromName("yow") + entry.IATAs = []string{"YOW"} + payload := []byte{0xde, 0xad, 0xbe, 0xef} + // Fixed HMAC-SHA256 fixture: SHA256("#yow")[:16], payload type 4. + if got := matchTransportScope([]scopestore.Entry{entry}, "YOW", 4, payload, 21698); got == nil || *got != "#yow" { + t.Fatal(got) + } + if got := matchTransportScope([]scopestore.Entry{entry}, "YVR", 4, payload, 21698); got != nil { + t.Fatal("imported key escaped source region") + } + entry.IATAs = nil + if got := matchTransportScope([]scopestore.Entry{entry}, "YVR", 4, payload, 21698); got == nil { + t.Fatal("manual key was region-restricted") + } + // Distinct keys with the same short code for this payload must remain unknown. + a, b := scopestore.FromName("collision-320"), scopestore.FromName("collision-322") + for _, entries := range [][]scopestore.Entry{{a, b}, {b, a}} { + if computeTransportCode(entries[0].TransportKey, 4, payload) != 37018 { + t.Fatal("collision fixture changed") + } + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got != nil { + t.Fatal("ambiguous short code resolved", *got) + } + } +} + +type importCaptureDB struct{ *frameCaptureDB } + +func (s *importCaptureDB) GetTransportScopeByName(context.Context, string) (int32, error) { + return 123, nil +} + +func TestImportedScopeFlowsThroughPacketIngest(t *testing.T) { + for _, iata := range []string{"YOW", "YVR"} { + t.Run(iata, func(t *testing.T) { + w, base := newTestWorker() + capture := &importCaptureDB{&frameCaptureDB{stubDB: base}} + w.db = capture + key := scopestore.FromName("yow") + key.IATAs = []string{"YOW"} + store := scopestore.New() + store.Load([]scopestore.Entry{key}) + w.scopes = store + packet := buildGrpTxtPacket(t, 0x1a, make([]byte, 16)) + packet.Header = meshcore.MakeHeader(meshcore.RouteTypeTransportFlood, meshcore.PayloadTypeGrpTxt, 0) + packet.TransportCode1 = computeTransportCode(key.TransportKey, packet.PayloadType(), packet.Payload) + w.handlePacket(context.Background(), iata, "0102", packetEnvelope(t, packet)) + if len(capture.packets) != 1 { + t.Fatal("packet not stored") + } + got := capture.packets[0].ScopeID + if iata == "YOW" && (got == nil || *got != 123) { + t.Fatal("scope not persisted", got) + } + if iata == "YVR" && got != nil { + t.Fatal("scope assigned outside source region", got) + } + }) + } +} + +func BenchmarkTransportScopeCatalogue(b *testing.B) { + for _, count := range []int{0, 64, 1024} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + entries := []scopestore.Entry{scopestore.FromName("manual")} + for n := 0; n < count; n++ { + entry := scopestore.FromName(fmt.Sprintf("scope-%d", n)) + entry.IATAs = []string{fmt.Sprintf("R%02d", n/64)} + entries = append(entries, entry) + } + store := scopestore.New() + store.Load(entries) + payload := make([]byte, 128) + b.ReportAllocs() + for b.Loop() { + matchTransportScope(store.Entries(), "R00", 4, payload, 1234) + } + }) + } +} diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go new file mode 100644 index 0000000..a8b64ce --- /dev/null +++ b/internal/meshmapper/scopes.go @@ -0,0 +1,294 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package meshmapper imports the published regional scope catalogue, off the ingest path. +package meshmapper + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "slices" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +const ( + MaxScopes = 64 // per source; at most 16 configured sources (1024 imported identities) + MaxBody = 64 << 10 + PollInterval = 15 * time.Second // one source per tick, at most four requests/minute +) + +// Cache stores source provenance and freshness separately from packet evidence. +type Cache struct { + Payload json.RawMessage + ETag string + CheckedAt, AttemptedAt, NextAttempt time.Time + LastError string +} + +type Store interface { + GetScopeCatalogue(context.Context, string, string) (*Cache, error) + SaveScopeCatalogue(context.Context, string, string, Cache, []scopestore.Entry) error +} + +type source struct { + iata, url string + cache Cache + entries []scopestore.Entry + generated time.Time +} + +// Importer is owned by one background task; ScopeStore synchronizes its consumers. +type Importer struct { + store Store + scopes *scopestore.ScopeStore + manual []scopestore.Entry + sources []source + interval time.Duration + client *http.Client + retryAfter time.Time + onChange func(context.Context) +} + +// SetCacheInvalidator is wired once at startup, before the background task starts. +func (i *Importer) SetCacheInvalidator(fn func(context.Context)) { i.onChange = fn } + +// New restores validated snapshots before ingestion, without making HTTP requests. +func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) { + i := &Importer{store: store, scopes: scopes, manual: manual, interval: cfg.Interval(), client: &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} + if !cfg.Enabled { + return i, nil + } + keys := make([]string, 0, len(cfg.Sources)) + for key := range cfg.Sources { + keys = append(keys, key) + } + slices.Sort(keys) + for _, key := range keys { + s := source{iata: key, url: cfg.Sources[key]} + cached, err := store.GetScopeCatalogue(ctx, s.iata, s.url) + if err != nil { + return nil, fmt.Errorf("restore scope catalogue %s: %w", key, err) + } + if cached != nil { + s.cache = *cached + if len(cached.Payload) > 0 { + s.entries, s.generated, err = decode(cached.Payload, key) + if err != nil { + return nil, fmt.Errorf("invalid saved scope catalogue %s: %w", key, err) + } + } + if cached.LastError == "HTTP 429" && cached.NextAttempt.After(i.retryAfter) { + i.retryAfter = cached.NextAttempt + } + } + i.sources = append(i.sources, s) + i.log(s, "restored") + } + i.publish() + return i, nil +} + +// Refresh checks only one due source. The scheduler serializes calls every 15s. +func (i *Importer) Refresh(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + now := time.Now().UTC() + if now.Before(i.retryAfter) { + return nil + } + for n := range i.sources { + s := &i.sources[n] + if now.Before(s.cache.NextAttempt) { + continue + } + return i.refresh(ctx, s, now) + } + return nil +} + +func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error { + update := Cache{AttemptedAt: now, NextAttempt: now.Add(i.interval)} + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil) + if err != nil { + return err + } + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", "Beacon-MeshMapper-Scopes/1") + if len(s.cache.Payload) > 0 && s.cache.ETag != "" { + request.Header.Set("If-None-Match", s.cache.ETag) + } + response, err := i.client.Do(request) + var entries []scopestore.Entry + var generated time.Time + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + update.LastError = "request failed" // do not persist untrusted response bodies or URLs from errors + } else { + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + body, readErr := io.ReadAll(io.LimitReader(response.Body, MaxBody+1)) + if readErr == nil { + entries, generated, readErr = decode(body, s.iata) + } + if readErr != nil { + update.LastError = "invalid response" + } else { + update.Payload = body + } + case http.StatusNotModified: + if len(s.cache.Payload) == 0 || s.cache.ETag == "" { + update.LastError = "304 without cached catalogue" + } + default: + update.LastError = fmt.Sprintf("HTTP %d", response.StatusCode) + if response.StatusCode == http.StatusTooManyRequests || response.StatusCode == http.StatusServiceUnavailable { + // The published contract uses seconds. Ignore invalid/overflowing headers. + if seconds, parseErr := strconv.ParseInt(response.Header.Get("Retry-After"), 10, 64); parseErr == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) { + until := now.Add(time.Duration(seconds) * time.Second) + if until.After(update.NextAttempt) { + update.NextAttempt = until + } + } + } + } + if update.LastError == "" { + etag := response.Header.Get("ETag") + if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") { + update.LastError = "invalid ETag" + update.Payload = nil + entries = nil + } else { + update.ETag = etag + if response.StatusCode == http.StatusNotModified && etag == "" { + update.ETag = s.cache.ETag + } + update.CheckedAt = now + } + } + } + if update.LastError == "HTTP 429" { + i.retryAfter = update.NextAttempt + } + if err := i.store.SaveScopeCatalogue(ctx, s.iata, s.url, update, entries); err != nil { + // Avoid a retry every scheduler tick during a DB outage; never publish uncommitted keys. + s.cache.NextAttempt = update.NextAttempt + return fmt.Errorf("persist scope catalogue %s: %w", s.iata, err) + } + if len(update.Payload) > 0 { + s.cache.Payload = update.Payload + s.entries = entries + s.generated = generated + } + if !update.CheckedAt.IsZero() { + s.cache.CheckedAt = update.CheckedAt + s.cache.ETag = update.ETag + } + s.cache.AttemptedAt = now + s.cache.NextAttempt = update.NextAttempt + s.cache.LastError = update.LastError + if len(update.Payload) > 0 { + i.publish() + if i.onChange != nil { + i.onChange(ctx) + } + } + i.log(*s, "checked") + return nil +} + +func (i *Importer) publish() { + byName := make(map[string]scopestore.Entry, len(i.manual)) + for _, e := range i.manual { + byName[e.Name] = e + } + for _, s := range i.sources { + for _, candidate := range s.entries { + entry, exists := byName[candidate.Name] + if exists && entry.IATAs == nil { + continue + } // manual metadata/keys win + if !exists { + entry = candidate + } + entry.IATAs = append(entry.IATAs, s.iata) + byName[entry.Name] = entry + } + } + names := make([]string, 0, len(byName)) + for name := range byName { + names = append(names, name) + } + slices.Sort(names) + entries := make([]scopestore.Entry, 0, len(names)) + for _, name := range names { + entries = append(entries, byName[name]) + } + i.scopes.Load(entries) +} + +func (i *Importer) log(s source, action string) { + level := slog.LevelInfo + if s.cache.LastError != "" { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, "MeshMapper scopes "+action, "component", "meshmapper.scopes", "iata", s.iata, "source", s.url, + "names", len(s.entries), "generated_at", s.generated, "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError) +} + +func decode(body []byte, iata string) ([]scopestore.Entry, time.Time, error) { + var document struct { + GeneratedAt time.Time `json:"generated_at"` + Region string `json:"region"` + Zones []string `json:"zones"` + Repeaters, Scoped *int + Scopes *[]struct { + Name string + Repeaters, Default *int + Monitored, Wardriving *bool + } + } + if len(body) > MaxBody || !utf8.Valid(body) { + return nil, time.Time{}, fmt.Errorf("invalid catalogue size/encoding") + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, time.Time{}, err + } + if document.GeneratedAt.IsZero() || document.Region != iata || len(document.Zones) != 1 || document.Zones[0] != iata || + document.Repeaters == nil || document.Scoped == nil || *document.Scoped < 0 || *document.Repeaters < *document.Scoped || + document.Scopes == nil || len(*document.Scopes) > MaxScopes { + return nil, time.Time{}, fmt.Errorf("invalid regional catalogue") + } + entries := make([]scopestore.Entry, 0, len(*document.Scopes)) + seen := map[string]bool{} + for _, s := range *document.Scopes { + if s.Name == "" || len(s.Name) > 128 || strings.TrimSpace(s.Name) != s.Name || strings.ContainsFunc(s.Name, unicode.IsControl) || + s.Repeaters == nil || s.Default == nil || s.Monitored == nil || s.Wardriving == nil || + *s.Default < 0 || *s.Repeaters < *s.Default || *s.Repeaters > *document.Scoped { + return nil, time.Time{}, fmt.Errorf("invalid scope entry") + } + entry := scopestore.FromName(s.Name) + if len(entry.Name) < 2 || seen[entry.Name] { + return nil, time.Time{}, fmt.Errorf("empty or duplicate scope name") + } + seen[entry.Name] = true + entries = append(entries, entry) + } + return entries, document.GeneratedAt, nil +} diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go new file mode 100644 index 0000000..66d854a --- /dev/null +++ b/internal/meshmapper/scopes_test.go @@ -0,0 +1,253 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func catalogue(names ...string) []byte { + rows := make([]map[string]any, 0, len(names)) + for _, name := range names { + rows = append(rows, map[string]any{"name": name, "repeaters": 0, "default": 0, "monitored": true, "wardriving": false}) + } + b, _ := json.Marshal(map[string]any{"generated_at": "2026-09-27T16:58:49Z", "region": "YOW", "zones": []string{"YOW"}, "repeaters": 5, "scoped": 3, "scopes": rows}) + return b +} + +type memoryStore struct { + rows map[string]Cache + fail bool + saved []scopestore.Entry +} + +func (s *memoryStore) GetScopeCatalogue(_ context.Context, iata, url string) (*Cache, error) { + row, ok := s.rows[iata+url] + if !ok { + return nil, nil + } + return &row, nil +} +func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, next Cache, entries []scopestore.Entry) error { + if s.fail { + return errors.New("offline") + } + old := s.rows[iata+url] + if next.Payload == nil { + next.Payload = old.Payload + } + if next.CheckedAt.IsZero() { + next.CheckedAt = old.CheckedAt + next.ETag = old.ETag + } + s.rows[iata+url] = next + s.saved = append(s.saved, entries...) + return nil +} + +func names(store *scopestore.ScopeStore) []string { + out := []string{} + for _, e := range store.Entries() { + out = append(out, e.Name) + } + return out +} + +func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { + ctx := context.Background() + status, body, etag, after := 200, catalogue("yow", "manual"), `"v1"`, "" + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls > 1 && r.Header.Get("If-None-Match") != `"v1"` { + t.Errorf("missing conditional refresh: %v", r.Header) + } + w.Header().Set("ETag", etag) + w.Header().Set("Retry-After", after) + w.WriteHeader(status) + if status != 304 { + _, _ = w.Write(body) + } + })) + defer server.Close() + cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} + store := &memoryStore{rows: map[string]Cache{}} + scopes := scopestore.New() + manual := scopestore.FromName("manual") + manual.TransportKey = []byte("manual wins") + imp, err := New(ctx, cfg, store, scopes, []scopestore.Entry{manual}) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + check := func() { + t.Helper() + now = now.Add(time.Minute) + if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { + t.Fatal(err) + } + } + check() + if got := names(scopes); !reflect.DeepEqual(got, []string{"#manual", "#yow"}) { + t.Fatal(got) + } + if string(scopes.Entries()[0].TransportKey) != "manual wins" || scopes.Entries()[0].IATAs != nil { + t.Fatal("manual key replaced/restricted") + } + if !reflect.DeepEqual(scopes.Entries()[1].IATAs, []string{"YOW"}) { + t.Fatal("import not bound to region") + } + generated := imp.sources[0].generated + status = 304 + check() + checked := imp.sources[0].cache.CheckedAt + if checked != now || imp.sources[0].generated != generated { + t.Fatal("304 lost freshness or changed generation time") + } + for _, failure := range []int{200, 404, 503, 302} { + status, body = failure, []byte(`{"scopes":null}`) + check() + if imp.sources[0].cache.CheckedAt != checked || len(scopes.Entries()) != 2 { + t.Fatal("failed refresh replaced known catalogue") + } + } + status, after = 429, "7200" + check() + if !imp.retryAfter.Equal(now.Add(2 * time.Hour)) { + t.Fatal("Retry-After ignored") + } + restored := scopestore.New() + restart, err := New(ctx, cfg, store, restored, []scopestore.Entry{manual}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(names(restored), names(scopes)) { + t.Fatal("outage restart lost cached names") + } + before := calls + if err := restart.Refresh(ctx); err != nil { + t.Fatal(err) + } + if calls != before { + t.Fatal("restart ignored shared rate-limit cooldown") + } + status, after, body = 200, "", catalogue() + check() + if got := names(scopes); !reflect.DeepEqual(got, []string{"#manual"}) { + t.Fatalf("empty snapshot: %v", got) + } + store.fail = true + body = catalogue("new") + now = now.Add(time.Hour) + if err := imp.refresh(ctx, &imp.sources[0], now); err == nil || len(scopes.Entries()) != 1 { + t.Fatal("uncommitted names became active") + } +} + +func TestDecodeCatalogueBoundaries(t *testing.T) { + valid := string(catalogue("yow", "YOW", "$private")) + bad := []string{ + `null`, `{}`, `[]`, valid + `{}`, + strings.Replace(valid, `"region":"YOW"`, `"region":"PNW"`, 1), + strings.Replace(valid, `"zones":["YOW"]`, `"zones":["YOW","YVR"]`, 1), + strings.Replace(valid, `"scoped":3`, `"scoped":6`, 1), + strings.Replace(valid, `"repeaters":0`, `"repeaters":-1`, 1), + strings.Replace(valid, `"monitored":true,`, ``, 1), + strings.Replace(valid, `"generated_at":"2026-09-27T16:58:49Z"`, `"generated_at":null`, 1), + string(catalogue("yow", "#yow")), string(catalogue("#")), string(catalogue(" yow")), string(catalogue("a\nb")), + strings.Repeat(" ", MaxBody+1), + } + tooMany := []string{} + for n := 0; n <= MaxScopes; n++ { + tooMany = append(tooMany, fmt.Sprint(n)) + } + bad = append(bad, string(catalogue(tooMany...))) + for n, raw := range bad { + if _, _, err := decode([]byte(raw), "YOW"); err == nil { + t.Errorf("invalid case %d accepted", n) + } + } + entries, _, err := decode([]byte(valid), "YOW") + if err != nil || len(entries) != 3 { + t.Fatal(entries, err) + } + if entries[0].Name != "#yow" || entries[1].Name != "#YOW" || entries[2].Name != "$private" { + t.Fatal("case/prefix changed", entries) + } +} + +func TestNoRedirectAndResponseBounds(t *testing.T) { + targetCalls := 0 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { targetCalls++ })) + defer target.Close() + mode := "redirect" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch mode { + case "redirect": + http.Redirect(w, r, target.URL, 302) + case "oversize": + _, _ = w.Write([]byte(strings.Repeat("x", MaxBody+1))) + case "timeout": + select { + case <-r.Context().Done(): + case <-time.After(time.Second): + } + case "304": + w.WriteHeader(304) + } + })) + defer server.Close() + store := &memoryStore{rows: map[string]Cache{}} + imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + imp.client.Timeout = 20 * time.Millisecond + for _, value := range []string{"redirect", "oversize", "timeout", "304"} { + mode = value + if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil { + t.Fatal(err) + } + if imp.sources[0].cache.LastError == "" || len(imp.sources[0].entries) != 0 { + t.Fatal("invalid response accepted", value) + } + } + if targetCalls != 0 { + t.Fatal("followed redirect") + } +} + +func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { + ctx := context.Background() + store := &memoryStore{rows: map[string]Cache{}} + for _, iata := range []string{"YOW", "YVR"} { + store.rows[iata+iata] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata))} + } + scopes := scopestore.New() + _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "YOW", "YVR": "YVR"}}, store, scopes, nil) + if err != nil || len(scopes.Entries()) != 1 || !reflect.DeepEqual(scopes.Entries()[0].IATAs, []string{"YOW", "YVR"}) { + t.Fatal(scopes.Entries(), err) + } + _, err = New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "changed-source"}}, store, scopes, nil) + if err != nil || len(scopes.Entries()) != 0 { + t.Fatal("removed/changed source remained active", err) + } + // Disabled mode never calls the store, even if URLs remain configured. + _, err = New(ctx, config.MeshMapperScopesConfig{Sources: map[string]string{"YOW": "YOW"}}, nil, scopes, nil) + if err != nil { + t.Fatal(err) + } +} diff --git a/internal/scopestore/scopestore.go b/internal/scopestore/scopestore.go index 8abbac3..b95bad8 100644 --- a/internal/scopestore/scopestore.go +++ b/internal/scopestore/scopestore.go @@ -5,13 +5,29 @@ // loaded from the database at startup. package scopestore -import "sync" +import ( + "crypto/sha256" + "strings" + "sync" +) // Entry holds a single transport scope key and its metadata. type Entry struct { Name string - TransportKey []byte // 16 bytes - KeyFingerprint []byte // 8 bytes + TransportKey []byte // 16 bytes + KeyFingerprint []byte // 8 bytes + IATAs []string // nil for manual keys; imported candidates are regional +} + +// FromName uses the same case-sensitive derivation for manual and imported scopes. +func FromName(name string) Entry { + if !strings.HasPrefix(name, "#") && !strings.HasPrefix(name, "$") { + name = "#" + name + } + h := sha256.Sum256([]byte(name)) + key := h[:16] + fingerprint := sha256.Sum256(key) + return Entry{Name: name, TransportKey: key, KeyFingerprint: fingerprint[:8]} } // ScopeStore holds all known transport scope keys in memory. From 48138d1a7e0f5ea116e144d475a8b4175f0a9e2f Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 18:07:07 -0400 Subject: [PATCH 8/9] test(scopes): run race checks on the Linux CI runner --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6be1def..6690234 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,9 @@ jobs: - name: Test run: go test ./... + - name: Verify scope refresh concurrency + run: go test -race ./internal/meshmapper ./internal/scopestore + - name: Verify backup startup against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable From 235e2e0fd672febf8e236d5b5859ef3d660028df Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 27 Sep 2026 18:10:20 -0400 Subject: [PATCH 9/9] test(scopes): synchronize the timeout fixture state --- internal/meshmapper/scopes_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go index 66d854a..f84fcc1 100644 --- a/internal/meshmapper/scopes_test.go +++ b/internal/meshmapper/scopes_test.go @@ -12,6 +12,7 @@ import ( "net/http/httptest" "reflect" "strings" + "sync/atomic" "testing" "time" @@ -193,9 +194,10 @@ func TestNoRedirectAndResponseBounds(t *testing.T) { targetCalls := 0 target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { targetCalls++ })) defer target.Close() - mode := "redirect" + var mode atomic.Value + mode.Store("redirect") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch mode { + switch mode.Load().(string) { case "redirect": http.Redirect(w, r, target.URL, 302) case "oversize": @@ -217,7 +219,7 @@ func TestNoRedirectAndResponseBounds(t *testing.T) { } imp.client.Timeout = 20 * time.Millisecond for _, value := range []string{"redirect", "oversize", "timeout", "304"} { - mode = value + mode.Store(value) if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil { t.Fatal(err) }