diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beb7284..328b452 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,9 @@ jobs: - name: Test run: go test ./... + - name: Verify scope refresh concurrency + run: go test -race ./internal/meshmapper ./internal/scopestore + - name: Verify backup startup against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable @@ -55,7 +58,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index 5418223..3ecf2d5 100644 --- a/README.md +++ b/README.md @@ -533,3 +533,41 @@ log: `LOG_LEVEL` and `LOG_FORMAT` override file settings; empty settings use `info` and `text`. Invalid values prevent startup. Configuration-loading failures can use the bootstrap text logger before file settings are available; failures after initialization retain error severity at every supported level. Records include a component field. Ingest workers also include their broker name, and HTTP completion records include the validated client address, route, status and duration. Query strings and protocol hello payloads are excluded. Expected ingest skips and routine WebSocket lifecycle details are debug-level. Changing the application's format does not change Caddy/Apache access logs or their fail2ban configuration. Collect/rotate stderr through Docker or systemd. + +### Analytics retention + +Hourly traffic, payload, observer activity, talker, advertiser, Signal and Paths +summaries retain 30 days independently of `packets.retention`. Cleanup saves only +aggregates before deleting each packet batch, in the same transaction. Raw +packets, observations and message bodies still expire under packet retention. +The summaries use UTC hourly buckets and appear on the normal view-refresh cycle. +`mv_hourly_iata_stats` now covers 30 days instead of migration 001's seven days; +`/stats/observations?since=` can therefore return retained summaries older than a week. + +Migration 039 starts from data still present; previously deleted history cannot +be reconstructed. Telemetry has its own retention setting. Packet drill-down, +sub-hour observer activity, exact observer comparison and current entity/scope +counts continue to describe retained raw data or current entities, rather than +claiming archived packet detail. Archived summaries expire without waiting for +new packet deletions. A failed archive leaves its entire raw batch intact. + +### Observer monitoring metrics + +Observer activity returns buckets in `[windowStart, windowEnd)`, including the current partial bucket for live requests, plus +`generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the +sum of stored observations in those buckets; repeated broker delivery of the +same retained packet/observer pair counts once. Unknown payload types appear as +`-1` rather than disappearing. Freshness fields are measured at `generatedAt` even for historical `until` requests; `recordedPackets` alone follows the selected window. `lastCompleteHour` uses the previous complete UTC +hour and includes its own start/end; `latestRecordedAt` is the latest retained +reception timestamp. Missing records do not prove downtime. The optional `until` +(epoch milliseconds, within the last 30 days) aligns two observers' charts. + +The existing observer `observationCount` remains a legacy cumulative presence +counter for compatibility, including status/neighbour events. It is not a +period packet total. Broker presence and packet-arrival timestamps are now +updated separately; this cannot reconstruct previously overwritten timestamps. +The initial analytics archive migration includes unknown-type activity. Existing +development previews that used the earlier draft require a separate operator repair; +radio samples already discarded for those legacy rows cannot be recovered. + +Saved-route evidence: see [the operator guide](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/app_documentation/saved-route-evidence.md). diff --git a/cmd/beacon/main.go b/cmd/beacon/main.go index 8e322e6..1fa2ea3 100644 --- a/cmd/beacon/main.go +++ b/cmd/beacon/main.go @@ -29,6 +29,7 @@ import ( "github.com/MeshCore-Beacon/beacon-server/internal/ingest" "github.com/MeshCore-Beacon/beacon-server/internal/keystore" "github.com/MeshCore-Beacon/beacon-server/internal/logging" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" "github.com/MeshCore-Beacon/beacon-server/internal/presence" "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" @@ -189,6 +190,16 @@ func main() { } scopes.Load(scopeEntries) slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup") + var scopeImporter *meshmapper.Importer + if cfg.MeshMapper.Scopes.Enabled { + restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second) + scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, scopes, scopeEntries) + cancelRestore() + if err != nil { + slog.Error("failed to restore MeshMapper scope catalogues", "component", "startup", "error", err) + os.Exit(1) + } + } // ── Build channel keystore ────────────────────────────────────────────── entries := make(map[string][]keystore.Entry) @@ -281,6 +292,9 @@ func main() { ) if cr, ok := reader.(*cache.CachedReader); ok { + if scopeImporter != nil { + scopeImporter.SetCacheInvalidator(cr.InvalidateScopeNames) + } broker1.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver) broker2.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver) } @@ -301,6 +315,9 @@ func main() { } tasks = append(tasks, background.ObserverCleanupTask(coalescer, resolved.ObserverDeleteAfter, resolved.CleanupInterval, onDelete)) } + if scopeImporter != nil { + tasks = append(tasks, background.Task{Name: "meshmapper.scopes", Interval: meshmapper.PollInterval, Run: scopeImporter.Refresh}) + } scheduler := background.New(tasks) go scheduler.Start(ctx) diff --git a/config.yaml.example b/config.yaml.example index a0a86cc..3e2a66f 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -214,6 +214,21 @@ cache: # allow_countries: [CA, US] # allow_continents: [NA] +# Optional public MeshMapper scope-name discovery. Manual scopes remain authoritative. +# No API key is needed. Sources must belong to a configured region's IATAs. +# Group endpoints are not supported: their names cannot be attributed to each IATA. +# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping. +# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names. +# Refresh success/failure/freshness is logged under component=meshmapper.scopes. +# Removing/turning off a source deactivates its imported matching keys on restart; +# historical scope identities and recorded evidence are retained. +#meshmapper: +# scopes: +# enabled: false +# refresh_interval: 1h # 5m-24h; one source checked per 15s tick +# sources: +# YOW: https://yow.meshmapper.net/get_scopes.php + # Background task intervals. # Shorter intervals are useful during initial deployment to confirm data is # flowing. Back off to 1h or more once stable. diff --git a/db/analytics_retention_integration_test.go b/db/analytics_retention_integration_test.go new file mode 100644 index 0000000..10e3333 --- /dev/null +++ b/db/analytics_retention_integration_test.go @@ -0,0 +1,295 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "os" + "strings" + "testing" + "time" +) + +var retainedViews = []string{"mv_hourly_iata_stats", "mv_payload_breakdown_by_iata", "mv_top_observers_by_iata", "mv_top_talkers_by_iata", "mv_top_advertisers_by_iata", "mv_observer_activity_hourly", "mv_signal_stats_hourly", "mv_path_stats_hourly"} + +func analyticsSnapshot(t *testing.T, ctx context.Context, tx pgx.Tx) map[string]string { + t.Helper() + result := map[string]string{} + for _, view := range retainedViews { + if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW "+view); err != nil { + t.Fatal(err) + } + var rows string + if err := tx.QueryRow(ctx, "SELECT COALESCE(jsonb_agg(r ORDER BY r::text),'[]')::text FROM (SELECT to_jsonb(v) r FROM "+view+" v) s").Scan(&rows); err != nil { + t.Fatal(err) + } + result[view] = rows + } + return result +} + +func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash,last_heard_at) VALUES ('\x01',NOW()-interval '5 days'); INSERT INTO observers(id) VALUES ('00000000-0000-0000-0000-000000000001')`); err != nil { + t.Fatal(err) + } + var schema string + if err := tx.QueryRow(ctx, "SELECT current_schema()").Scan(&schema); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + conn := tx.Conn() + quoted := pgx.Identifier{schema}.Sanitize() + t.Cleanup(func() { + if _, err := conn.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error(err) + } + }) + if _, err := conn.Exec(ctx, "SET search_path TO "+quoted); err != nil { + t.Fatal(err) + } + writer, err := pgx.Connect(ctx, os.Getenv("BEACON_TEST_POSTGRES_DSN")) + if err != nil { + t.Fatal(err) + } + defer writer.Close(context.Background()) + if _, err = writer.Exec(ctx, "SET search_path TO "+quoted); err != nil { + t.Fatal(err) + } + ingest, err := writer.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer ingest.Rollback(context.Background()) + // An in-flight FK insert holds KEY SHARE on the packet until it commits. + if _, err = ingest.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\x01','00000000-0000-0000-0000-000000000001','YVR',NOW()-interval '5 days')`); err != nil { + t.Fatal(err) + } + q := sqlc.New(conn) + params := sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: time.Now().Add(-72 * time.Hour), Valid: true}, BatchSize: 1000} + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("must skip in-flight reception: %d %v", n, err) + } + if err := ingest.Commit(ctx); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 1 { + t.Fatalf("retry after ingestion: %d %v", n, err) + } + var count int + if err := conn.QueryRow(ctx, "SELECT observation_count FROM analytics_hourly_iata_stats").Scan(&count); err != nil || count != 1 { + t.Fatalf("committed reception not archived: %d %v", count, err) + } + // A second cleanup runner must wait before touching shared aggregate keys. + first, err := writer.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer first.Rollback(context.Background()) + if _, err := sqlc.New(first).DeleteOldPackets(ctx, params); err != nil { + t.Fatal(err) + } + if _, err := conn.Exec(ctx, "SET statement_timeout='300ms'"); err != nil { + t.Fatal(err) + } + if _, err := q.DeleteOldPackets(ctx, params); err == nil { + t.Fatalf("overlapping cleanup did not wait for the first transaction: %v", err) + } + if _, err := conn.Exec(ctx, "SET statement_timeout=0"); err != nil { + t.Fatal(err) + } + if err := first.Commit(ctx); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("cleanup lock was not released: %d %v", n, err) + } +} + +// Minimal real tables keep this regression runnable against an empty CI database. +func analyticsTables(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + isolateStatsSchema(t, ctx, tx) + _, err := tx.Exec(ctx, ` +CREATE TABLE packets (packet_hash bytea PRIMARY KEY, payload_type smallint, payload_version smallint, + route_type smallint, raw_payload bytea, raw_header bytea, origin_pubkey bytea, + first_heard_at timestamptz, last_heard_at timestamptz); +CREATE INDEX ON packets(last_heard_at); +CREATE TABLE packet_observations (id bigserial PRIMARY KEY, packet_hash bytea REFERENCES packets ON DELETE CASCADE, + observer_id uuid NOT NULL, iata char(3) NOT NULL, heard_at timestamptz NOT NULL, + path_length_byte smallint, hash_size smallint, hop_count smallint, path_bytes bytea, + snr real, rssi smallint, airtime_ms real, payload_type smallint, UNIQUE(packet_hash,observer_id)); +CREATE TABLE observers (id uuid PRIMARY KEY, public_key bytea, display_name text, observer_type text); +CREATE TABLE nodes (id uuid PRIMARY KEY, public_key bytea UNIQUE, node_type smallint, name text); +CREATE TABLE channel_messages (id bigserial PRIMARY KEY, channel_id integer, packet_hash bytea UNIQUE REFERENCES packets ON DELETE CASCADE, + sender_name text, content text, sent_at timestamptz); +`) + if err != nil { + t.Fatal(err) + } +} + +func TestAnalyticsRetentionPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + _, err := tx.Exec(ctx, ` + INSERT INTO observers (id,public_key,display_name,observer_type) SELECT ('00000000-0000-0000-0000-00000000000'||i)::uuid,int4send(i),CASE WHEN i=1 THEN NULL ELSE 'Observer '||i END,CASE WHEN i=1 THEN NULL ELSE 'test' END FROM generate_series(1,3) i; + INSERT INTO nodes (id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2,'Repeater'); + INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) + SELECT int4send(i),4,0,CASE WHEN i%2=0 THEN 1 ELSE 2 END,'\x00','\x00','\x02',date_trunc('hour',NOW())-interval '5 days', + CASE WHEN i=4 THEN NOW() ELSE date_trunc('hour',NOW())-interval '5 days' END FROM generate_series(1,4) i; + INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,airtime_ms,payload_type) + SELECT packet_hash,('00000000-0000-0000-0000-00000000000'||n)::uuid,iata,first_heard_at+make_interval(secs=>n),0,1,0, + CASE WHEN n=1 THEN 0 ELSE NULL END,CASE WHEN n=1 THEN -100 ELSE NULL END,1.5,4 + FROM packets CROSS JOIN (VALUES ('YVR',1),('YVR',2),('YYZ',3)) v(iata,n); + INSERT INTO channel_messages (channel_id,packet_hash,sender_name,content,sent_at) + SELECT 1,packet_hash,'Sender','body that must expire',first_heard_at FROM packets; + CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS SELECT iata,date_trunc('hour',heard_at) AS hour,count(*) AS observation_count,count(DISTINCT packet_hash) AS unique_packets,count(DISTINCT observer_id) AS active_observers FROM packet_observations GROUP BY 1,2; + `) + if err != nil { + t.Fatal(err) + } + for _, migration := range []string{"016_mv_payload_breakdown.sql", "017_mv_top_observers.sql", "018_mv_top_talkers.sql", "019_mv_top_advertisers.sql", "021_mv_top_advertisers_route_type.sql", "032_mv_observer_activity.sql", "035_mv_signal_stats.sql", "036_mv_path_stats.sql"} { + applyStatsMigration(t, ctx, tx, migration) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + for _, name := range []string{"idx_mv_signal_stats_hourly_hour", "idx_mv_path_stats_hourly_hour"} { + var definition string + if err := tx.QueryRow(ctx, "SELECT pg_get_indexdef(to_regclass($1))", name).Scan(&definition); err != nil || !strings.Contains(definition, "(hour)") { + t.Fatalf("missing unfiltered time index %s: %s %v", name, definition, err) + } + } + before := analyticsSnapshot(t, ctx, tx) + q := sqlc.New(tx) + cutoff := time.Now().Add(-72 * time.Hour) + // Failure in one archive must roll back every archive write and the raw deletion. + if _, err := tx.Exec(ctx, `SAVEPOINT archive_failure; ALTER TABLE analytics_signal_stats_hourly ADD CONSTRAINT fail_archive CHECK(receptions < 0) NOT VALID`); err != nil { + t.Fatal(err) + } + if _, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}); err == nil { + t.Fatal("expected archive failure") + } + if _, err := tx.Exec(ctx, "ROLLBACK TO archive_failure"); err != nil { + t.Fatal(err) + } + if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 4 { + t.Fatal("archive failure deleted raw packets") + } + for i := 0; i < 3; i++ { + n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}) + if err != nil || n != 1 { + t.Fatalf("delete batch: %d %v", n, err) + } + } + if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 1 || countRows(t, ctx, tx, "SELECT count(*) FROM channel_messages") != 1 { + t.Fatal("raw packets/messages did not expire") + } + after := analyticsSnapshot(t, ctx, tx) + for _, view := range retainedViews { + if after[view] != before[view] { + t.Errorf("%s lost or double-counted history after raw expiry: before=%s after=%s", view, before[view], after[view]) + } + } + // Retry and migration journal retry preserve the exact summaries. + if n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1000}); err != nil || n != 0 { + t.Fatalf("retry: %d %v", n, err) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != before[view] { + t.Errorf("retry changed %s", view) + } + } + // Later receptions on a still-live packet must join its old bucket before expiry. + if _, err := tx.Exec(ctx, `INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000004'); INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,payload_type) + SELECT packet_hash,'00000000-0000-0000-0000-000000000004','YVR',first_heard_at,0,1,0,'NaN',-80,NULL FROM packets; + UPDATE packets SET last_heard_at=first_heard_at`); err != nil { + t.Fatal(err) + } + late := analyticsSnapshot(t, ctx, tx) + if countRows(t, ctx, tx, "SELECT COALESCE(SUM(observations),0) FROM mv_observer_activity_hourly WHERE payload_type=-1") != 1 { + t.Fatal("unknown-payload observation was omitted") + } + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != late[view] { + t.Errorf("late reception changed %s: %s != %s", view, rows, late[view]) + } + } + // Archived labels follow a current rename; no entity FK may erase retained history. + if _, err := tx.Exec(ctx, `UPDATE nodes SET name='Renamed'; UPDATE observers SET display_name=NULL,observer_type=NULL;`); err != nil { + t.Fatal(err) + } + analyticsSnapshot(t, ctx, tx) + if countRows(t, ctx, tx, "SELECT count(*) FROM mv_top_advertisers_by_iata WHERE name='Renamed'") != 2 { + t.Fatal("archive name did not follow rename") + } + if _, err := q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{Column1: pgtype.Interval{Microseconds: int64(30 * 24 * time.Hour / time.Microsecond), Valid: true}, Limit: 10}); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, "DELETE FROM nodes; DELETE FROM observers"); err != nil { + t.Fatal(err) + } + if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_top_advertisers_by_iata") != 2 { + t.Fatal("entity deletion erased archive") + } + // Independent expiry runs even when there are no packets left to delete. + for _, view := range retainedViews { + table := "analytics_" + view[3:] + at := "bucket" + if view == "mv_hourly_iata_stats" || view == "mv_signal_stats_hourly" || view == "mv_path_stats_hourly" { + at = "hour" + } + if _, err := tx.Exec(ctx, "UPDATE "+table+" SET "+at+"="+at+"-interval '31 days'"); err != nil { + t.Fatal(err) + } + } + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + for _, view := range retainedViews { + if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) != 0 { + t.Errorf("%s archive did not expire", view) + } + } + // More than one populated cohort, with shared hourly keys across the boundary. + if _, err := tx.Exec(ctx, ` +INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000001'); +INSERT INTO nodes (id,public_key,node_type) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2); +INSERT INTO packets (packet_hash,payload_type,route_type,origin_pubkey,first_heard_at,last_heard_at,raw_payload) +SELECT int4send(i),4,1,'\x02',date_trunc('hour',NOW())-interval '5 days',date_trunc('hour',NOW())-interval '5 days',decode(repeat('ab',1024),'hex') FROM generate_series(10000,11000) i; +INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) +SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YVR',first_heard_at,0,1,0,4,0,-100 FROM packets; +INSERT INTO channel_messages (packet_hash,sender_name,sent_at) SELECT packet_hash,'Sender',first_heard_at FROM packets; +`); err != nil { + t.Fatal(err) + } + populated := analyticsSnapshot(t, ctx, tx) + started := time.Now() + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + t.Logf("1001 packets with bodies and observations archived/deleted in %s", time.Since(started)) + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != populated[view] { + t.Errorf("batch boundary changed %s", view) + } + } + var archiveRows int + for _, view := range retainedViews { + archiveRows += countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) + } + if archiveRows > 12 { + t.Fatalf("expected compact rollups, got %d archive rows for 1001 packets", archiveRows) + } +} diff --git a/db/channel_scopes_integration_test.go b/db/channel_scopes_integration_test.go new file mode 100644 index 0000000..e02f576 --- /dev/null +++ b/db/channel_scopes_integration_test.go @@ -0,0 +1,184 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "encoding/json" + "net/http/httptest" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/api/handlers" + "github.com/MeshCore-Beacon/beacon-server/internal/ingest" + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestChannelMessageScopesPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"transport_scopes", "packets", "packet_observations", "channels", "channel_messages"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + _, err = tx.Exec(ctx, ` +INSERT INTO transport_scopes(id,name,transport_key,key_fingerprint) VALUES +(1,'#yow',decode(repeat('00',16),'hex'),decode(repeat('01',8),'hex')), +(2,'#can',decode(repeat('02',16),'hex'),decode(repeat('03',8),'hex')); +INSERT INTO channels(id,channel_hash,key_fingerprint,name,is_hashtag,key_known) VALUES (123,'\x11','\x01020304','#fixture',true,true); +INSERT INTO packets(packet_hash,payload_type,payload_version,route_type,transport_codes_present,scope_id,raw_payload,raw_header,first_heard_at,last_heard_at) +SELECT decode(lpad(to_hex(i),2,'0'),'hex'),5,0,1,transport,scope_id,'\x00','\x15',NOW(),NOW() +FROM (VALUES(1,true,1),(2,true,NULL),(3,false,NULL),(4,NULL,NULL),(5,true,2),(6,false,NULL)) v(i,transport,scope_id); +INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count) +SELECT packet_hash,md5(n::text)::uuid,'YOW',NOW(),0,1,0 FROM packets CROSS JOIN generate_series(1,2) n; +`) + if err != nil { + t.Fatal(err) + } + s := &Store{q: sqlc.New(tx)} + // Later matching reception must not relabel the immutable first stored packet. + scopeID := int32(1) + _, err = s.UpsertPacket(ctx, ingest.UpsertPacketParams{PacketHash: []byte{6}, PayloadType: 5, ScopeID: &scopeID, TransportCodes: []byte{1, 2, 3, 4}, RawPayload: []byte{0}, RawHeader: []byte{0}}) + if err != nil { + t.Fatal(err) + } + statuses := []api.ChannelScopeStatus{"matched", "unknown", "unscoped", "unavailable", "matched", "unscoped"} + wants := map[int64]*ingest.InsertedChannelMessage{} + for n, status := range statuses { + m := ingest.InsertChannelMessageParams{ChannelID: 123, PacketHash: []byte{byte(n + 1)}, SenderName: "Fixture", Content: "Synthetic message", SentAt: time.Now()} + inserted, err := s.InsertChannelMessage(ctx, m) + if err != nil || inserted == nil || inserted.ScopeStatus != status { + t.Fatalf("insert %d: %+v %v", n, inserted, err) + } + wants[inserted.ID] = inserted + duplicate, err := s.InsertChannelMessage(ctx, m) + if err != nil || duplicate != nil { + t.Fatal("duplicate changed live delivery", duplicate, err) + } + } + check := func(items []api.ChannelMessage) { + t.Helper() + if len(items) != 6 { + t.Fatalf("messages=%d", len(items)) + } + for _, m := range items { + want := wants[m.ID] + if want == nil || m.ScopeStatus != want.ScopeStatus { + t.Fatal("history differs from insertion", m) + } + if (m.Scope == nil) != (want.Scope == nil) || (m.Scope != nil && *m.Scope != *want.Scope) { + t.Fatal("scope differs", m) + } + if m.ObservationCount != 2 { + t.Fatal("duplicate observations changed message grain", m) + } + b, _ := json.Marshal(m) + var wire map[string]any + _ = json.Unmarshal(b, &wire) + if _, ok := wire["scope"]; !ok || wire["scopeStatus"] != string(m.ScopeStatus) { + t.Fatal("wire metadata absent", string(b)) + } + } + } + ch := int32(123) + for _, id := range []*int32{nil, &ch} { + p, err := s.ListChannelMessages(ctx, id, time.Time{}, 20, []string{"YOW"}, "", 0) + if err != nil { + t.Fatal(err) + } + check(p.Items) + } + hash, err := s.ListChannelMessagesByHash(ctx, []byte{0x11}, time.Time{}, 20, []string{"YOW"}, "", 0) + if err != nil { + t.Fatal(err) + } + check(hash.Items) + catchup, err := s.ListMessagesAfterID(ctx, 0, []string{"YOW"}, "", 20) + if err != nil { + t.Fatal(err) + } + check(catchup) + router := chi.NewRouter() + router.Mount("/channels", handlers.ChannelsRouter(s)) + router.Mount("/messages", handlers.MessagesRouter(s)) + for _, path := range []string{"/channels/123/messages", "/messages", "/messages?channelHash=11", "/messages/backfill?afterId=0"} { + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest("GET", path, nil).WithContext(ctx)) + if response.Code != 200 { + t.Fatalf("%s: %d %s", path, response.Code, response.Body.String()) + } + if path == "/messages/backfill?afterId=0" { + var items []api.ChannelMessage + if err := json.Unmarshal(response.Body.Bytes(), &items); err != nil { + t.Fatal(err) + } + check(items) + } else { + var body struct { + Items []api.ChannelMessage `json:"items"` + } + if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + check(body.Items) + } + } + filtered, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, []string{"YOW"}, "#yow", 0) + if err != nil || len(filtered.Items) != 1 || filtered.Items[0].Scope == nil || *filtered.Items[0].Scope != "#yow" { + t.Fatal(filtered, err) + } + catchup, err = s.ListMessagesAfterID(ctx, 0, []string{"YOW"}, "#yow", 20) + if err != nil || len(catchup) != 1 { + t.Fatal(catchup, err) + } + other, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, []string{"YVR"}, "", 0) + if err != nil || len(other.Items) != 0 { + t.Fatal("region escaped", other, err) + } + first, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 2, nil, "", 0) + if err != nil || !first.HasMore || first.NextCursor == nil { + t.Fatal(first, err) + } + second, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 2, nil, "", *first.NextCursor) + if err != nil || len(second.Items) != 2 || second.Items[0].ID >= *first.NextCursor { + t.Fatal("cursor drift", second, err) + } + // Even without the production cascade, expired packet evidence must not leak orphan messages. + if _, err := tx.Exec(ctx, "DELETE FROM packets"); err != nil { + t.Fatal(err) + } + expired, err := s.ListChannelMessages(ctx, &ch, time.Time{}, 20, nil, "", 0) + if err != nil || len(expired.Items) != 0 { + t.Fatal(expired, err) + } +} diff --git a/db/channels.go b/db/channels.go index 599230a..9082e5a 100644 --- a/db/channels.go +++ b/db/channels.go @@ -154,16 +154,16 @@ func (s *Store) GetChannel(ctx context.Context, channelID int32) (*api.Channel, return &channel, nil } -func (s *Store) InsertChannelMessage(ctx context.Context, m ingest.InsertChannelMessageParams) (bool, error) { +func (s *Store) InsertChannelMessage(ctx context.Context, m ingest.InsertChannelMessageParams) (*ingest.InsertedChannelMessage, error) { params := sqlc.InsertChannelMessageParams{ChannelID: int32(m.ChannelID), PacketHash: m.PacketHash, SenderName: &m.SenderName, Content: &m.Content, SentAt: pgtype.Timestamptz{Time: m.SentAt, Valid: true}} - _, err := s.q.InsertChannelMessage(ctx, params) + row, err := s.q.InsertChannelMessage(ctx, params) if errors.Is(err, pgx.ErrNoRows) { - return false, nil // duplicate + return nil, nil // duplicate } if err != nil { - return false, err + return nil, err } - return true, nil + return &ingest.InsertedChannelMessage{ID: row.ID, Scope: row.ScopeName, ScopeStatus: api.RecordedChannelScopeStatus(row.ScopeName, row.TransportCodesPresent)}, nil } func (s *Store) ListChannelMessages(ctx context.Context, channelID *int32, since time.Time, limit int32, iatas []string, scope string, cursor int64) (api.Page[api.ChannelMessage], error) { @@ -187,7 +187,7 @@ func (s *Store) ListChannelMessages(ctx context.Context, channelID *int32, since } messages = make([]api.ChannelMessage, 0, len(rows)) for _, v := range rows { - messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount)) + messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount, v.ScopeName, v.TransportCodesPresent)) } } else { rows, err := s.q.ListChannelMessages(ctx, sqlc.ListChannelMessagesParams{ @@ -207,7 +207,7 @@ func (s *Store) ListChannelMessages(ctx context.Context, channelID *int32, since } messages = make([]api.ChannelMessage, 0, len(rows)) for _, v := range rows { - messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount)) + messages = append(messages, toChannelMessage(v.ID, v.PacketHashHex, v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount, v.ScopeName, v.TransportCodesPresent)) } } @@ -241,7 +241,7 @@ func (s *Store) ListChannelMessagesByHash(ctx context.Context, hash []byte, sinc } messages := make([]api.ChannelMessage, 0, len(rows)) for _, v := range rows { - messages = append(messages, toChannelMessage(v.ID, hex.EncodeToString(v.PacketHash), v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount)) + messages = append(messages, toChannelMessage(v.ID, hex.EncodeToString(v.PacketHash), v.ChannelHash, v.SenderName, v.Content, v.SentAt, v.ObservationCount, v.ScopeName, v.TransportCodesPresent)) } var nextCursor *int64 if hasMore && len(messages) > 0 { @@ -275,6 +275,8 @@ func (s *Store) ListMessagesAfterID(ctx context.Context, afterID int64, iatas [] v.Content, v.SentAt, v.ObservationCount, + v.ScopeName, + v.TransportCodesPresent, )) } return items, nil diff --git a/db/meshmapper.go b/db/meshmapper.go new file mode 100644 index 0000000..3ddbb28 --- /dev/null +++ b/db/meshmapper.go @@ -0,0 +1,49 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "errors" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func (s *Store) GetScopeCatalogue(ctx context.Context, iata, url string) (*meshmapper.Cache, error) { + row, err := s.q.GetScopeCatalogue(ctx, sqlc.GetScopeCatalogueParams{Iata: iata, Url: url}) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + cache := &meshmapper.Cache{Payload: row.Payload, CheckedAt: row.CheckedAt.Time, AttemptedAt: row.AttemptedAt.Time, NextAttempt: row.NextAttempt.Time, LastError: row.LastError} + if row.Etag != nil { + cache.ETag = *row.Etag + } + return cache, nil +} + +func (s *Store) SaveScopeCatalogue(ctx context.Context, iata, url string, cache meshmapper.Cache, entries []scopestore.Entry) error { + params := sqlc.SaveScopeCatalogueParams{ + Iata: iata, Url: url, Payload: cache.Payload, LastError: cache.LastError, + CheckedAt: pgtype.Timestamptz{Time: cache.CheckedAt, Valid: !cache.CheckedAt.IsZero()}, + AttemptedAt: pgtype.Timestamptz{Time: cache.AttemptedAt, Valid: true}, + NextAttempt: pgtype.Timestamptz{Time: cache.NextAttempt, Valid: true}, + Names: []string{}, Keys: [][]byte{}, Fingerprints: [][]byte{}, + } + if !cache.CheckedAt.IsZero() { + params.Etag = &cache.ETag + } + for _, entry := range entries { + params.Names = append(params.Names, entry.Name) + params.Keys = append(params.Keys, entry.TransportKey) + params.Fingerprints = append(params.Fingerprints, entry.KeyFingerprint) + } + return s.q.SaveScopeCatalogue(ctx, params) +} diff --git a/db/meshmapper_integration_test.go b/db/meshmapper_integration_test.go new file mode 100644 index 0000000..06d7109 --- /dev/null +++ b/db/meshmapper_integration_test.go @@ -0,0 +1,139 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "os" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/meshmapper" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestMeshMapperCataloguePostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatal(err) + } + if err := RunMigrations(ctx, pool); err != nil { + pool.Close() + t.Fatal(err) + } + pool.Close() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + tx, err := conn.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback(context.Background()) + for _, table := range []string{"transport_scopes", "meshmapper_scope_catalogues"} { + if _, err := tx.Exec(ctx, "CREATE TEMP TABLE "+table+" (LIKE public."+table+" INCLUDING ALL) ON COMMIT DROP"); err != nil { + t.Fatal(err) + } + } + store := &Store{q: sqlc.New(tx)} + manual := scopestore.FromName("manual") + manual.TransportKey[0] ^= 1 + if err := store.UpsertTransportScope(ctx, manual.Name, "Operator label", manual.TransportKey, manual.KeyFingerprint); err != nil { + t.Fatal(err) + } + url := "https://yow.meshmapper.net/get_scopes.php" + if row, err := store.GetScopeCatalogue(ctx, "YOW", url); err != nil || row != nil { + t.Fatal(row, err) + } + now := time.Now().UTC().Truncate(time.Second) + payload := []byte(`{"generated_at":"2026-09-27T16:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":3,"scopes":[{"name":"yow","repeaters":0,"default":0,"monitored":true,"wardriving":false}]}`) + update := meshmapper.Cache{Payload: payload, ETag: `"v1"`, CheckedAt: now, AttemptedAt: now, NextAttempt: now.Add(time.Hour)} + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("manual"), scopestore.FromName("yow")}); err != nil { + t.Fatal(err) + } + manualRows, err := store.GetTransportScopes(ctx) + if err != nil || len(manualRows) != 1 || manualRows[0].TransportKey[0] != manual.TransportKey[0] { + t.Fatal("manual key changed", manualRows, err) + } + var label string + var imported bool + if err := tx.QueryRow(ctx, "SELECT display_name,imported_only FROM transport_scopes WHERE name='#manual'").Scan(&label, &imported); err != nil || label != "Operator label" || imported { + t.Fatal(label, imported, err) + } + row, err := store.GetScopeCatalogue(ctx, "YOW", url) + if err != nil || row.ETag != `"v1"` || !row.CheckedAt.Equal(now) { + t.Fatal(row, err) + } + scopes := scopestore.New() + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + if err != nil || len(scopes.Entries()) != 2 { + t.Fatal("restart did not restore imported membership", err) + } + // A conditional success preserves the document, advances checked time and clears a failure. + checked := now.Add(time.Hour) + if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{ETag: `"v1"`, CheckedAt: checked, AttemptedAt: checked, NextAttempt: checked.Add(time.Hour)}, nil); err != nil { + t.Fatal(err) + } + if err := store.SaveScopeCatalogue(ctx, "YOW", url, meshmapper.Cache{AttemptedAt: checked.Add(time.Minute), NextAttempt: checked.Add(2 * time.Hour), LastError: "HTTP 429"}, nil); err != nil { + t.Fatal(err) + } + row, err = store.GetScopeCatalogue(ctx, "YOW", url) + if err != nil || !row.CheckedAt.Equal(checked) || row.ETag != `"v1"` || row.LastError != "HTTP 429" || len(row.Payload) == 0 { + t.Fatal("failed refresh lost durable snapshot", row, err) + } + // Empty membership deactivates matching but retains the old scope identity for historical rows. + empty := []byte(`{"generated_at":"2026-09-27T17:58:49Z","region":"YOW","zones":["YOW"],"repeaters":5,"scoped":0,"scopes":[]}`) + update.Payload = empty + update.ETag = "" + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, nil); err != nil { + t.Fatal(err) + } + _, err = meshmapper.New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": url}}, store, scopes, manualRows) + if err != nil || len(scopes.Entries()) != 1 { + t.Fatal("removed imported membership remained active", err) + } + if _, err := store.GetTransportScopeByName(ctx, "#yow"); err != nil { + t.Fatal("deleted historical identity", err) + } + row, _ = store.GetScopeCatalogue(ctx, "YOW", url) + if row.ETag != "" { + t.Fatal("200 without ETag retained obsolete validator") + } + // Reconfiguring a known imported name as manual promotes it without losing its identity. + yow := scopestore.FromName("yow") + if err := store.UpsertTransportScope(ctx, yow.Name, "Local", yow.TransportKey, yow.KeyFingerprint); err != nil { + t.Fatal(err) + } + manualRows, err = store.GetTransportScopes(ctx) + if err != nil || len(manualRows) != 2 { + t.Fatal("manual promotion failed", err) + } + // A failed cache write also rolls back the identity insertion in the same statement. + if _, err := tx.Exec(ctx, "ALTER TABLE meshmapper_scope_catalogues ADD CHECK(last_error <> 'force-failure'); SAVEPOINT rejected"); err != nil { + t.Fatal(err) + } + update.LastError = "force-failure" + if err := store.SaveScopeCatalogue(ctx, "YOW", url, update, []scopestore.Entry{scopestore.FromName("uncommitted")}); err == nil { + t.Fatal("expected constraint failure") + } + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT rejected"); err != nil { + t.Fatal(err) + } + var count int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM transport_scopes WHERE name='#uncommitted'").Scan(&count); err != nil || count != 0 { + t.Fatal("partially committed catalogue", count, err) + } +} diff --git a/db/migrations/039_analytics_retention.sql b/db/migrations/039_analytics_retention.sql new file mode 100644 index 0000000..069204a --- /dev/null +++ b/db/migrations/039_analytics_retention.sql @@ -0,0 +1,613 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Archive only aggregates of deleted packet cohorts, never packet/message bodies +-- or raw paths. Live and archived cohorts are disjoint in every refresh snapshot. +-- Thirty days matches the longest analytics window; raw retention is independent. + + +CREATE TABLE IF NOT EXISTS analytics_hourly_iata_stats ( + iata char(3), + hour timestamptz, + observation_count bigint NOT NULL DEFAULT 0, + unique_packets bigint NOT NULL DEFAULT 0, + PRIMARY KEY (iata,hour) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_hourly_iata_stats_time ON analytics_hourly_iata_stats (hour); + +CREATE TABLE IF NOT EXISTS analytics_payload_breakdown_by_iata ( + iata char(3), + payload_type smallint, + bucket timestamptz, + count bigint NOT NULL DEFAULT 0, + PRIMARY KEY (iata,payload_type,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_payload_breakdown_by_iata_time ON analytics_payload_breakdown_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_observers_by_iata ( + iata char(3), + observer_id uuid, + bucket timestamptz, + observation_count bigint NOT NULL DEFAULT 0, + display_name text, + observer_type text, + PRIMARY KEY (iata,observer_id,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_observers_by_iata_time ON analytics_top_observers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_talkers_by_iata ( + iata char(3), + sender_name text, + bucket timestamptz, + message_count bigint NOT NULL DEFAULT 0, + last_sent timestamptz, + PRIMARY KEY (iata,sender_name,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_talkers_by_iata_time ON analytics_top_talkers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_advertisers_by_iata ( + iata char(3), + node_id uuid, + bucket timestamptz, + advert_count bigint NOT NULL DEFAULT 0, + flood_advert_count bigint NOT NULL DEFAULT 0, + direct_advert_count bigint NOT NULL DEFAULT 0, + last_heard timestamptz, + name text, + node_type smallint, + PRIMARY KEY (iata,node_id,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_advertisers_by_iata_time ON analytics_top_advertisers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_observer_activity_hourly ( + observer_id uuid, + payload_type smallint, + bucket timestamptz, + observations bigint NOT NULL DEFAULT 0, + airtime_ms real, + airtime_n bigint NOT NULL DEFAULT 0, + snr_sum real, + snr_n bigint NOT NULL DEFAULT 0, + snr_min real, + rssi_sum bigint, + rssi_n bigint NOT NULL DEFAULT 0, + PRIMARY KEY (observer_id,payload_type,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_observer_activity_hourly_time ON analytics_observer_activity_hourly (bucket); + +CREATE TABLE IF NOT EXISTS analytics_signal_stats_hourly ( + iata char(3), + hour timestamptz, + kind integer, + snr_bin integer, + rssi_bin integer, + receptions bigint NOT NULL DEFAULT 0, + snr_samples bigint NOT NULL DEFAULT 0, + snr_sum double precision, + rssi_samples bigint NOT NULL DEFAULT 0, + rssi_sum double precision, + PRIMARY KEY (iata,hour,kind,snr_bin,rssi_bin) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_signal_stats_hourly_time ON analytics_signal_stats_hourly (hour); + +CREATE TABLE IF NOT EXISTS analytics_path_stats_hourly ( + iata char(3), + hour timestamptz, + category integer, + hash_bytes integer, + entries integer, + receptions bigint NOT NULL DEFAULT 0, + PRIMARY KEY (iata,hour,category,hash_bytes,entries) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_path_stats_hourly_time ON analytics_path_stats_hourly (hour); + +DROP MATERIALIZED VIEW IF EXISTS mv_hourly_iata_stats; + +CREATE OR REPLACE VIEW analytics_live_hourly_iata_stats AS +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS +WITH combined AS ( + SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats + UNION ALL + SELECT iata, hour, observation_count, unique_packets FROM analytics_hourly_iata_stats + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observers_by_hour AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, observer_id + FROM packet_observations WHERE heard_at > NOW() - INTERVAL '30 days' + UNION + SELECT iata, bucket AS hour, observer_id FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observer_counts AS (SELECT iata, hour, COUNT(*) AS n FROM observers_by_hour GROUP BY iata,hour) +SELECT combined.iata, combined.hour, SUM(observation_count)::bigint AS observation_count, SUM(unique_packets)::bigint AS unique_packets, MAX(o.n)::bigint AS active_observers +FROM combined JOIN observer_counts o USING (iata,hour) GROUP BY combined.iata,combined.hour; + +CREATE UNIQUE INDEX idx_analytics_hourly_iata_stats_view ON mv_hourly_iata_stats (iata,hour); + + +DROP MATERIALIZED VIEW IF EXISTS mv_payload_breakdown_by_iata; + +CREATE OR REPLACE VIEW analytics_live_payload_breakdown_by_iata AS +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS +WITH combined AS ( + SELECT iata, payload_type, bucket, count FROM analytics_live_payload_breakdown_by_iata + UNION ALL + SELECT iata, payload_type, bucket, count FROM analytics_payload_breakdown_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, payload_type, bucket, SUM(count)::bigint AS count +FROM combined GROUP BY iata,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_payload_breakdown_by_iata_view ON mv_payload_breakdown_by_iata (iata,payload_type,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_observers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_observers_by_iata AS +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS +WITH combined AS ( + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_live_top_observers_by_iata + UNION ALL + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, observer_id, bucket, SUM(observation_count)::bigint AS observation_count, MAX(display_name) AS display_name, MAX(observer_type) AS observer_type +FROM combined GROUP BY iata,observer_id,bucket) +SELECT t.iata, t.observer_id, COALESCE(o.display_name,t.display_name) AS display_name, + COALESCE(o.observer_type,t.observer_type) AS observer_type, t.bucket, t.observation_count +FROM totals t LEFT JOIN observers o ON o.id=t.observer_id; + +CREATE UNIQUE INDEX idx_analytics_top_observers_by_iata_view ON mv_top_observers_by_iata (iata,observer_id,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_talkers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_talkers_by_iata AS +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN packet_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS +WITH combined AS ( + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_live_top_talkers_by_iata + UNION ALL + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_top_talkers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, sender_name, bucket, SUM(message_count)::bigint AS message_count, MAX(last_sent) AS last_sent +FROM combined GROUP BY iata,sender_name,bucket; + +CREATE UNIQUE INDEX idx_analytics_top_talkers_by_iata_view ON mv_top_talkers_by_iata (iata,sender_name,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_advertisers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_advertisers_by_iata AS +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN packet_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS +WITH combined AS ( + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_live_top_advertisers_by_iata + UNION ALL + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_top_advertisers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, node_id, bucket, SUM(advert_count)::bigint AS advert_count, SUM(flood_advert_count)::bigint AS flood_advert_count, SUM(direct_advert_count)::bigint AS direct_advert_count, MAX(last_heard) AS last_heard, MAX(name) AS name, MAX(node_type) AS node_type +FROM combined GROUP BY iata,node_id,bucket) +SELECT t.iata, t.node_id, COALESCE(n.name,t.name) AS name, COALESCE(n.node_type,t.node_type) AS node_type, + t.bucket, t.advert_count, t.flood_advert_count, t.direct_advert_count, t.last_heard +FROM totals t LEFT JOIN nodes n ON n.id=t.node_id; + +CREATE UNIQUE INDEX idx_analytics_top_advertisers_by_iata_view ON mv_top_advertisers_by_iata (iata,node_id,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_observer_activity_hourly; +CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS +WITH combined AS ( + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly + UNION ALL + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n +FROM combined GROUP BY observer_id,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); + + + +DROP MATERIALIZED VIEW IF EXISTS mv_signal_stats_hourly; + +CREATE OR REPLACE VIEW analytics_live_signal_stats_hourly AS +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); + +CREATE MATERIALIZED VIEW mv_signal_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_live_signal_stats_hourly + UNION ALL + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_signal_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, kind, snr_bin, rssi_bin, SUM(receptions)::bigint AS receptions, SUM(snr_samples)::bigint AS snr_samples, SUM(snr_sum)::double precision AS snr_sum, SUM(rssi_samples)::bigint AS rssi_samples, SUM(rssi_sum)::double precision AS rssi_sum +FROM combined GROUP BY iata,hour,kind,snr_bin,rssi_bin; + +CREATE UNIQUE INDEX idx_analytics_signal_stats_hourly_view ON mv_signal_stats_hourly (iata,hour,kind,snr_bin,rssi_bin); +CREATE INDEX idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly (hour); + + +DROP MATERIALIZED VIEW IF EXISTS mv_path_stats_hourly; + +CREATE OR REPLACE VIEW analytics_live_path_stats_hourly AS +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; + +CREATE MATERIALIZED VIEW mv_path_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_live_path_stats_hourly + UNION ALL + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_path_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, category, hash_bytes, entries, SUM(receptions)::bigint AS receptions +FROM combined GROUP BY iata,hour,category,hash_bytes,entries; + +CREATE UNIQUE INDEX idx_analytics_path_stats_hourly_view ON mv_path_stats_hourly (iata,hour,category,hash_bytes,entries); +CREATE INDEX idx_mv_path_stats_hourly_hour ON mv_path_stats_hourly (hour); + + +-- VOLATILE gives the archive statement a new READ COMMITTED snapshot after +-- acquiring parent locks. FK inserts cannot race between that snapshot and the +-- cascade. SKIP LOCKED leaves packets being ingested for a later cleanup tick. +CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) +RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ +DECLARE hashes bytea[]; deleted bigint; +BEGIN + -- Serialize cleanup runners before overlapping aggregate upserts. + PERFORM pg_advisory_xact_lock(hashtext('beacon.archive_delete_packets')); + IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; + SELECT array_agg(p.packet_hash) INTO hashes FROM ( + SELECT ep.packet_hash FROM packets ep + WHERE ep.last_heard_at < cutoff + ORDER BY ep.last_heard_at, ep.packet_hash + LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED + ) p; + + WITH expired_observations AS MATERIALIZED ( + SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) + ), +archived_hourly_iata_stats AS ( + INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) + SELECT iata, hour, observation_count, unique_packets FROM ( +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,hour) DO UPDATE SET + observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, + unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets +), +archived_payload_breakdown_by_iata AS ( + INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) + SELECT iata, payload_type, bucket, count FROM ( +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET + count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count +), +archived_top_observers_by_iata AS ( + INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM expired_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET + observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, + display_name = EXCLUDED.display_name, + observer_type = EXCLUDED.observer_type +), +archived_top_talkers_by_iata AS ( + INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) + SELECT iata, sender_name, bucket, message_count, last_sent FROM ( +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN expired_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') + ) batch + ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET + message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, + last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) +), +archived_top_advertisers_by_iata AS ( + INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN expired_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,node_id,bucket) DO UPDATE SET + advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, + flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, + direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, + last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), + name = EXCLUDED.name, + node_type = EXCLUDED.node_type +), +archived_observer_activity_hourly AS ( + INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET + observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, + airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, + airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, + snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, + snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, + snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), + rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, + rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n +), +archived_signal_stats_hourly AS ( + INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) + ) batch + ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET + receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, + snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, + snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, + rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, + rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum +), +archived_path_stats_hourly AS ( + INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) + SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries + ) batch + ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET + receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions +) + DELETE FROM packets WHERE packet_hash = ANY(hashes); + GET DIAGNOSTICS deleted = ROW_COUNT; + + -- Once per completed cleanup, including when there were no raw packets to delete. + IF deleted < batch_size THEN + DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + END IF; + RETURN deleted; +END $$; diff --git a/db/migrations/041_route_evidence_index.sql b/db/migrations/041_route_evidence_index.sql new file mode 100644 index 0000000..71088cc --- /dev/null +++ b/db/migrations/041_route_evidence_index.sql @@ -0,0 +1,9 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- One statement outside a transaction. RunMigrations recovers interrupted builds; +-- avoid IF NOT EXISTS, which would silently accept an invalid index. +-- A compact digest narrows candidates; queries still compare the complete bytes. +CREATE INDEX CONCURRENTLY idx_observations_route_evidence +ON packet_observations (iata, hash_size, (decode(md5(path_bytes), 'hex')), heard_at DESC, id DESC) +WHERE path_bytes IS NOT NULL AND payload_type IS NOT NULL AND payload_type <> 9 AND hop_count >= 2; diff --git a/db/migrations/042_meshmapper_scopes.sql b/db/migrations/042_meshmapper_scopes.sql new file mode 100644 index 0000000..43b48b6 --- /dev/null +++ b/db/migrations/042_meshmapper_scopes.sql @@ -0,0 +1,17 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Preserve historical identities while distinguishing import-only matching keys. +ALTER TABLE transport_scopes ADD COLUMN IF NOT EXISTS imported_only BOOLEAN NOT NULL DEFAULT FALSE; + +CREATE TABLE IF NOT EXISTS meshmapper_scope_catalogues ( + iata TEXT NOT NULL, + url TEXT NOT NULL, + payload JSONB, + etag TEXT, + checked_at TIMESTAMPTZ, + attempted_at TIMESTAMPTZ NOT NULL, + next_attempt TIMESTAMPTZ NOT NULL, + last_error TEXT NOT NULL DEFAULT '', + PRIMARY KEY (iata, url) +); diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go new file mode 100644 index 0000000..91dbeca --- /dev/null +++ b/db/observer_metrics_integration_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "encoding/json" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "strings" + "testing" + "time" +) + +func TestObserverMetricsPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + // Use the real observer DDL without depending on a preinstalled public schema. + initial, err := migrationFiles.ReadFile("migrations/001_initial_schema.sql") + if err != nil { + t.Fatal(err) + } + ddl := string(initial) + start := strings.Index(ddl, "CREATE TABLE observers (") + end := strings.Index(ddl[start:], "\n);") + start + 3 + if _, err := tx.Exec(ctx, "DROP TABLE observers; "+ddl[start:end]+" ALTER TABLE observers ADD COLUMN region_scope TEXT;"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, ` + CREATE TABLE observer_brokers (observer_id uuid,broker_name text,last_seen timestamptz,last_packet_at timestamptz,PRIMARY KEY(observer_id,broker_name)); + INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); + INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash IN (int4send(1),int4send(2)) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; + `); err != nil { + t.Fatal(err) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + store := &Store{q: sqlc.New(tx)} + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { + t.Fatal(err) + } + // The initial archive migration retains unknown payloads without a second rebuild. + if err := store.RefreshObserverActivity(ctx); err != nil { + t.Fatal(err) + } + until := time.Now().UTC().Truncate(time.Hour) + activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until) + if err != nil { + t.Fatal(err) + } + if _, err := json.Marshal(activity); err != nil { + t.Fatalf("non-finite sample leaked into JSON: %v", err) + } + if activity.Summary.RecordedPackets != 3 || len(activity.PayloadTypes) != 2 || activity.Summary.LatestRecordedAt != nil || activity.Summary.LastCompleteHour != 0 { + t.Fatalf("archive summary: %+v types=%+v", activity.Summary, activity.PayloadTypes) + } + if activity.WindowEnd != until.UnixMilli() || activity.WindowStart != until.Add(-7*24*time.Hour).UnixMilli() || activity.Source != "hourly" { + t.Fatalf("window: %+v", activity) + } + if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,last_heard_at) VALUES ('\xaa',NOW()); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,'NaN'::real,-100); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW()) ON CONFLICT DO NOTHING;`); err != nil { + t.Fatal(err) + } + raw, err := store.GetObserverActivity(ctx, id, time.Hour, 15*time.Minute, until) + if err != nil { + t.Fatal(err) + } + if _, err := json.Marshal(raw); err != nil { + t.Fatalf("raw non-finite sample leaked into JSON: %v", err) + } + if raw.Summary.RecordedPackets != 1 || raw.Summary.LastCompleteHour != 1 || raw.Summary.LatestRecordedAt == nil || raw.PayloadTypes[0].PayloadType != -1 { + t.Fatalf("raw/duplicate: %+v", raw) + } + if err := store.UpsertObserverBroker(ctx, id, "one", false); err != nil { + t.Fatal(err) + } + var packet pgtype.Timestamptz + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if packet.Valid { + t.Fatal("status created a packet timestamp") + } + if err := store.UpsertObserverBroker(ctx, id, "one", true); err != nil { + t.Fatal(err) + } + if err := store.TouchObserverBrokers(ctx, []uuid.UUID{id}, []string{"one"}, []time.Time{until.Add(time.Hour)}, []time.Time{{}}); err != nil { + t.Fatal(err) + } + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if !packet.Valid { + t.Fatal("status erased packet timestamp") + } +} diff --git a/db/observer_retention_integration_test.go b/db/observer_retention_integration_test.go index 8672ea4..8e64ea9 100644 --- a/db/observer_retention_integration_test.go +++ b/db/observer_retention_integration_test.go @@ -100,7 +100,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil { t.Fatal(err) } - if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture"); err != nil { + if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture", true); err != nil { t.Fatal(err) } // A future cutoff simulates this cached observer having aged past retention. @@ -112,7 +112,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil || newID == oldID { t.Fatalf("returning observer reused a deleted ID: %v", err) } - if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture"); err != nil { + if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture", true); err != nil { t.Fatal(err) } var brokers int diff --git a/db/observer_window_test.go b/db/observer_window_test.go new file mode 100644 index 0000000..205d1ef --- /dev/null +++ b/db/observer_window_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/google/uuid" + "go.uber.org/mock/gomock" +) + +func TestObserverActivityLiveAndFixedWindow(t *testing.T) { + for _, interval := range []time.Duration{15 * time.Minute, time.Hour, 6 * time.Hour} { + for _, fixed := range []bool{false, true} { + t.Run(interval.String()+"/"+map[bool]string{false: "live", true: "fixed"}[fixed], func(t *testing.T) { + mock := mockdb.NewMockQuerier(gomock.NewController(t)) + id := uuid.New() + mock.EXPECT().GetObserverByID(gomock.Any(), id).Return(sqlc.Observer{}, nil) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil) + if interval < time.Hour { + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()).Return(nil, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) + } else { + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()).Return(nil, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) + } + before := time.Now().UTC() + var until time.Time + if fixed { + until = before.Add(-24*time.Hour + 123*time.Millisecond) + } + got, err := (&Store{q: mock}).GetObserverActivity(context.Background(), id, 48*time.Hour, interval, until) + if err != nil { + t.Fatal(err) + } + if fixed { + if got.WindowEnd != until.Truncate(interval).UnixMilli() { + t.Fatalf("fixed end was not aligned: %d", got.WindowEnd) + } + } else if got.WindowEnd < before.UnixMilli() || got.WindowEnd > time.Now().UnixMilli() { + t.Fatalf("live end dropped the partial bucket: %d", got.WindowEnd) + } + // Freshness is always live, even when the activity window is historical. + if got.Summary.LastCompleteHourEnd != time.UnixMilli(got.GeneratedAt).UTC().Truncate(time.Hour).UnixMilli() { + t.Fatal("freshness was anchored to the historical activity window") + } + }) + } + } +} diff --git a/db/observers.go b/db/observers.go index a3448cf..fc10efa 100644 --- a/db/observers.go +++ b/db/observers.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "fmt" "log/slog" + "math" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -230,12 +231,38 @@ func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uui // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. // Buckets of an hour or coarser come from the hourly rollup; anything finer reads observations directly. // Range and Interval are left empty for the handler to fill. -func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { +func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { obs, err := s.q.GetObserverByID(ctx, observerID) if err != nil { return nil, err } - activity := &api.ObserverActivity{} + now := time.Now().UTC() + if until.IsZero() { + until = now + } else { + until = until.UTC().Truncate(interval) + } + since := until.Add(-window) + if !since.Equal(since.Truncate(interval)) { + since = since.Truncate(interval).Add(interval) + } + activity := &api.ObserverActivity{WindowStart: since.UnixMilli(), WindowEnd: until.UnixMilli(), GeneratedAt: now.UnixMilli(), Source: "raw", Summary: &api.ObserverActivitySummary{}} + if interval >= time.Hour { + activity.Source = "hourly" + } + live, err := s.q.GetObserverActivityLiveSummary(ctx, sqlc.GetObserverActivityLiveSummaryParams{ + ObserverID: observerID, GeneratedAt: pgtype.Timestamptz{Time: now, Valid: true}, + HourStart: pgtype.Timestamptz{Time: now.Truncate(time.Hour).Add(-time.Hour), Valid: true}, HourEnd: pgtype.Timestamptz{Time: now.Truncate(time.Hour), Valid: true}}) + if err != nil { + return nil, err + } + activity.Summary.LastCompleteHour = live.LastCompleteHour + if live.LatestRecordedAt.Valid { + v := live.LatestRecordedAt.Time.UnixMilli() + activity.Summary.LatestRecordedAt = &v + } + activity.Summary.LastCompleteHourStart = now.Truncate(time.Hour).Add(-time.Hour).UnixMilli() + activity.Summary.LastCompleteHourEnd = now.Truncate(time.Hour).UnixMilli() // radio is non-nil only when airtime is actually costable, so radio != null implies costed buckets if obs.RadioSf != nil && obs.RadioBwKhz != nil && obs.RadioCr != nil && *obs.RadioSf >= 7 && *obs.RadioSf <= 12 && *obs.RadioBwKhz > 0 && *obs.RadioCr > 0 { @@ -247,12 +274,6 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w PreambleSymbols: lora.PreambleSymbols(int(*obs.RadioSf)), } } - // round the window start up to a bucket boundary so the first bucket is never a partial one - start := time.Now().Add(-window).UTC() - since := start.Truncate(interval) - if since.Before(start) { - since = since.Add(interval) - } sinceTS := pgtype.Timestamptz{Time: since, Valid: true} binWidth := pgtype.Interval{Microseconds: interval.Microseconds(), Valid: true} @@ -261,6 +282,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w ObserverID: observerID, Column2: sinceTS, Column3: binWidth, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err @@ -268,6 +290,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) for _, r := range rows { p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations if r.AirtimeN > 0 { airtime := r.AirtimeMs p.AirtimeMs = &airtime @@ -275,7 +298,12 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w if r.SnrN > 0 { avg := r.SnrSum / float32(r.SnrN) min := r.SnrMin - p.SNRAvg, p.SNRMin = &avg, &min + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } } if r.RssiN > 0 { avg := float32(r.RssiSum) / float32(r.RssiN) @@ -286,18 +314,17 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityHourlyPayloadTypes(ctx, sqlc.GetObserverActivityHourlyPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - if v.PayloadType == nil { - continue - } + payloadType := v.PayloadType activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), Count: v.Count, }) } @@ -308,6 +335,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w ObserverID: observerID, Column2: sinceTS, Column3: binWidth, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err @@ -315,13 +343,19 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) for _, r := range rows { p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations if r.AirtimeN > 0 { airtime := r.AirtimeMs p.AirtimeMs = &airtime } if r.SnrN > 0 { avg, min := r.SnrAvg, r.SnrMin - p.SNRAvg, p.SNRMin = &avg, &min + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } } if r.RssiN > 0 { avg := r.RssiAvg @@ -332,18 +366,20 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityRawPayloadTypes(ctx, sqlc.GetObserverActivityRawPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - if v.PayloadType == nil { - continue + payloadType := int16(-1) + if v.PayloadType != nil { + payloadType = *v.PayloadType } activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), Count: v.Count, }) } @@ -424,10 +460,11 @@ func (s *Store) GetObserverRadio(ctx context.Context, observerID uuid.UUID) (ing return settings, nil } -func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { params := sqlc.UpsertObserverBrokerParams{ ObserverID: observerID, BrokerName: brokerName, + IsPacket: isPacket, } return s.q.UpsertObserverBroker(ctx, params) } diff --git a/db/observers_test.go b/db/observers_test.go index 76ccf54..f6c67d3 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -569,6 +569,7 @@ func f32(v float32) *float32 { return &v } func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -587,12 +588,12 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { }}, nil) mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{ - {PayloadType: i16(4), Count: 9}, - {PayloadType: nil, Count: 3}, + {PayloadType: 4, Count: 9}, + {PayloadType: -1, Count: 3}, }, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -615,8 +616,8 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { if p.RSSIAvg != nil { t.Errorf("expected nil RSSIAvg, got %v", *p.RSSIAvg) } - if len(got.PayloadTypes) != 1 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 { - t.Fatalf("expected one payload type 4 with count 9, got %+v", got.PayloadTypes) + if len(got.PayloadTypes) != 2 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 || got.PayloadTypes[1].PayloadType != -1 || got.PayloadTypes[1].Count != 3 { + t.Fatalf("expected known and unknown payloads to reconcile, got %+v", got.PayloadTypes) } if got.PayloadTypes[0].PayloadTypeName != api.PayloadTypeName(4) { t.Errorf("expected payload type name %q, got %q", api.PayloadTypeName(4), got.PayloadTypes[0].PayloadTypeName) @@ -638,6 +639,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -658,7 +660,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -689,6 +691,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { func TestGetObserverActivity_RawPath(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -709,7 +712,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{{PayloadType: i16(1), Count: 5}}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -737,6 +740,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) { func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -757,7 +761,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -779,6 +783,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { func TestGetObserverActivity_RawPathEmpty(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -789,7 +794,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -804,6 +809,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) { func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -814,7 +820,7 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -826,13 +832,14 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { func TestGetObserverActivity_UnknownObserver(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000002") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). Return(sqlc.Observer{}, pgx.ErrNoRows) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour, time.Time{}) if !errors.Is(err, pgx.ErrNoRows) { t.Fatalf("expected pgx.ErrNoRows, got %v", err) } @@ -858,6 +865,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { t.Run(tc.name, func(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -868,7 +876,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -882,6 +890,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") window, interval := 24*time.Hour, 6*time.Hour @@ -897,12 +906,12 @@ func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) - before := time.Now().Add(-window) + before := time.Now().UTC().Truncate(interval).Add(-window) store := &Store{q: mock} - if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval); err != nil { + if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval, time.Time{}); err != nil { t.Fatalf("unexpected error: %v", err) } - after := time.Now().Add(-window) + after := time.Now().UTC().Truncate(interval).Add(-window) if gotSince.UnixNano()%int64(interval) != 0 { t.Errorf("since %s is not aligned to %s", gotSince, interval) @@ -910,7 +919,7 @@ func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { if gotSince.Before(before) { t.Errorf("since %s is before the window start %s", gotSince, before) } - if !gotSince.Before(after.Add(interval)) { + if gotSince.After(after.Add(interval)) { t.Errorf("since %s is more than one interval past the window start %s", gotSince, after) } if !gotInterval.Valid || gotInterval.Microseconds != interval.Microseconds() { diff --git a/db/presence.go b/db/presence.go index ea2d6fe..4122266 100644 --- a/db/presence.go +++ b/db/presence.go @@ -20,11 +20,12 @@ func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time }) } -func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error { +func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error { return s.q.TouchObserverBrokers(ctx, sqlc.TouchObserverBrokersParams{ Column1: ids, Column2: brokers, Column3: toTimestamptzs(seen), + Column4: nullableTimes(packets), }) } @@ -42,3 +43,11 @@ func toTimestamptzs(ts []time.Time) []pgtype.Timestamptz { } return out } + +func nullableTimes(ts []time.Time) []pgtype.Timestamptz { + out := make([]pgtype.Timestamptz, len(ts)) + for i, t := range ts { + out[i] = pgtype.Timestamptz{Time: t, Valid: !t.IsZero()} + } + return out +} diff --git a/db/queries/queries.sql b/db/queries/queries.sql index ccc8492..328960e 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -49,10 +49,11 @@ VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO UPDATE SET display_name = EXCLUDED.display_name, transport_key = EXCLUDED.transport_key, - key_fingerprint = EXCLUDED.key_fingerprint; + key_fingerprint = EXCLUDED.key_fingerprint, + imported_only = FALSE; -- name: GetTransportScopes :many -SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name; +SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name; -- name: GetTransportScopeByName :one SELECT id FROM transport_scopes WHERE name = $1; @@ -275,14 +276,14 @@ SELECT COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz GROUP BY bucket ORDER BY bucket; -- name: GetObserverActivityRawPayloadTypes :many SELECT payload_type, COUNT(*)::bigint AS count FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -299,14 +300,14 @@ SELECT COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz GROUP BY 1 ORDER BY 1; -- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -363,19 +364,20 @@ RETURNING o.id; -- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN @is_packet::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW(); + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at); -- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name; @@ -661,18 +663,9 @@ ORDER BY po.id ASC LIMIT $6; --- name: DeleteOldPackets :execrows --- One batch of expired packets; observations and channel messages cascade. -WITH expired AS ( - SELECT ep.packet_hash - FROM packets ep - WHERE ep.last_heard_at < @cutoff - ORDER BY ep.last_heard_at - LIMIT @batch_size - FOR UPDATE OF ep SKIP LOCKED -) -DELETE FROM packets p USING expired e -WHERE p.packet_hash = e.packet_hash; +-- name: DeleteOldPackets :one +-- Locks, archives and cascades one bounded packet cohort in a single transaction. +SELECT archive_delete_packets(@cutoff::timestamptz, @batch_size::integer)::bigint; -- name: DeleteOldNodes :exec -- Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- @@ -951,17 +944,25 @@ SELECT * FROM channels WHERE id = $1; -- ============================================================ -- name: InsertChannelMessage :one -INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) -VALUES ($1, $2, $3, $4, $5) -ON CONFLICT (packet_hash) DO NOTHING -RETURNING id; +-- Read the immutable first-packet scope in the same statement as insertion. +-- A later reception's transport code must not give live and historical messages different tags. +WITH inserted AS ( + INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (packet_hash) DO NOTHING + RETURNING id, packet_hash +) +SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present +FROM inserted +JOIN packets p ON p.packet_hash = inserted.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id; -- name: ListChannelMessages :many -- Returns messages for a channel identified by integer ID. -- Pass a zero/null timestamp for since to return all messages up to limit. -- Pass empty string for iata to skip IATA filtering. -- Pass cursor=0 to start from the beginning. -SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -980,7 +981,7 @@ LIMIT $6; -- Returns all messages across all channels with optional time, IATA, scope and cursor filters. -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. -SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -999,7 +1000,7 @@ LIMIT $5; -- May return messages from multiple channels if the hash collides across different keys. -- Pass empty string for iata or scope to skip those filters. -- Pass cursor=0 to start from the beginning. -SELECT DISTINCT ON (cm.id) cm.*, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.*, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -1017,7 +1018,7 @@ LIMIT $6; -- name: ListMessagesAfterID :many -- Returns messages after the given message ID, ordered oldest first. -- Used for WS reconnect backfill. -SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.*, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -1580,3 +1581,43 @@ WITH target AS MATERIALIZED ( ) SELECT EXISTS(SELECT 1 FROM target) AS found, EXISTS(SELECT 1 FROM changed) AS deactivated; + +-- name: GetObserverActivityLiveSummary :one +-- Two indexed ranges, bounded to one observer; no legacy presence counters. +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = @observer_id::uuid + AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly; + + +-- name: GetScopeCatalogue :one +SELECT * FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2; + +-- name: SaveScopeCatalogue :exec +-- One statement commits the validated snapshot and its lookup identities together. +-- Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +-- after an error or 304. Imported names never replace existing manual metadata. +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest(@names::text[]) AS name, unnest(@keys::bytea[]) AS key, + unnest(@fingerprints::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES (@iata, @url, sqlc.narg(payload)::jsonb, sqlc.narg(etag)::text, + sqlc.narg(checked_at)::timestamptz, @attempted_at, @next_attempt, @last_error) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error; diff --git a/db/queries/route_evidence.sql b/db/queries/route_evidence.sql new file mode 100644 index 0000000..dcc056f --- /dev/null +++ b/db/queries/route_evidence.sql @@ -0,0 +1,23 @@ +-- name: GetRouteEvidenceRoute :one +SELECT * FROM known_routes +WHERE iata = @iata::bpchar AND path_key = @path_key::bytea; + +-- name: ListRouteEvidence :many +-- Leading index equalities and the time/ID boundary bound both custom and generic plans. +-- Full-byte equality is required even when the compact digest matches. TRACE path bytes +-- carry readings; unclassified legacy observations cannot be safely called ordinary paths. +SELECT po.id, po.packet_hash, po.observer_id, o.display_name AS observer_name, + po.heard_at, po.payload_type, po.rssi, po.snr +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.iata = @iata::bpchar + AND po.hash_size = @hash_size::smallint + AND decode(md5(po.path_bytes), 'hex') = @path_digest::bytea + AND po.path_bytes = @path_bytes::bytea + AND po.hop_count >= 2 + AND po.hop_count = @hop_count::smallint + AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 + AND po.heard_at >= @since::timestamptz AND po.heard_at < @until::timestamptz + AND (po.heard_at, po.id) < (@before_at::timestamptz, @before_id::bigint) +ORDER BY po.heard_at DESC, po.id DESC +LIMIT @page_limit::integer; diff --git a/db/retention_integration_test.go b/db/retention_integration_test.go index 5e40dbd..d630717 100644 --- a/db/retention_integration_test.go +++ b/db/retention_integration_test.go @@ -46,10 +46,9 @@ func countRows(t *testing.T, ctx context.Context, tx pgx.Tx, sql string) int { // More than one batch of old packets; observations go with them. func TestDeleteOldPacketsBatchesPostgres(t *testing.T) { ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") _, err := tx.Exec(ctx, ` -CREATE TEMP TABLE packets (LIKE public.packets INCLUDING ALL) ON COMMIT DROP; -CREATE TEMP TABLE packet_observations (LIKE public.packet_observations INCLUDING ALL) ON COMMIT DROP; -ALTER TABLE packet_observations ADD FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) SELECT int4send(i), 4, 0, 1, '\x00', '\x00', CASE WHEN i <= 2500 THEN '2026-01-01'::timestamptz ELSE '2026-02-01'::timestamptz END, diff --git a/db/route_evidence.go b/db/route_evidence.go new file mode 100644 index 0000000..fc7209e --- /dev/null +++ b/db/route_evidence.go @@ -0,0 +1,93 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "math" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5/pgtype" +) + +func savedRoutePath(prefixes [][]byte, count int32) (int16, []byte, bool) { + if count < 2 || count > 63 || len(prefixes) != int(count) { + return 0, nil, false + } + width := len(prefixes[0]) + if width < 1 || width > 3 { + return 0, nil, false + } + for _, p := range prefixes { + if len(p) != width { + return 0, nil, false + } + } + return int16(width), bytes.Join(prefixes, nil), true +} + +func (s *Store) GetRouteEvidence(ctx context.Context, iata, key string, query api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + if !api.ValidRouteEvidenceKey(iata, key) || !api.ValidRouteEvidenceWindow(query.Since, query.Until) || query.Limit < 1 || query.Limit > 200 { + return nil, api.ErrRouteEvidenceInput + } + if c := query.Cursor; c != nil { + if c.IATA != iata || c.PathKey != key || !c.Since.Equal(query.Since) || !c.Until.Equal(query.Until) || c.ID <= 0 || c.HeardAt.Before(query.Since) || !c.HeardAt.Before(query.Until) { + return nil, api.ErrRouteEvidenceInput + } + } + keyBytes, _ := hex.DecodeString(key) + row, err := s.q.GetRouteEvidenceRoute(ctx, sqlc.GetRouteEvidenceRouteParams{Iata: iata, PathKey: keyBytes}) + if err != nil { + return nil, err + } + nodes, err := s.GetNodesByIDs(ctx, row.NodeIds) + if err != nil { + return nil, err + } + route := toKnownRoutes([]knownRouteRow{{ID: row.ID, NodeIds: row.NodeIds, HashPrefix: row.HashPrefix, Iata: row.Iata, HopCount: row.HopCount, FirstSeen: row.FirstSeen, LastSeen: row.LastSeen, ObservationCount: row.ObservationCount}}, nodes)[0] + route.PathKey = hex.EncodeToString(row.PathKey) + out := &api.RouteEvidence{Page: api.Page[api.RouteObservation]{Items: []api.RouteObservation{}}, Route: route, WindowStart: query.Since.UnixMilli(), WindowEnd: query.Until.UnixMilli(), GeneratedAt: time.Now().UnixMilli(), MatchType: "saved_path_prefixes"} + width, path, valid := savedRoutePath(row.HashPrefix, row.HopCount) + if !valid { + return out, nil + } + out.MatchAvailable = true + out.HashSize = width + out.PathBytes = hex.EncodeToString(path) + beforeAt, beforeID := query.Until, int64(math.MaxInt64) + if query.Cursor != nil { + beforeAt, beforeID = query.Cursor.HeardAt, query.Cursor.ID + } + ts := func(t time.Time) pgtype.Timestamptz { return pgtype.Timestamptz{Time: t, Valid: true} } + digest := md5.Sum(path) + rows, err := s.q.ListRouteEvidence(ctx, sqlc.ListRouteEvidenceParams{Iata: iata, HashSize: width, HopCount: int16(row.HopCount), PathDigest: digest[:], PathBytes: path, Since: ts(query.Since), Until: ts(query.Until), BeforeAt: ts(beforeAt), BeforeID: beforeID, PageLimit: query.Limit + 1}) + if err != nil { + return nil, err + } + out.HasMore = len(rows) > int(query.Limit) + if out.HasMore { + rows = rows[:query.Limit] + } + for _, r := range rows { + payload := int16(-1) + if r.PayloadType != nil { + payload = *r.PayloadType + } + snr := r.Snr + if snr != nil && (math.IsNaN(float64(*snr)) || math.IsInf(float64(*snr), 0)) { + snr = nil + } + out.Items = append(out.Items, api.RouteObservation{ID: r.ID, PacketHash: hex.EncodeToString(r.PacketHash), ObserverID: r.ObserverID, ObserverName: r.ObserverName, HeardAt: r.HeardAt.Time.UnixMilli(), PayloadType: payload, PayloadTypeName: api.PayloadTypeName(payload), RSSI: r.Rssi, SNR: snr}) + } + if out.HasMore { + last := rows[len(rows)-1] + cursor := (api.RouteEvidenceCursor{IATA: iata, PathKey: key, Since: query.Since, Until: query.Until, HeardAt: last.HeardAt.Time, ID: last.ID}).String() + out.NextPageCursor = &cursor + } + return out, nil +} diff --git a/db/route_evidence_integration_test.go b/db/route_evidence_integration_test.go new file mode 100644 index 0000000..f4a3e17 --- /dev/null +++ b/db/route_evidence_integration_test.go @@ -0,0 +1,214 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +type evidenceCapture struct { + pgx.Tx + statement string + args []any +} + +func (c *evidenceCapture) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) { + if strings.Contains(sql, "-- name: ListRouteEvidence") { + c.statement = sql + c.args = args + } + return c.Tx.Query(ctx, sql, args...) +} + +func TestRouteEvidencePostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + ddl, err := migrationFiles.ReadFile("migrations/024_known_routes_pathkey.sql") + if err != nil { + t.Fatal(err) + } + text := string(ddl) + start := strings.Index(text, "CREATE TABLE known_routes_new (") + end := start + strings.Index(text[start:], "\n);") + 3 + table := strings.ReplaceAll(text[start:end], "known_routes_new", "known_routes") + _, err = tx.Exec(ctx, `CREATE TABLE iata_codes (iata char(3) PRIMARY KEY); INSERT INTO iata_codes VALUES ('YOW'),('YVR'); + ALTER TABLE nodes ADD COLUMN latitude double precision, ADD COLUMN longitude double precision;`+table+` + INSERT INTO observers(id,display_name) VALUES ('00000000-0000-0000-0000-000000000001','One'),('00000000-0000-0000-0000-000000000002','Two'); + INSERT INTO nodes(id,public_key,name) VALUES ('00000000-0000-0000-0000-000000000001','\xaa','A'),('00000000-0000-0000-0000-000000000002','\xbb','B'); + INSERT INTO packets(packet_hash) SELECT int4send(i) FROM generate_series(1,10) i;`) + if err != nil { + t.Fatal(err) + } + index, err := migrationFiles.ReadFile("migrations/041_route_evidence_index.sql") + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, strings.Replace(string(index), "CREATE INDEX CONCURRENTLY", "CREATE INDEX", 1)); err != nil { + t.Fatal(err) + } + capture := &evidenceCapture{Tx: tx} + store := &Store{q: sqlc.New(capture)} + ids := []uuid.UUID{uuid.MustParse("00000000-0000-0000-0000-000000000001"), uuid.MustParse("00000000-0000-0000-0000-000000000002")} + if err := store.UpsertKnownRoute(ctx, ids, [][]byte{{0xaa}, {0xbb}}, "YOW", 2); err != nil { + t.Fatal(err) + } + key := hex.EncodeToString(routePathKey(ids)) + since := time.Now().UTC().Truncate(time.Millisecond).Add(-time.Hour) + until := since.Add(time.Hour) + at := since.Add(time.Minute + 123456*time.Microsecond) + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type,snr) + SELECT i,int4send(i), '00000000-0000-0000-0000-000000000001',CASE WHEN i=5 THEN 'YVR' ELSE 'YOW' END, + CASE WHEN i=9 THEN $2::timestamptz WHEN i=1 THEN $1::timestamptz-interval '2 microseconds' ELSE $1::timestamptz END, + CASE WHEN i=4 THEN 2 ELSE 1 END,CASE WHEN i=4 THEN 1 WHEN i=10 THEN 3 ELSE 2 END, + CASE WHEN i=6 THEN '\xabbb'::bytea WHEN i=10 THEN '\xaabbcc'::bytea ELSE '\xaabb'::bytea END, + CASE WHEN i=7 THEN 9 WHEN i=8 THEN NULL ELSE 4 END,CASE WHEN i=1 THEN 'NaN'::real ELSE 0 END FROM generate_series(1,10) i; + `, at, until) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, `INSERT INTO packet_observations(id,packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) VALUES (11,int4send(1),'00000000-0000-0000-0000-000000000002','YOW',$1,1,2,'\xaabb',4)`, at.Add(-time.Microsecond)); err != nil { + t.Fatal(err) + } + query := api.RouteEvidenceQuery{Since: since, Until: until, Limit: 2} + first, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil { + t.Fatal(err) + } + if !first.HasMore || first.NextPageCursor == nil || len(first.Items) != 2 || first.Items[0].ID != 3 || first.Items[1].ID != 2 || first.Route.PathKey != key || first.HashSize != 1 || first.PathBytes != "aabb" { + t.Fatalf("first page: %+v", first) + } + query.Cursor, err = api.ParseRouteEvidenceCursor(*first.NextPageCursor) + if err != nil { + t.Fatal(err) + } + second, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil { + t.Fatal(err) + } + if second.HasMore || len(second.Items) != 2 || second.Items[0].ID != 11 || second.Items[1].ID != 1 || second.Items[1].SNR != nil { + t.Fatalf("precise second page: %+v", second) + } + if _, err = json.Marshal(second); err != nil { + t.Fatalf("non-finite JSON: %v", err) + } + query.Cursor.IATA = "YVR" + if _, err = store.GetRouteEvidence(ctx, "YOW", key, query); !errors.Is(err, api.ErrRouteEvidenceInput) { + t.Fatal("cross-route cursor accepted") + } + query.Cursor = nil + if _, err = store.GetRouteEvidence(ctx, "YVR", key, query); !errors.Is(err, pgx.ErrNoRows) { + t.Fatal("unknown route did not return not-found") + } + if _, err = tx.Exec(ctx, "DELETE FROM packets"); err != nil { + t.Fatal(err) + } + empty, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || !empty.MatchAvailable || len(empty.Items) != 0 || empty.HasMore { + t.Fatalf("expired evidence: %+v %v", empty, err) + } + + // Many unrelated paths must not turn this into a raw-table or route-ID scan. + started := time.Now() + _, err = tx.Exec(ctx, `INSERT INTO packets(packet_hash) SELECT int4send(i) FROM generate_series(100,200099) i;`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,hash_size,hop_count,path_bytes,payload_type) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',$1,1,2, + CASE WHEN get_byte(packet_hash,3)=0 THEN '\xaabb'::bytea ELSE packet_hash END,4 FROM packets;`, at) + if err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(ctx, "ANALYZE packet_observations; ANALYZE observers;"); err != nil { + t.Fatal(err) + } + for _, mode := range []string{"force_custom_plan", "force_generic_plan"} { + if _, err = tx.Exec(ctx, "SET LOCAL plan_cache_mode = "+mode); err != nil { + t.Fatal(err) + } + result, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || len(result.Items) != 2 { + t.Fatalf("indexed read: %v", err) + } + var plan string + if err = tx.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+capture.statement, capture.args...).Scan(&plan); err != nil { + t.Fatal(err) + } + if !strings.Contains(plan, "idx_observations_route_evidence") { + t.Fatalf("missing evidence index: %s", plan) + } + t.Logf("%s plan: %s", mode, plan) + } + t.Logf("200000-row fixture insert/analyze/read: %s", time.Since(started)) + if _, err = tx.Exec(ctx, `UPDATE known_routes SET hash_prefix=ARRAY['\xaa'::bytea]`); err != nil { + t.Fatal(err) + } + unavailable, err := store.GetRouteEvidence(ctx, "YOW", key, query) + if err != nil || unavailable.MatchAvailable || len(unavailable.Items) != 0 { + t.Fatalf("malformed stored path: %+v %v", unavailable, err) + } +} + +func TestRouteEvidenceIndexPostgres(t *testing.T) { + dsn := os.Getenv("BEACON_TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("set BEACON_TEST_POSTGRES_DSN") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := pgx.Connect(ctx, dsn) + if err != nil { + t.Fatal(err) + } + defer conn.Close(context.Background()) + schema := fmt.Sprintf("route_evidence_%d", time.Now().UnixNano()) + ident := pgx.Identifier{schema}.Sanitize() + if _, err = conn.Exec(ctx, "CREATE SCHEMA "+ident+"; SET search_path TO "+ident); err != nil { + t.Fatal(err) + } + defer func() { _, _ = conn.Exec(context.Background(), "DROP SCHEMA "+ident+" CASCADE") }() + if _, err = conn.Exec(ctx, `CREATE TABLE packet_observations (id bigint,iata char(3),hash_size smallint,path_bytes bytea,heard_at timestamptz,payload_type smallint,hop_count smallint)`); err != nil { + t.Fatal(err) + } + migration, err := migrationFiles.ReadFile("migrations/041_route_evidence_index.sql") + if err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if err = applyMigration(ctx, conn, string(migration)); err != nil { + t.Fatalf("migration/retry: %v", err) + } + } + if _, err = conn.Exec(ctx, `UPDATE pg_index SET indisvalid=false WHERE indexrelid='idx_observations_route_evidence'::regclass`); err != nil { + t.Fatal(err) + } + if err = applyMigration(ctx, conn, string(migration)); err != nil { + t.Fatal(err) + } + var valid bool + if err = conn.QueryRow(ctx, `SELECT indisvalid FROM pg_index WHERE indexrelid='idx_observations_route_evidence'::regclass`).Scan(&valid); err != nil || !valid { + t.Fatalf("invalid index after recovery: %v", err) + } + if _, err = conn.Exec(ctx, `INSERT INTO packet_observations(id,iata,hash_size,path_bytes,heard_at,payload_type,hop_count) +SELECT n,'YOW',1,'\xaabb',NOW(),4,n FROM generate_series(0,2) n; +ANALYZE packet_observations`); err != nil { + t.Fatal(err) + } + var indexed float32 + if err = conn.QueryRow(ctx, `SELECT reltuples FROM pg_class WHERE oid='idx_observations_route_evidence'::regclass`).Scan(&indexed); err != nil || indexed != 1 { + t.Fatalf("0/1-hop rows entered the route index: %v %v", indexed, err) + } +} diff --git a/db/route_evidence_test.go b/db/route_evidence_test.go new file mode 100644 index 0000000..868b565 --- /dev/null +++ b/db/route_evidence_test.go @@ -0,0 +1,23 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + "bytes" + "testing" +) + +func TestSavedRoutePath(t *testing.T) { + size, path, ok := savedRoutePath([][]byte{{0xaa}, {0xbb}}, 2) + if !ok || size != 1 || !bytes.Equal(path, []byte{0xaa, 0xbb}) { + t.Fatal("valid path lost") + } + for _, prefixes := range [][][]byte{nil, {{1}}, {{1}, {2, 3}}, {{}, {2}}, {{1, 2, 3, 4}, {5, 6, 7, 8}}} { + if _, _, ok := savedRoutePath(prefixes, int32(len(prefixes))); ok { + t.Fatalf("accepted malformed path %x", prefixes) + } + } + if _, _, ok := savedRoutePath([][]byte{{1}, {2}}, 3); ok { + t.Fatal("accepted inconsistent hop count") + } +} diff --git a/db/routes.go b/db/routes.go index 187e07e..090d2a0 100644 --- a/db/routes.go +++ b/db/routes.go @@ -118,6 +118,7 @@ func (s *Store) SearchKnownRoutes(ctx context.Context, iata, fromHash, toHash st hops = append(hops, hop) } items = append(items, api.KnownRoute{ + PathKey: hex.EncodeToString(routePathKey(r.NodeIds)), ID: r.ID, IATA: r.Iata, HopCount: int32(len(hops)), @@ -348,6 +349,7 @@ func toKnownRoutes(rows []knownRouteRow, nodes map[uuid.UUID]*api.ResolvedNode) hops = append(hops, hop) } items = append(items, api.KnownRoute{ + PathKey: hex.EncodeToString(routePathKey(r.NodeIds)), ID: r.ID, IATA: r.Iata, HopCount: r.HopCount, diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index aeae62b..8a9a7ef 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -source=db/sqlc/querier.go -destination=db/sqlc/mock/querier.go -package=mockdb +// mockgen -source db/sqlc/querier.go -destination db/sqlc/mock/querier.go -package mockdb // // Package mockdb is a generated GoMock package. @@ -384,6 +384,21 @@ func (mr *MockQuerierMockRecorder) GetObserverActivityHourlyPayloadTypes(ctx, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityHourlyPayloadTypes", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityHourlyPayloadTypes), ctx, arg) } +// GetObserverActivityLiveSummary mocks base method. +func (m *MockQuerier) GetObserverActivityLiveSummary(ctx context.Context, arg db.GetObserverActivityLiveSummaryParams) (db.GetObserverActivityLiveSummaryRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityLiveSummary", ctx, arg) + ret0, _ := ret[0].(db.GetObserverActivityLiveSummaryRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityLiveSummary indicates an expected call of GetObserverActivityLiveSummary. +func (mr *MockQuerierMockRecorder) GetObserverActivityLiveSummary(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityLiveSummary", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityLiveSummary), ctx, arg) +} + // GetObserverActivityRaw mocks base method. func (m *MockQuerier) GetObserverActivityRaw(ctx context.Context, arg db.GetObserverActivityRawParams) ([]db.GetObserverActivityRawRow, error) { m.ctrl.T.Helper() @@ -669,6 +684,21 @@ func (mr *MockQuerierMockRecorder) GetRegionIATAs(ctx, regionID any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegionIATAs", reflect.TypeOf((*MockQuerier)(nil).GetRegionIATAs), ctx, regionID) } +// GetRouteEvidenceRoute mocks base method. +func (m *MockQuerier) GetRouteEvidenceRoute(ctx context.Context, arg db.GetRouteEvidenceRouteParams) (db.KnownRoute, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRouteEvidenceRoute", ctx, arg) + ret0, _ := ret[0].(db.KnownRoute) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRouteEvidenceRoute indicates an expected call of GetRouteEvidenceRoute. +func (mr *MockQuerierMockRecorder) GetRouteEvidenceRoute(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRouteEvidenceRoute", reflect.TypeOf((*MockQuerier)(nil).GetRouteEvidenceRoute), ctx, arg) +} + // GetScopeByName mocks base method. func (m *MockQuerier) GetScopeByName(ctx context.Context, name string) (db.GetScopeByNameRow, error) { m.ctrl.T.Helper() @@ -684,6 +714,21 @@ func (mr *MockQuerierMockRecorder) GetScopeByName(ctx, name any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeByName", reflect.TypeOf((*MockQuerier)(nil).GetScopeByName), ctx, name) } +// GetScopeCatalogue mocks base method. +func (m *MockQuerier) GetScopeCatalogue(ctx context.Context, arg db.GetScopeCatalogueParams) (db.MeshmapperScopeCatalogue, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetScopeCatalogue", ctx, arg) + ret0, _ := ret[0].(db.MeshmapperScopeCatalogue) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetScopeCatalogue indicates an expected call of GetScopeCatalogue. +func (mr *MockQuerierMockRecorder) GetScopeCatalogue(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).GetScopeCatalogue), ctx, arg) +} + // GetScopeNames mocks base method. func (m *MockQuerier) GetScopeNames(ctx context.Context) ([]string, error) { m.ctrl.T.Helper() @@ -895,10 +940,10 @@ func (mr *MockQuerierMockRecorder) GetTransportScopes(ctx any) *gomock.Call { } // InsertChannelMessage mocks base method. -func (m *MockQuerier) InsertChannelMessage(ctx context.Context, arg db.InsertChannelMessageParams) (int64, error) { +func (m *MockQuerier) InsertChannelMessage(ctx context.Context, arg db.InsertChannelMessageParams) (db.InsertChannelMessageRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InsertChannelMessage", ctx, arg) - ret0, _ := ret[0].(int64) + ret0, _ := ret[0].(db.InsertChannelMessageRow) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -1208,6 +1253,21 @@ func (mr *MockQuerierMockRecorder) ListRegions(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegions", reflect.TypeOf((*MockQuerier)(nil).ListRegions), ctx) } +// ListRouteEvidence mocks base method. +func (m *MockQuerier) ListRouteEvidence(ctx context.Context, arg db.ListRouteEvidenceParams) ([]db.ListRouteEvidenceRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRouteEvidence", ctx, arg) + ret0, _ := ret[0].([]db.ListRouteEvidenceRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRouteEvidence indicates an expected call of ListRouteEvidence. +func (mr *MockQuerierMockRecorder) ListRouteEvidence(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRouteEvidence", reflect.TypeOf((*MockQuerier)(nil).ListRouteEvidence), ctx, arg) +} + // ListTraceTags mocks base method. func (m *MockQuerier) ListTraceTags(ctx context.Context, arg db.ListTraceTagsParams) ([]db.ListTraceTagsRow, error) { m.ctrl.T.Helper() @@ -1497,6 +1557,20 @@ func (mr *MockQuerierMockRecorder) ResolvePathHashesP4(ctx, arg any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePathHashesP4", reflect.TypeOf((*MockQuerier)(nil).ResolvePathHashesP4), ctx, arg) } +// SaveScopeCatalogue mocks base method. +func (m *MockQuerier) SaveScopeCatalogue(ctx context.Context, arg db.SaveScopeCatalogueParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SaveScopeCatalogue", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SaveScopeCatalogue indicates an expected call of SaveScopeCatalogue. +func (mr *MockQuerierMockRecorder) SaveScopeCatalogue(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveScopeCatalogue", reflect.TypeOf((*MockQuerier)(nil).SaveScopeCatalogue), ctx, arg) +} + // SearchKnownRoutes mocks base method. func (m *MockQuerier) SearchKnownRoutes(ctx context.Context, arg db.SearchKnownRoutesParams) ([]db.SearchKnownRoutesRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/models.go b/db/sqlc/models.go index b5ffb05..86fe98e 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -16,6 +16,164 @@ type Account struct { DeactivatedAt pgtype.Timestamptz `json:"deactivated_at"` } +type AnalyticsHourlyIataStat struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount int64 `json:"observation_count"` + UniquePackets int64 `json:"unique_packets"` +} + +type AnalyticsLiveHourlyIataStat struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount int64 `json:"observation_count"` + UniquePackets int64 `json:"unique_packets"` +} + +type AnalyticsLiveObserverActivityHourly struct { + ObserverID uuid.UUID `json:"observer_id"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin float32 `json:"snr_min"` + RssiSum int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + +type AnalyticsLivePathStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +type AnalyticsLivePayloadBreakdownByIatum struct { + Iata string `json:"iata"` + PayloadType *int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Count int64 `json:"count"` +} + +type AnalyticsLiveSignalStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrSum float64 `json:"snr_sum"` + RssiSamples int64 `json:"rssi_samples"` + RssiSum float64 `json:"rssi_sum"` +} + +type AnalyticsLiveTopAdvertisersByIatum struct { + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + NodeType int16 `json:"node_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` + LastHeard interface{} `json:"last_heard"` +} + +type AnalyticsLiveTopObserversByIatum struct { + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + ObservationCount int64 `json:"observation_count"` +} + +type AnalyticsLiveTopTalkersByIatum struct { + Iata string `json:"iata"` + SenderName *string `json:"sender_name"` + Bucket pgtype.Timestamptz `json:"bucket"` + MessageCount int64 `json:"message_count"` + LastSent interface{} `json:"last_sent"` +} + +type AnalyticsObserverActivityHourly struct { + ObserverID uuid.UUID `json:"observer_id"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs *float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum *float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin *float32 `json:"snr_min"` + RssiSum *int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + +type AnalyticsPathStatsHourly struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +type AnalyticsPayloadBreakdownByIatum struct { + Iata string `json:"iata"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Count int64 `json:"count"` +} + +type AnalyticsSignalStatsHourly struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrSum *float64 `json:"snr_sum"` + RssiSamples int64 `json:"rssi_samples"` + RssiSum *float64 `json:"rssi_sum"` +} + +type AnalyticsTopAdvertisersByIatum struct { + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Bucket pgtype.Timestamptz `json:"bucket"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` + LastHeard pgtype.Timestamptz `json:"last_heard"` + Name *string `json:"name"` + NodeType *int16 `json:"node_type"` +} + +type AnalyticsTopObserversByIatum struct { + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + Bucket pgtype.Timestamptz `json:"bucket"` + ObservationCount int64 `json:"observation_count"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` +} + +type AnalyticsTopTalkersByIatum struct { + Iata string `json:"iata"` + SenderName string `json:"sender_name"` + Bucket pgtype.Timestamptz `json:"bucket"` + MessageCount int64 `json:"message_count"` + LastSent pgtype.Timestamptz `json:"last_sent"` +} + type Channel struct { ID int32 `json:"id"` ChannelHash []byte `json:"channel_hash"` @@ -76,6 +234,17 @@ type KnownRoute struct { LastReconfirmedAt pgtype.Timestamptz `json:"last_reconfirmed_at"` } +type MeshmapperScopeCatalogue struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` +} + type MvHourlyIataStat struct { Iata string `json:"iata"` Hour pgtype.Timestamptz `json:"hour"` @@ -86,7 +255,7 @@ type MvHourlyIataStat struct { type MvObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` - PayloadType *int16 `json:"payload_type"` + PayloadType int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs float32 `json:"airtime_ms"` @@ -370,4 +539,5 @@ type TransportScope struct { TransportKey []byte `json:"transport_key"` KeyFingerprint []byte `json:"key_fingerprint"` CreatedAt pgtype.Timestamptz `json:"created_at"` + ImportedOnly bool `json:"imported_only"` } diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 1547c1c..1f80ab6 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -29,7 +29,7 @@ type Querier interface { // Opt-in age-out: preserve retained history and manually recorded ownership. // Bound deletions per cleanup tick and skip observers being updated by ingest. DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) - // One batch of expired packets; observations and channel messages cascade. + // Locks, archives and cascades one bounded packet cohort in a single transaction. DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) // One batch of routes past retention, or past grace with too few observations. // GREATEST keeps the scan on idx_known_routes_last_seen. @@ -58,6 +58,8 @@ type Querier interface { // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) + // Two indexed ranges, bounded to one observer; no legacy presence counters. + GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) // Sub-hour activity buckets straight off idx_observations_observer; no join to packets. // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. @@ -85,7 +87,11 @@ type Querier interface { GetRegion(ctx context.Context, id int32) (GetRegionRow, error) GetRegionBySlug(ctx context.Context, slug string) (GetRegionBySlugRow, error) GetRegionIATAs(ctx context.Context, regionID int32) ([]string, error) + GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) GetScopeByName(ctx context.Context, name string) (GetScopeByNameRow, error) + // Copyright 2026 Beacon Contributors + // SPDX-License-Identifier: AGPL-3.0-or-later + GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) GetScopeNames(ctx context.Context) ([]string, error) // Aggregate matching observations once, separately from node memberships to avoid // a cross-join. Empty IATAs keep the original global counts, including associations @@ -122,7 +128,9 @@ type Querier interface { // ============================================================ // CHANNEL MESSAGES // ============================================================ - InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (int64, error) + // Read the immutable first-packet scope in the same statement as insertion. + // A later reception's transport code must not give live and historical messages different tags. + InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) // ============================================================ // PACKET OBSERVATIONS // ============================================================ @@ -196,6 +204,10 @@ type Querier interface { // REGIONS // ============================================================ ListRegions(ctx context.Context) ([]ListRegionsRow, error) + // Leading index equalities and the time/ID boundary bound both custom and generic plans. + // Full-byte equality is required even when the compact digest matches. TRACE path bytes + // carry readings; unclassified legacy observations cannot be safely called ordinary paths. + ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) // ============================================================ // TRACES // ============================================================ @@ -243,6 +255,10 @@ type Querier interface { ResolvePathHashesP2(ctx context.Context, arg ResolvePathHashesP2Params) ([]ResolvePathHashesP2Row, error) ResolvePathHashesP3(ctx context.Context, arg ResolvePathHashesP3Params) ([]ResolvePathHashesP3Row, error) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashesP4Params) ([]ResolvePathHashesP4Row, error) + // One statement commits the validated snapshot and its lookup identities together. + // Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data + // after an error or 304. Imported names never replace existing manual metadata. + SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error // Returns known routes containing a subsequence from source to destination hash prefix. // Verifies source appears before destination in the route. SearchKnownRoutes(ctx context.Context, arg SearchKnownRoutesParams) ([]SearchKnownRoutesRow, error) diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index c047b44..e6fb1ed 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -124,17 +124,8 @@ func (q *Queries) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timest return items, nil } -const deleteOldPackets = `-- name: DeleteOldPackets :execrows -WITH expired AS ( - SELECT ep.packet_hash - FROM packets ep - WHERE ep.last_heard_at < $1 - ORDER BY ep.last_heard_at - LIMIT $2 - FOR UPDATE OF ep SKIP LOCKED -) -DELETE FROM packets p USING expired e -WHERE p.packet_hash = e.packet_hash +const deleteOldPackets = `-- name: DeleteOldPackets :one +SELECT archive_delete_packets($1::timestamptz, $2::integer)::bigint ` type DeleteOldPacketsParams struct { @@ -142,13 +133,12 @@ type DeleteOldPacketsParams struct { BatchSize int32 `json:"batch_size"` } -// One batch of expired packets; observations and channel messages cascade. +// Locks, archives and cascades one bounded packet cohort in a single transaction. func (q *Queries) DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) { - result, err := q.db.Exec(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) - if err != nil { - return 0, err - } - return result.RowsAffected(), nil + row := q.db.QueryRow(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err } const deleteOldRoutes = `-- name: DeleteOldRoutes :execrows @@ -660,7 +650,7 @@ SELECT COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $4::timestamptz GROUP BY 1 ORDER BY 1 ` @@ -669,6 +659,7 @@ type GetObserverActivityHourlyParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityHourlyRow struct { @@ -685,7 +676,12 @@ type GetObserverActivityHourlyRow struct { // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityHourly, arg.ObserverID, arg.Column2, arg.Column3) + rows, err := q.db.Query(ctx, getObserverActivityHourly, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + ) if err != nil { return nil, err } @@ -717,7 +713,7 @@ func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserver const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -725,15 +721,16 @@ ORDER BY count DESC type GetObserverActivityHourlyPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityHourlyPayloadTypesRow struct { - PayloadType *int16 `json:"payload_type"` - Count int64 `json:"count"` + PayloadType int16 `json:"payload_type"` + Count int64 `json:"count"` } func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2) + rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) if err != nil { return nil, err } @@ -752,6 +749,45 @@ func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg return items, nil } +const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = $1::uuid + AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly +` + +type GetObserverActivityLiveSummaryParams struct { + ObserverID uuid.UUID `json:"observer_id"` + GeneratedAt pgtype.Timestamptz `json:"generated_at"` + HourStart pgtype.Timestamptz `json:"hour_start"` + HourEnd pgtype.Timestamptz `json:"hour_end"` +} + +type GetObserverActivityLiveSummaryRow struct { + LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"` + LastCompleteHour int64 `json:"last_complete_hour"` +} + +// Two indexed ranges, bounded to one observer; no legacy presence counters. +func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) { + row := q.db.QueryRow(ctx, getObserverActivityLiveSummary, + arg.ObserverID, + arg.GeneratedAt, + arg.HourStart, + arg.HourEnd, + ) + var i GetObserverActivityLiveSummaryRow + err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour) + return i, err +} + const getObserverActivityRaw = `-- name: GetObserverActivityRaw :many SELECT date_bin($3::interval, heard_at, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, @@ -764,7 +800,7 @@ SELECT COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $4::timestamptz GROUP BY bucket ORDER BY bucket ` @@ -773,6 +809,7 @@ type GetObserverActivityRawParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityRawRow struct { @@ -791,7 +828,12 @@ type GetObserverActivityRawRow struct { // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverActivityRawParams) ([]GetObserverActivityRawRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityRaw, arg.ObserverID, arg.Column2, arg.Column3) + rows, err := q.db.Query(ctx, getObserverActivityRaw, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + ) if err != nil { return nil, err } @@ -823,7 +865,7 @@ func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverAct const getObserverActivityRawPayloadTypes = `-- name: GetObserverActivityRawPayloadTypes :many SELECT payload_type, COUNT(*)::bigint AS count FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -831,6 +873,7 @@ ORDER BY count DESC type GetObserverActivityRawPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityRawPayloadTypesRow struct { @@ -839,7 +882,7 @@ type GetObserverActivityRawPayloadTypesRow struct { } func (q *Queries) GetObserverActivityRawPayloadTypes(ctx context.Context, arg GetObserverActivityRawPayloadTypesParams) ([]GetObserverActivityRawPayloadTypesRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2) + rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) if err != nil { return nil, err } @@ -1466,6 +1509,34 @@ func (q *Queries) GetScopeByName(ctx context.Context, name string) (GetScopeByNa return i, err } +const getScopeCatalogue = `-- name: GetScopeCatalogue :one + +SELECT iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error FROM meshmapper_scope_catalogues WHERE iata = $1 AND url = $2 +` + +type GetScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` +} + +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +func (q *Queries) GetScopeCatalogue(ctx context.Context, arg GetScopeCatalogueParams) (MeshmapperScopeCatalogue, error) { + row := q.db.QueryRow(ctx, getScopeCatalogue, arg.Iata, arg.Url) + var i MeshmapperScopeCatalogue + err := row.Scan( + &i.Iata, + &i.Url, + &i.Payload, + &i.Etag, + &i.CheckedAt, + &i.AttemptedAt, + &i.NextAttempt, + &i.LastError, + ) + return i, err +} + const getScopeNames = `-- name: GetScopeNames :many SELECT name FROM transport_scopes ORDER BY name ` @@ -2008,7 +2079,7 @@ func (q *Queries) GetTransportScopeByName(ctx context.Context, name string) (int } const getTransportScopes = `-- name: GetTransportScopes :many -SELECT name, transport_key, key_fingerprint FROM transport_scopes ORDER BY name +SELECT name, transport_key, key_fingerprint FROM transport_scopes WHERE NOT imported_only ORDER BY name ` type GetTransportScopesRow struct { @@ -2039,10 +2110,16 @@ func (q *Queries) GetTransportScopes(ctx context.Context) ([]GetTransportScopesR const insertChannelMessage = `-- name: InsertChannelMessage :one -INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) -VALUES ($1, $2, $3, $4, $5) -ON CONFLICT (packet_hash) DO NOTHING -RETURNING id +WITH inserted AS ( + INSERT INTO channel_messages (channel_id, packet_hash, sender_name, content, sent_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (packet_hash) DO NOTHING + RETURNING id, packet_hash +) +SELECT inserted.id, ts.name AS scope_name, p.transport_codes_present +FROM inserted +JOIN packets p ON p.packet_hash = inserted.packet_hash +LEFT JOIN transport_scopes ts ON ts.id = p.scope_id ` type InsertChannelMessageParams struct { @@ -2053,10 +2130,18 @@ type InsertChannelMessageParams struct { SentAt pgtype.Timestamptz `json:"sent_at"` } +type InsertChannelMessageRow struct { + ID int64 `json:"id"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` +} + // ============================================================ // CHANNEL MESSAGES // ============================================================ -func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (int64, error) { +// Read the immutable first-packet scope in the same statement as insertion. +// A later reception's transport code must not give live and historical messages different tags. +func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMessageParams) (InsertChannelMessageRow, error) { row := q.db.QueryRow(ctx, insertChannelMessage, arg.ChannelID, arg.PacketHash, @@ -2064,9 +2149,9 @@ func (q *Queries) InsertChannelMessage(ctx context.Context, arg InsertChannelMes arg.Content, arg.SentAt, ) - var id int64 - err := row.Scan(&id) - return id, err + var i InsertChannelMessageRow + err := row.Scan(&i.ID, &i.ScopeName, &i.TransportCodesPresent) + return i, err } const insertObservation = `-- name: InsertObservation :one @@ -2238,7 +2323,7 @@ func (q *Queries) ListAccounts(ctx context.Context) ([]Account, error) { } const listAllChannelMessages = `-- name: ListAllChannelMessages :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -2262,16 +2347,18 @@ type ListAllChannelMessagesParams struct { } type ListAllChannelMessagesRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - PacketHashHex string `json:"packet_hash_hex"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + PacketHashHex string `json:"packet_hash_hex"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns all messages across all channels with optional time, IATA, scope and cursor filters. @@ -2302,6 +2389,8 @@ func (q *Queries) ListAllChannelMessages(ctx context.Context, arg ListAllChannel &i.SentAt, &i.PacketHashHex, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -2315,7 +2404,7 @@ func (q *Queries) ListAllChannelMessages(ctx context.Context, arg ListAllChannel } const listChannelMessages = `-- name: ListChannelMessages :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -2341,16 +2430,18 @@ type ListChannelMessagesParams struct { } type ListChannelMessagesRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - PacketHashHex string `json:"packet_hash_hex"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + PacketHashHex string `json:"packet_hash_hex"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns messages for a channel identified by integer ID. @@ -2383,6 +2474,8 @@ func (q *Queries) ListChannelMessages(ctx context.Context, arg ListChannelMessag &i.SentAt, &i.PacketHashHex, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -2396,7 +2489,7 @@ func (q *Queries) ListChannelMessages(ctx context.Context, arg ListChannelMessag } const listChannelMessagesByHash = `-- name: ListChannelMessagesByHash :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -2422,15 +2515,17 @@ type ListChannelMessagesByHashParams struct { } type ListChannelMessagesByHashRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns messages for all channels matching a hash byte. @@ -2462,6 +2557,8 @@ func (q *Queries) ListChannelMessagesByHash(ctx context.Context, arg ListChannel &i.Content, &i.SentAt, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -2704,7 +2801,7 @@ func (q *Queries) ListKnownRoutes(ctx context.Context, arg ListKnownRoutesParams } const listMessagesAfterID = `-- name: ListMessagesAfterID :many -SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, +SELECT DISTINCT ON (cm.id) cm.id, cm.channel_id, cm.packet_hash, cm.sender_name, cm.sender_pubkey, cm.content, cm.sent_at, encode(cm.packet_hash, 'hex') as packet_hash_hex, c.channel_hash, ts.name AS scope_name, p.transport_codes_present, (SELECT COUNT(*) FROM packet_observations po2 WHERE po2.packet_hash = cm.packet_hash) AS observation_count FROM channel_messages cm JOIN channels c ON c.id = cm.channel_id @@ -2726,16 +2823,18 @@ type ListMessagesAfterIDParams struct { } type ListMessagesAfterIDRow struct { - ID int64 `json:"id"` - ChannelID int32 `json:"channel_id"` - PacketHash []byte `json:"packet_hash"` - SenderName *string `json:"sender_name"` - SenderPubkey []byte `json:"sender_pubkey"` - Content *string `json:"content"` - SentAt pgtype.Timestamptz `json:"sent_at"` - PacketHashHex string `json:"packet_hash_hex"` - ChannelHash []byte `json:"channel_hash"` - ObservationCount int64 `json:"observation_count"` + ID int64 `json:"id"` + ChannelID int32 `json:"channel_id"` + PacketHash []byte `json:"packet_hash"` + SenderName *string `json:"sender_name"` + SenderPubkey []byte `json:"sender_pubkey"` + Content *string `json:"content"` + SentAt pgtype.Timestamptz `json:"sent_at"` + PacketHashHex string `json:"packet_hash_hex"` + ChannelHash []byte `json:"channel_hash"` + ScopeName *string `json:"scope_name"` + TransportCodesPresent *bool `json:"transport_codes_present"` + ObservationCount int64 `json:"observation_count"` } // Returns messages after the given message ID, ordered oldest first. @@ -2764,6 +2863,8 @@ func (q *Queries) ListMessagesAfterID(ctx context.Context, arg ListMessagesAfter &i.SentAt, &i.PacketHashHex, &i.ChannelHash, + &i.ScopeName, + &i.TransportCodesPresent, &i.ObservationCount, ); err != nil { return nil, err @@ -4328,6 +4429,60 @@ func (q *Queries) ResolvePathHashesP4(ctx context.Context, arg ResolvePathHashes return items, nil } +const saveScopeCatalogue = `-- name: SaveScopeCatalogue :exec +WITH inserted AS ( + INSERT INTO transport_scopes (name, transport_key, key_fingerprint, imported_only) + SELECT entry.name, entry.key, entry.fingerprint, TRUE + FROM (SELECT unnest($9::text[]) AS name, unnest($10::bytea[]) AS key, + unnest($11::bytea[]) AS fingerprint) AS entry + ON CONFLICT (name) DO NOTHING +) +INSERT INTO meshmapper_scope_catalogues (iata, url, payload, etag, checked_at, attempted_at, next_attempt, last_error) +VALUES ($1, $2, $3::jsonb, $4::text, + $5::timestamptz, $6, $7, $8) +ON CONFLICT (iata, url) DO UPDATE SET + payload = COALESCE(EXCLUDED.payload, meshmapper_scope_catalogues.payload), + etag = COALESCE(EXCLUDED.etag, meshmapper_scope_catalogues.etag), + checked_at = COALESCE(EXCLUDED.checked_at, meshmapper_scope_catalogues.checked_at), + attempted_at = EXCLUDED.attempted_at, + next_attempt = EXCLUDED.next_attempt, + last_error = EXCLUDED.last_error +` + +type SaveScopeCatalogueParams struct { + Iata string `json:"iata"` + Url string `json:"url"` + Payload []byte `json:"payload"` + Etag *string `json:"etag"` + CheckedAt pgtype.Timestamptz `json:"checked_at"` + AttemptedAt pgtype.Timestamptz `json:"attempted_at"` + NextAttempt pgtype.Timestamptz `json:"next_attempt"` + LastError string `json:"last_error"` + Names []string `json:"names"` + Keys [][]byte `json:"keys"` + Fingerprints [][]byte `json:"fingerprints"` +} + +// One statement commits the validated snapshot and its lookup identities together. +// Empty arrays insert nothing. NULL payload/checked_at retain last-known-good data +// after an error or 304. Imported names never replace existing manual metadata. +func (q *Queries) SaveScopeCatalogue(ctx context.Context, arg SaveScopeCatalogueParams) error { + _, err := q.db.Exec(ctx, saveScopeCatalogue, + arg.Iata, + arg.Url, + arg.Payload, + arg.Etag, + arg.CheckedAt, + arg.AttemptedAt, + arg.NextAttempt, + arg.LastError, + arg.Names, + arg.Keys, + arg.Fingerprints, + ) + return err +} + const searchKnownRoutes = `-- name: SearchKnownRoutes :many SELECT id, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count FROM known_routes @@ -4432,11 +4587,12 @@ func (q *Queries) SetPacketDecrypted(ctx context.Context, packetHash []byte) err const touchObserverBrokers = `-- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name ` @@ -4445,10 +4601,16 @@ type TouchObserverBrokersParams struct { Column1 []uuid.UUID `json:"column_1"` Column2 []string `json:"column_2"` Column3 []pgtype.Timestamptz `json:"column_3"` + Column4 []pgtype.Timestamptz `json:"column_4"` } func (q *Queries) TouchObserverBrokers(ctx context.Context, arg TouchObserverBrokersParams) error { - _, err := q.db.Exec(ctx, touchObserverBrokers, arg.Column1, arg.Column2, arg.Column3) + _, err := q.db.Exec(ctx, touchObserverBrokers, + arg.Column1, + arg.Column2, + arg.Column3, + arg.Column4, + ) return err } @@ -4953,22 +5115,23 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe const upsertObserverBroker = `-- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN $3::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW() + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at) ` type UpsertObserverBrokerParams struct { ObserverID uuid.UUID `json:"observer_id"` BrokerName string `json:"broker_name"` + IsPacket bool `json:"is_packet"` } // ============================================================ // OBSERVER BROKERS // ============================================================ func (q *Queries) UpsertObserverBroker(ctx context.Context, arg UpsertObserverBrokerParams) error { - _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName) + _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName, arg.IsPacket) return err } @@ -5177,7 +5340,8 @@ VALUES ($1, $2, $3, $4) ON CONFLICT (name) DO UPDATE SET display_name = EXCLUDED.display_name, transport_key = EXCLUDED.transport_key, - key_fingerprint = EXCLUDED.key_fingerprint + key_fingerprint = EXCLUDED.key_fingerprint, + imported_only = FALSE ` type UpsertTransportScopeParams struct { diff --git a/db/sqlc/route_evidence.sql.go b/db/sqlc/route_evidence.sql.go new file mode 100644 index 0000000..0099c47 --- /dev/null +++ b/db/sqlc/route_evidence.sql.go @@ -0,0 +1,126 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: route_evidence.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const getRouteEvidenceRoute = `-- name: GetRouteEvidenceRoute :one +SELECT id, path_key, node_ids, hash_prefix, iata, hop_count, first_seen, last_seen, observation_count, last_reconfirmed_at FROM known_routes +WHERE iata = $1::bpchar AND path_key = $2::bytea +` + +type GetRouteEvidenceRouteParams struct { + Iata string `json:"iata"` + PathKey []byte `json:"path_key"` +} + +func (q *Queries) GetRouteEvidenceRoute(ctx context.Context, arg GetRouteEvidenceRouteParams) (KnownRoute, error) { + row := q.db.QueryRow(ctx, getRouteEvidenceRoute, arg.Iata, arg.PathKey) + var i KnownRoute + err := row.Scan( + &i.ID, + &i.PathKey, + &i.NodeIds, + &i.HashPrefix, + &i.Iata, + &i.HopCount, + &i.FirstSeen, + &i.LastSeen, + &i.ObservationCount, + &i.LastReconfirmedAt, + ) + return i, err +} + +const listRouteEvidence = `-- name: ListRouteEvidence :many +SELECT po.id, po.packet_hash, po.observer_id, o.display_name AS observer_name, + po.heard_at, po.payload_type, po.rssi, po.snr +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.iata = $1::bpchar + AND po.hash_size = $2::smallint + AND decode(md5(po.path_bytes), 'hex') = $3::bytea + AND po.path_bytes = $4::bytea + AND po.hop_count >= 2 + AND po.hop_count = $5::smallint + AND po.path_bytes IS NOT NULL AND po.payload_type IS NOT NULL AND po.payload_type <> 9 + AND po.heard_at >= $6::timestamptz AND po.heard_at < $7::timestamptz + AND (po.heard_at, po.id) < ($8::timestamptz, $9::bigint) +ORDER BY po.heard_at DESC, po.id DESC +LIMIT $10::integer +` + +type ListRouteEvidenceParams struct { + Iata string `json:"iata"` + HashSize int16 `json:"hash_size"` + PathDigest []byte `json:"path_digest"` + PathBytes []byte `json:"path_bytes"` + HopCount int16 `json:"hop_count"` + Since pgtype.Timestamptz `json:"since"` + Until pgtype.Timestamptz `json:"until"` + BeforeAt pgtype.Timestamptz `json:"before_at"` + BeforeID int64 `json:"before_id"` + PageLimit int32 `json:"page_limit"` +} + +type ListRouteEvidenceRow struct { + ID int64 `json:"id"` + PacketHash []byte `json:"packet_hash"` + ObserverID uuid.UUID `json:"observer_id"` + ObserverName *string `json:"observer_name"` + HeardAt pgtype.Timestamptz `json:"heard_at"` + PayloadType *int16 `json:"payload_type"` + Rssi *int16 `json:"rssi"` + Snr *float32 `json:"snr"` +} + +// Leading index equalities and the time/ID boundary bound both custom and generic plans. +// Full-byte equality is required even when the compact digest matches. TRACE path bytes +// carry readings; unclassified legacy observations cannot be safely called ordinary paths. +func (q *Queries) ListRouteEvidence(ctx context.Context, arg ListRouteEvidenceParams) ([]ListRouteEvidenceRow, error) { + rows, err := q.db.Query(ctx, listRouteEvidence, + arg.Iata, + arg.HashSize, + arg.PathDigest, + arg.PathBytes, + arg.HopCount, + arg.Since, + arg.Until, + arg.BeforeAt, + arg.BeforeID, + arg.PageLimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListRouteEvidenceRow{} + for rows.Next() { + var i ListRouteEvidenceRow + if err := rows.Scan( + &i.ID, + &i.PacketHash, + &i.ObserverID, + &i.ObserverName, + &i.HeardAt, + &i.PayloadType, + &i.Rssi, + &i.Snr, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/db/store.go b/db/store.go index 20866d7..2f9f35a 100644 --- a/db/store.go +++ b/db/store.go @@ -130,7 +130,7 @@ func tristate(b *bool) string { } // toChannelMessage maps raw sqlc row fields to an api.ChannelMessage. -func toChannelMessage(id int64, packetHashHex string, channelHash []byte, senderName *string, content *string, sentAt pgtype.Timestamptz, observationCount int64) api.ChannelMessage { +func toChannelMessage(id int64, packetHashHex string, channelHash []byte, senderName *string, content *string, sentAt pgtype.Timestamptz, observationCount int64, scope *string, transport *bool) api.ChannelMessage { sn := "" if senderName != nil { sn = *senderName @@ -147,6 +147,8 @@ func toChannelMessage(id int64, packetHashHex string, channelHash []byte, sender Content: ct, SentAt: sentAt.Time.UnixMilli(), ObservationCount: observationCount, + Scope: scope, + ScopeStatus: api.RecordedChannelScopeStatus(scope, transport), } } diff --git a/db/store_test.go b/db/store_test.go index 4d0521b..41f4393 100644 --- a/db/store_test.go +++ b/db/store_test.go @@ -59,7 +59,7 @@ func TestToChannelMessage(t *testing.T) { channelHash := []byte{0xab} sentAt := pgtype.Timestamptz{Time: time.UnixMilli(1700000000000), Valid: true} - msg := toChannelMessage(42, "deadbeef", channelHash, &senderName, &content, sentAt, 7) + msg := toChannelMessage(42, "deadbeef", channelHash, &senderName, &content, sentAt, 7, nil, nil) if msg.ID != 42 { t.Errorf("expected ID 42, got %d", msg.ID) @@ -86,7 +86,7 @@ func TestToChannelMessage(t *testing.T) { func TestToChannelMessage_NilFields(t *testing.T) { sentAt := pgtype.Timestamptz{Time: time.UnixMilli(0), Valid: true} - msg := toChannelMessage(1, "abc", []byte{0x01}, nil, nil, sentAt, 0) + msg := toChannelMessage(1, "abc", []byte{0x01}, nil, nil, sentAt, 0, nil, nil) if msg.SenderName != "" { t.Errorf("expected empty SenderName, got %s", msg.SenderName) } diff --git a/docs/docs.go b/docs/docs.go index 43ed843..288a70b 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1468,6 +1468,12 @@ const docTemplate = `{ "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", "name": "interval", "in": "query" + }, + { + "type": "integer", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until.", + "name": "until", + "in": "query" } ], "responses": { @@ -2125,8 +2131,95 @@ const docTemplate = `{ } } }, + "/routes/{iata}/{pathKey}/observations": { + "get": { + "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Get retained reports matching a full saved route prefix sequence", + "parameters": [ + { + "type": "string", + "description": "Three-character IATA code", + "name": "iata", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Stable 32-hex pathKey from a known-route response", + "name": "pathKey", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Window start epoch ms; provide with until (default last 24h)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Exclusive window end epoch ms; maximum span 30d, no future end", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Default 50; positive, capped at 200", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/scopes": { "get": { + "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", "produces": [ "application/json" ], @@ -3388,6 +3481,19 @@ const docTemplate = `{ "description": "hex-encoded packet hash for correlation with packet events", "type": "string" }, + "scope": { + "description": "matched scope on the first stored packet; null when none was recorded", + "type": "string" + }, + "scopeStatus": { + "type": "string", + "enum": [ + "matched", + "unscoped", + "unknown", + "unavailable" + ] + }, "senderName": { "description": "display name from the decrypted payload", "type": "string" @@ -3595,6 +3701,10 @@ const docTemplate = `{ }, "observationCount": { "type": "integer" + }, + "pathKey": { + "description": "stable identity within this IATA; use for route evidence links", + "type": "string" } } }, @@ -3924,6 +4034,7 @@ const docTemplate = `{ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -3979,6 +4090,10 @@ const docTemplate = `{ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { "type": "object", "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, "interval": { "type": "string" }, @@ -3999,6 +4114,21 @@ const docTemplate = `{ }, "range": { "type": "string" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive end, epoch ms; live requests include the current partial bucket", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" } } }, @@ -4046,6 +4176,27 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "description": "stored observations within windowStart/windowEnd", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { @@ -4798,6 +4949,50 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence": { + "type": "object", + "properties": { + "generatedAt": { + "type": "integer" + }, + "hasMore": { + "type": "boolean" + }, + "hashSize": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation" + } + }, + "matchAvailable": { + "type": "boolean" + }, + "matchType": { + "type": "string" + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + }, + "pathBytes": { + "type": "string" + }, + "route": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + }, + "windowEnd": { + "type": "integer" + }, + "windowStart": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop": { "type": "object", "properties": { @@ -4818,6 +5013,38 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation": { + "type": "object", + "properties": { + "heardAt": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "observerId": { + "type": "string" + }, + "observerName": { + "type": "string" + }, + "packetHash": { + "type": "string" + }, + "payloadType": { + "type": "integer" + }, + "payloadTypeName": { + "type": "string" + }, + "rssi": { + "type": "integer" + }, + "snr": { + "type": "number" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 8a83f4c..5679444 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1466,6 +1466,12 @@ "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", "name": "interval", "in": "query" + }, + { + "type": "integer", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until.", + "name": "until", + "in": "query" } ], "responses": { @@ -2123,8 +2129,95 @@ } } }, + "/routes/{iata}/{pathKey}/observations": { + "get": { + "description": "Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope.", + "produces": [ + "application/json" + ], + "tags": [ + "Routes" + ], + "summary": "Get retained reports matching a full saved route prefix sequence", + "parameters": [ + { + "type": "string", + "description": "Three-character IATA code", + "name": "iata", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Stable 32-hex pathKey from a known-route response", + "name": "pathKey", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor", + "name": "range", + "in": "query" + }, + { + "type": "integer", + "description": "Window start epoch ms; provide with until (default last 24h)", + "name": "since", + "in": "query" + }, + { + "type": "integer", + "description": "Exclusive window end epoch ms; maximum span 30d, no future end", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "Opaque precise cursor from nextPageCursor; window is pinned", + "name": "pageCursor", + "in": "query" + }, + { + "maximum": 200, + "minimum": 1, + "type": "integer", + "description": "Default 50; positive, capped at 200", + "name": "limit", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/internal_api_handlers.APIError" + } + } + } + } + }, "/scopes": { "get": { + "description": "The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership.", "produces": [ "application/json" ], @@ -3386,6 +3479,19 @@ "description": "hex-encoded packet hash for correlation with packet events", "type": "string" }, + "scope": { + "description": "matched scope on the first stored packet; null when none was recorded", + "type": "string" + }, + "scopeStatus": { + "type": "string", + "enum": [ + "matched", + "unscoped", + "unknown", + "unavailable" + ] + }, "senderName": { "description": "display name from the decrypted payload", "type": "string" @@ -3593,6 +3699,10 @@ }, "observationCount": { "type": "integer" + }, + "pathKey": { + "description": "stable identity within this IATA; use for route evidence links", + "type": "string" } } }, @@ -3922,6 +4032,7 @@ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -3977,6 +4088,10 @@ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { "type": "object", "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, "interval": { "type": "string" }, @@ -3997,6 +4112,21 @@ }, "range": { "type": "string" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive end, epoch ms; live requests include the current partial bucket", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" } } }, @@ -4044,6 +4174,27 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "description": "stored observations within windowStart/windowEnd", + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { @@ -4796,6 +4947,50 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence": { + "type": "object", + "properties": { + "generatedAt": { + "type": "integer" + }, + "hasMore": { + "type": "boolean" + }, + "hashSize": { + "type": "integer" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation" + } + }, + "matchAvailable": { + "type": "boolean" + }, + "matchType": { + "type": "string" + }, + "nextCursor": { + "type": "integer" + }, + "nextPageCursor": { + "type": "string" + }, + "pathBytes": { + "type": "string" + }, + "route": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute" + }, + "windowEnd": { + "type": "integer" + }, + "windowStart": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop": { "type": "object", "properties": { @@ -4816,6 +5011,38 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation": { + "type": "object", + "properties": { + "heardAt": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "observerId": { + "type": "string" + }, + "observerName": { + "type": "string" + }, + "packetHash": { + "type": "string" + }, + "payloadType": { + "type": "integer" + }, + "payloadTypeName": { + "type": "string" + }, + "rssi": { + "type": "integer" + }, + "snr": { + "type": "number" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f03b978..a58626f 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -135,6 +135,17 @@ definitions: packetHash: description: hex-encoded packet hash for correlation with packet events type: string + scope: + description: matched scope on the first stored packet; null when none was + recorded + type: string + scopeStatus: + enum: + - matched + - unscoped + - unknown + - unavailable + type: string senderName: description: display name from the decrypted payload type: string @@ -272,6 +283,9 @@ definitions: type: integer observationCount: type: integer + pathKey: + description: stable identity within this IATA; use for route evidence links + type: string type: object github_com_MeshCore-Beacon_beacon-server_internal_api.Node: properties: @@ -521,6 +535,7 @@ definitions: description: epoch ms type: integer observationCount: + description: legacy cumulative presence counter; includes non-packet events type: integer observerType: description: e.g. "meshcoretomqtt", "meshcoreha" @@ -560,6 +575,9 @@ definitions: type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity: properties: + generatedAt: + description: response computation time, not proof of continuous coverage + type: integer interval: type: string payloadTypes: @@ -574,6 +592,18 @@ definitions: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio' range: type: string + source: + description: raw or hourly; missing records do not prove an outage + type: string + summary: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary' + windowEnd: + description: exclusive end, epoch ms; live requests include the current partial + bucket + type: integer + windowStart: + description: inclusive complete-bucket start, epoch ms + type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint: properties: @@ -604,6 +634,20 @@ definitions: sf: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary: + properties: + lastCompleteHour: + type: integer + lastCompleteHourEnd: + type: integer + lastCompleteHourStart: + type: integer + latestRecordedAt: + type: integer + recordedPackets: + description: stored observations within windowStart/windowEnd + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker: properties: lastPacketAt: @@ -1123,6 +1167,35 @@ definitions: description: hex-encoded prefix used for resolution type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence: + properties: + generatedAt: + type: integer + hasMore: + type: boolean + hashSize: + type: integer + items: + items: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation' + type: array + matchAvailable: + type: boolean + matchType: + type: string + nextCursor: + type: integer + nextPageCursor: + type: string + pathBytes: + type: string + route: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.KnownRoute' + windowEnd: + type: integer + windowStart: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.RouteHop: properties: hashBytes: @@ -1135,6 +1208,27 @@ definitions: nodeId: type: string type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.RouteObservation: + properties: + heardAt: + type: integer + id: + type: integer + observerId: + type: string + observerName: + type: string + packetHash: + type: string + payloadType: + type: integer + payloadTypeName: + type: string + rssi: + type: integer + snr: + type: number + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ScopeDetail: properties: iataCount: @@ -2401,6 +2495,13 @@ paths: in: query name: interval type: string + - description: Optional exclusive end in epoch milliseconds; aligned down to + a complete bucket, at most 30 days old. Activity uses this window; summary + freshness/latestRecordedAt and the last complete hour are measured at response + generation time, independently of until. + in: query + name: until + type: integer produces: - application/json responses: @@ -2766,6 +2867,69 @@ paths: summary: List known routes tags: - Routes + /routes/{iata}/{pathKey}/observations: + get: + description: Matches the saved IATA, complete path bytes and hash width. Excludes + TRACE and unclassified reports; other widths and search-result subsegments + are not included. Byte matches do not prove hop identities or delivery. The + route and its historical counter can outlive raw evidence. Cursors retain + full timestamp precision and pin route/window scope. + parameters: + - description: Three-character IATA code + in: path + name: iata + required: true + type: string + - description: Stable 32-hex pathKey from a known-route response + in: path + name: pathKey + required: true + type: string + - description: Server-anchored duration (default 24h, max 720h); exclusive with + since/until/pageCursor + in: query + name: range + type: string + - description: Window start epoch ms; provide with until (default last 24h) + in: query + name: since + type: integer + - description: Exclusive window end epoch ms; maximum span 30d, no future end + in: query + name: until + type: integer + - description: Opaque precise cursor from nextPageCursor; window is pinned + in: query + name: pageCursor + type: string + - description: Default 50; positive, capped at 200 + in: query + maximum: 200 + minimum: 1 + name: limit + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.RouteEvidence' + "400": + description: Bad Request + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "404": + description: Not Found + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/internal_api_handlers.APIError' + summary: Get retained reports matching a full saved route prefix sequence + tags: + - Routes /routes/cross: get: parameters: @@ -2849,6 +3013,10 @@ paths: - Routes /scopes: get: + description: The unfiltered list includes stored imported names after an importer + is disabled or a source is removed. Those historical identities are retained + indefinitely; listing them does not establish observed traffic or current + source membership. parameters: - description: Filter by IATA code(s), comma-separated in: query diff --git a/internal/api/channel_scope_test.go b/internal/api/channel_scope_test.go new file mode 100644 index 0000000..f553a62 --- /dev/null +++ b/internal/api/channel_scope_test.go @@ -0,0 +1,27 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import "testing" + +func TestRecordedChannelScopeStatus(t *testing.T) { + name := "#yow" + for _, tc := range []struct { + name *string + transport, valid bool + want string + }{ + {&name, true, true, "matched"}, {nil, true, true, "unknown"}, + {nil, false, true, "unscoped"}, {nil, false, false, "unavailable"}, + {&name, false, false, "matched"}, + } { + var transport *bool + if tc.valid { + transport = &tc.transport + } + if got := RecordedChannelScopeStatus(tc.name, transport); string(got) != tc.want { + t.Fatalf("got %s, want %s", got, tc.want) + } + } +} diff --git a/internal/api/channels.go b/internal/api/channels.go index c609a23..00e9d85 100644 --- a/internal/api/channels.go +++ b/internal/api/channels.go @@ -56,13 +56,32 @@ func ParseChannelCursor(raw string) (*ChannelCursor, error) { // ChannelMessage represents a single decrypted channel message. // Only messages for channels with a known key are stored and returned. type ChannelMessage struct { - ID int64 `json:"id"` - PacketHash string `json:"packetHash"` // hex-encoded packet hash for correlation with packet events - ChannelHash string `json:"channelHash"` // hex-encoded single-byte channel hash - SenderName string `json:"senderName"` // display name from the decrypted payload - Content string `json:"content"` // decrypted message text - SentAt int64 `json:"sentAt"` // epoch ms, from the sender's embedded timestamp - ObservationCount int64 `json:"observationCount"` // number of packet_observations rows for this message's packet hash + ID int64 `json:"id"` + PacketHash string `json:"packetHash"` // hex-encoded packet hash for correlation with packet events + ChannelHash string `json:"channelHash"` // hex-encoded single-byte channel hash + SenderName string `json:"senderName"` // display name from the decrypted payload + Content string `json:"content"` // decrypted message text + SentAt int64 `json:"sentAt"` // epoch ms, from the sender's embedded timestamp + ObservationCount int64 `json:"observationCount"` // number of packet_observations rows for this message's packet hash + Scope *string `json:"scope"` // matched scope on the first stored packet; null when none was recorded + ScopeStatus ChannelScopeStatus `json:"scopeStatus" enums:"matched,unscoped,unknown,unavailable"` +} + +// ChannelScopeStatus describes stored packet evidence, not a channel key or all reception paths. +type ChannelScopeStatus string + +// RecordedChannelScopeStatus preserves missing capture metadata as unavailable. +func RecordedChannelScopeStatus(name *string, transport *bool) ChannelScopeStatus { + if name != nil { + return "matched" + } + if transport == nil { + return "unavailable" + } + if *transport { + return "unknown" + } + return "unscoped" } // ChannelSummary is the minimal channel representation used in list responses. diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 43147b5..66bddd8 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -256,6 +256,7 @@ var activityIntervals = map[string]time.Duration{ // @Param observerId path string true "Observer UUID" // @Param range query string false "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h" // @Param interval query string false "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)" +// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until." // @Success 200 {object} api.ObserverActivity // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError @@ -295,7 +296,17 @@ func getObserverActivity(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "range/interval exceeds 1000 buckets") return } - activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval) + until := time.Time{} + if values, ok := r.URL.Query()["until"]; ok { + n, err := strconv.ParseInt(values[0], 10, 64) + now := time.Now() + if len(values) != 1 || err != nil || n < 0 || n > now.UnixMilli() || n < now.Add(-720*time.Hour).UnixMilli() { + respondError(w, http.StatusBadRequest, "until must be one past epoch-millisecond timestamp within 30 days") + return + } + until = time.UnixMilli(n) + } + activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval, until) if err != nil { if errors.Is(err, pgx.ErrNoRows) { respondError(w, http.StatusNotFound, "observer not found") diff --git a/internal/api/handlers/observers_test.go b/internal/api/handlers/observers_test.go index 1bf865f..cfef1c9 100644 --- a/internal/api/handlers/observers_test.go +++ b/internal/api/handlers/observers_test.go @@ -200,7 +200,7 @@ func TestGetObserverActivity_Defaults(t *testing.T) { var gotWindow, gotInterval time.Duration r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -232,7 +232,7 @@ func TestGetObserverActivity_CustomRangeAndInterval(t *testing.T) { var gotWindow, gotInterval time.Duration r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -264,7 +264,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { t.Fatal("reader should not be called") return nil, nil }, @@ -286,7 +286,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { var gotWindow, gotInterval time.Duration ok := chi.NewRouter() ok.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -309,6 +309,10 @@ func TestGetObserverActivity_BadRequests(t *testing.T) { query string }{ {"invalid uuid", "not-a-uuid", ""}, + {"bad end", "00000000-0000-0000-0000-000000000001", "?until=banana"}, + {"negative end", "00000000-0000-0000-0000-000000000001", "?until=-1"}, + {"duplicate end", "00000000-0000-0000-0000-000000000001", "?until=1&until=2"}, + {"future end", "00000000-0000-0000-0000-000000000001", "?until=253402300799999"}, {"unparseable range", "00000000-0000-0000-0000-000000000001", "?range=banana"}, {"range too long", "00000000-0000-0000-0000-000000000001", "?range=721h"}, {"zero range", "00000000-0000-0000-0000-000000000001", "?range=0"}, @@ -319,7 +323,7 @@ func TestGetObserverActivity_BadRequests(t *testing.T) { t.Run(tt.name, func(t *testing.T) { r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { t.Fatal("reader should not be called") return nil, nil }, @@ -338,7 +342,7 @@ func TestGetObserverActivity_NotFound(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, pgx.ErrNoRows }, })) @@ -364,7 +368,7 @@ func TestGetObserverActivity_ReaderError(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, errors.New("boom") }, })) @@ -381,7 +385,7 @@ func TestGetObserverActivity_NullsSerialise(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return &api.ObserverActivity{ PayloadTypes: []api.PayloadBreakdownItem{}, Points: []api.ObserverActivityPoint{{T: 1757376000000, Observations: 3}}, diff --git a/internal/api/handlers/route_evidence.go b/internal/api/handlers/route_evidence.go new file mode 100644 index 0000000..6c156fe --- /dev/null +++ b/internal/api/handlers/route_evidence.go @@ -0,0 +1,118 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5" +) + +func routeEvidenceQuery(r *http.Request, iata, key string, now time.Time) (api.RouteEvidenceQuery, error) { + q := api.RouteEvidenceQuery{} + values := r.URL.Query() + for _, name := range []string{"pageCursor", "since", "until", "range", "limit"} { + if len(values[name]) > 1 { + return q, api.ErrRouteEvidenceInput + } + } + if values.Has("cursor") { + return q, api.ErrRouteEvidenceInput + } + limit, err := parseLimit(r, 50) + if err != nil { + return q, err + } + q.Limit = limit + if values.Has("pageCursor") { + c, err := api.ParseRouteEvidenceCursor(values.Get("pageCursor")) + if err != nil || c.IATA != iata || c.PathKey != key { + return q, api.ErrRouteEvidenceInput + } + q.Cursor = c + q.Since = c.Since + q.Until = c.Until + } else { + q.Until = now.UTC().Truncate(time.Millisecond) + q.Since = q.Until.Add(-24 * time.Hour) + } + if values.Has("range") { + if values.Has("pageCursor") || values.Has("since") || values.Has("until") { + return q, api.ErrRouteEvidenceInput + } + duration, err := time.ParseDuration(values.Get("range")) + if err != nil || duration < time.Millisecond || duration > api.MaxRouteEvidenceWindow || duration%time.Millisecond != 0 { + return q, api.ErrRouteEvidenceInput + } + q.Since = q.Until.Add(-duration) + } + if values.Has("since") || values.Has("until") { + if !values.Has("since") || !values.Has("until") { + return q, api.ErrRouteEvidenceInput + } + since, e1 := strconv.ParseInt(values.Get("since"), 10, 64) + until, e2 := strconv.ParseInt(values.Get("until"), 10, 64) + if e1 != nil || e2 != nil || since < 0 || until < 0 || since > 253402300799999 || until > 253402300799999 { + return q, api.ErrRouteEvidenceInput + } + q.Since = time.UnixMilli(since).UTC() + q.Until = time.UnixMilli(until).UTC() + if q.Cursor != nil && (!q.Since.Equal(q.Cursor.Since) || !q.Until.Equal(q.Cursor.Until)) { + return q, api.ErrRouteEvidenceInput + } + } + if !api.ValidRouteEvidenceWindow(q.Since, q.Until) || q.Until.After(now) { + return q, api.ErrRouteEvidenceInput + } + return q, nil +} + +// getRouteEvidence godoc +// +// @Summary Get retained reports matching a full saved route prefix sequence +// @Description Matches the saved IATA, complete path bytes and hash width. Excludes TRACE and unclassified reports; other widths and search-result subsegments are not included. Byte matches do not prove hop identities or delivery. The route and its historical counter can outlive raw evidence. Cursors retain full timestamp precision and pin route/window scope. +// @Tags Routes +// @Produce json +// @Param iata path string true "Three-character IATA code" +// @Param pathKey path string true "Stable 32-hex pathKey from a known-route response" +// @Param range query string false "Server-anchored duration (default 24h, max 720h); exclusive with since/until/pageCursor" +// @Param since query int false "Window start epoch ms; provide with until (default last 24h)" +// @Param until query int false "Exclusive window end epoch ms; maximum span 30d, no future end" +// @Param pageCursor query string false "Opaque precise cursor from nextPageCursor; window is pinned" +// @Param limit query int false "Default 50; positive, capped at 200" minimum(1) maximum(200) +// @Success 200 {object} api.RouteEvidence +// @Failure 400 {object} handlers.APIError +// @Failure 404 {object} handlers.APIError +// @Failure 500 {object} handlers.APIError +// @Router /routes/{iata}/{pathKey}/observations [get] +func getRouteEvidence(reader api.Reader) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + iata, key := strings.ToUpper(chi.URLParam(r, "iata")), strings.ToLower(chi.URLParam(r, "pathKey")) + if !api.ValidRouteEvidenceKey(iata, key) { + respondError(w, 400, "invalid IATA or route pathKey") + return + } + query, err := routeEvidenceQuery(r, iata, key, time.Now()) + if err != nil { + respondError(w, 400, "invalid route evidence window, limit or page cursor") + return + } + result, err := reader.GetRouteEvidence(r.Context(), iata, key, query) + switch { + case errors.Is(err, pgx.ErrNoRows), err == nil && result == nil: + respondError(w, 404, "saved route not found") + case errors.Is(err, api.ErrRouteEvidenceInput): + respondError(w, 400, "invalid route evidence request") + case err != nil: + respondError(w, 500, "internal server error") + default: + respond(w, 200, result) + } + } +} diff --git a/internal/api/handlers/route_evidence_test.go b/internal/api/handlers/route_evidence_test.go new file mode 100644 index 0000000..8340d85 --- /dev/null +++ b/internal/api/handlers/route_evidence_test.go @@ -0,0 +1,78 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package handlers + +import ( + "context" + "errors" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/jackc/pgx/v5" +) + +const evidenceKey = "f097439148601d9f3291c474f82fa64c" + +func TestRouteEvidenceQuery(t *testing.T) { + now := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC) + q, err := routeEvidenceQuery(httptest.NewRequest("GET", "/", nil), "YOW", evidenceKey, now) + if err != nil || q.Limit != 50 || q.Until != now || q.Until.Sub(q.Since) != 24*time.Hour { + t.Fatalf("defaults: %+v %v", q, err) + } + week, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?range=168h", nil), "YOW", evidenceKey, now) + if err != nil || !week.Until.Equal(now) || week.Until.Sub(week.Since) != 7*24*time.Hour { + t.Fatalf("server anchored range: %+v %v", week, err) + } + c := api.RouteEvidenceCursor{IATA: "YOW", PathKey: evidenceKey, Since: q.Since, Until: q.Until, HeardAt: q.Since.Add(time.Hour + 123*time.Microsecond), ID: 3} + next, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String())+"&limit=999", nil), "YOW", evidenceKey, now.Add(time.Hour)) + if err != nil || next.Limit != 200 || !next.Until.Equal(now) || !next.Cursor.HeardAt.Equal(c.HeardAt) { + t.Fatalf("pinned cursor: %+v %v", next, err) + } + for _, query := range []string{"?limit=0", "?limit=-1", "?limit=a", "?limit=1&limit=2", "?since=0", "?until=10", "?since=2&until=1", "?since=0&until=2678400000", "?since=0&until=99999999999999999", "?pageCursor=bad", "?pageCursor=", "?cursor=123", "?pageCursor=" + url.QueryEscape(c.String()) + "&since=0&until=10"} { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?pageCursor="+url.QueryEscape(c.String()), nil), "YVR", evidenceKey, now); err == nil { + t.Fatal("cursor crossed IATA") + } + for _, query := range []string{"range=0h", "range=-1h", "range=721h", "range=1ns", "range=7d", "range=24h&range=168h", "range=24h&since=0&until=10", "range=24h&pageCursor=" + url.QueryEscape(c.String())} { + if _, err := routeEvidenceQuery(httptest.NewRequest("GET", "/?"+query, nil), "YOW", evidenceKey, now); err == nil { + t.Errorf("accepted %s", query) + } + } +} + +func TestRouteEvidenceHandler(t *testing.T) { + for _, tc := range []struct { + path string + err error + status int + }{ + {"/yow/" + evidenceKey + "/observations", nil, 200}, + {"/YOW/bad/observations", nil, 400}, + {"/LONG/" + evidenceKey + "/observations", nil, 400}, + {"/YOW/" + evidenceKey + "/observations", pgx.ErrNoRows, 404}, + {"/YOW/" + evidenceKey + "/observations", errors.New("database unavailable"), 500}, + } { + called := false + router := RoutesRouter(stubReader{getRouteEvidence: func(_ context.Context, iata, key string, q api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + called = true + if iata != "YOW" || key != evidenceKey || q.Limit != 50 { + t.Fatal("wrong reader scope") + } + return &api.RouteEvidence{Page: api.Page[api.RouteObservation]{Items: []api.RouteObservation{}}}, tc.err + }}) + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", tc.path, nil)) + if w.Code != tc.status { + t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String()) + } + if tc.status == 400 && called { + t.Fatal("invalid request reached database") + } + } +} diff --git a/internal/api/handlers/routes.go b/internal/api/handlers/routes.go index a5bdd76..77f1117 100644 --- a/internal/api/handlers/routes.go +++ b/internal/api/handlers/routes.go @@ -22,6 +22,7 @@ func RoutesRouter(reader api.Reader) http.Handler { r.Get("/", listKnownRoutes(reader)) r.Get("/cross", searchCrossIATARoutes(reader)) r.Get("/search", searchKnownRoutes(reader)) + r.Get("/{iata}/{pathKey}/observations", getRouteEvidence(reader)) return r } diff --git a/internal/api/handlers/scopes.go b/internal/api/handlers/scopes.go index 5267ed1..c86de6f 100644 --- a/internal/api/handlers/scopes.go +++ b/internal/api/handlers/scopes.go @@ -24,6 +24,7 @@ func ScopesRouter(reader api.Reader) http.Handler { // listScopes godoc // // @Summary List transport scopes +// @Description The unfiltered list includes stored imported names after an importer is disabled or a source is removed. Those historical identities are retained indefinitely; listing them does not establish observed traffic or current source membership. // @Tags Scopes // @Produce json // @Param iatas query string false "Filter by IATA code(s), comma-separated" diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 42200c1..f9f885a 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -16,6 +16,7 @@ import ( // Unset fields return zero values. Use it for both validation tests // (leave all fields nil) and happy path tests (set only what you need). type stubReader struct { + getRouteEvidence func(context.Context, string, string, api.RouteEvidenceQuery) (*api.RouteEvidence, error) getObserverComparison func(context.Context, uuid.UUID, uuid.UUID, time.Time, time.Time, []string) (*api.ObserverComparison, error) listIATAs func(ctx context.Context) ([]api.IATA, error) getIATA func(ctx context.Context, iata string) (*api.IATA, error) @@ -32,7 +33,7 @@ type stubReader struct { getObserver func(ctx context.Context, observerID uuid.UUID) (*api.Observer, error) getObserverTelemetry func(ctx context.Context, observerID uuid.UUID, since, until time.Time, afterID int64) (*api.ObserverTelemetry, error) getObserverTelemetryBucketed func(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]api.ObserverTelemetryPoint, error) - getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) + getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) getObserverScopes func(ctx context.Context, observerID uuid.UUID) ([]string, error) listObserverAdverts func(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) listNodes func(ctx context.Context, nodeType int16, iatas []string, supportsMultibytePaths, supportsMultibyteTraces *bool, pubkey []byte, pubkeyPrefix, name, scope string, cursor int64, limit int32, includeNeighbors bool) (api.Page[api.NodeSummary], error) @@ -186,9 +187,9 @@ func (s stubReader) GetObserverTelemetryBucketed(ctx context.Context, observerID return nil, nil } -func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { +func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { if s.getObserverActivity != nil { - return s.getObserverActivity(ctx, observerID, window, interval) + return s.getObserverActivity(ctx, observerID, window, interval, until) } return nil, nil } @@ -409,3 +410,10 @@ func (s stubReader) GetNodesByIDs(ctx context.Context, ids []uuid.UUID) (map[uui } return nil, nil } + +func (s stubReader) GetRouteEvidence(ctx context.Context, iata, key string, q api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + if s.getRouteEvidence != nil { + return s.getRouteEvidence(ctx, iata, key, q) + } + return nil, nil +} diff --git a/internal/api/observers.go b/internal/api/observers.go index 206387b..2da680c 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -44,8 +44,8 @@ type Observer struct { LastStatusAt *int64 `json:"lastStatusAt,omitempty"` // epoch ms FirstSeen int64 `json:"firstSeen"` // epoch ms LastSeen int64 `json:"lastSeen"` // epoch ms - ObservationCount int64 `json:"observationCount"` - Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on + ObservationCount int64 `json:"observationCount"` // legacy cumulative presence counter; includes non-packet events + Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on } // ObserverTelemetryPoint is a single telemetry snapshot for an observer. @@ -88,8 +88,25 @@ type ObserverActivityPoint struct { RSSIAvg *float32 `json:"rssiAvg"` } +// ObserverActivitySummary describes stored packet records, never MQTT presence events. +// Freshness and lastCompleteHour are measured at generatedAt, even with an explicit until; +// only recordedPackets follows the selected activity window. +type ObserverActivitySummary struct { + RecordedPackets int64 `json:"recordedPackets"` // stored observations within windowStart/windowEnd + LastCompleteHour int64 `json:"lastCompleteHour"` + LastCompleteHourStart int64 `json:"lastCompleteHourStart"` + LastCompleteHourEnd int64 `json:"lastCompleteHourEnd"` + LatestRecordedAt *int64 `json:"latestRecordedAt"` +} + // ObserverActivity is the per-observer heard-activity response. type ObserverActivity struct { + WindowStart int64 `json:"windowStart"` // inclusive complete-bucket start, epoch ms + WindowEnd int64 `json:"windowEnd"` // exclusive end, epoch ms; live requests include the current partial bucket + GeneratedAt int64 `json:"generatedAt"` // response computation time, not proof of continuous coverage + Source string `json:"source"` // raw or hourly; missing records do not prove an outage + Summary *ObserverActivitySummary `json:"summary,omitempty"` + Range string `json:"range"` Interval string `json:"interval"` Radio *ObserverActivityRadio `json:"radio"` diff --git a/internal/api/reader.go b/internal/api/reader.go index 8aa3f1a..d46ec2d 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -23,6 +23,8 @@ type Page[T any] struct { } type Reader interface { + // GetRouteEvidence returns the full saved route and exact-prefix retained ordinary reports. + GetRouteEvidence(ctx context.Context, iata, pathKey string, query RouteEvidenceQuery) (*RouteEvidence, error) // ListIATAs returns all known IATA codes with display name and coordinates. // IATAs are auto-created on first packet arrival from that location. ListIATAs(ctx context.Context) ([]IATA, error) @@ -100,7 +102,7 @@ type Reader interface { // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. // interval >= 1h is served from the hourly rollup. Returns pgx.ErrNoRows for an unknown observer. // Range and Interval on the result are left empty for the handler to fill. - GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*ObserverActivity, error) + GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*ObserverActivity, error) // GetObserverScopes returns the names of all transport scopes an observer has // been seen forwarding packets for, ordered alphabetically. diff --git a/internal/api/route_evidence.go b/internal/api/route_evidence.go new file mode 100644 index 0000000..cadc056 --- /dev/null +++ b/internal/api/route_evidence.go @@ -0,0 +1,110 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import ( + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +const MaxRouteEvidenceWindow = 30 * 24 * time.Hour + +var ErrRouteEvidenceInput = errors.New("invalid route evidence request") + +// RouteObservation is a retained report reference, not a packet body or proof of hop identity. +type RouteObservation struct { + ID int64 `json:"id"` + PacketHash string `json:"packetHash"` + ObserverID uuid.UUID `json:"observerId"` + ObserverName *string `json:"observerName,omitempty"` + HeardAt int64 `json:"heardAt"` + PayloadType int16 `json:"payloadType"` + PayloadTypeName string `json:"payloadTypeName"` + RSSI *int16 `json:"rssi,omitempty"` + SNR *float32 `json:"snr,omitempty"` +} + +// RouteEvidence uses nextPageCursor rather than the lossy legacy numeric cursor. +type RouteEvidence struct { + Page[RouteObservation] + Route KnownRoute `json:"route"` + WindowStart int64 `json:"windowStart"` + WindowEnd int64 `json:"windowEnd"` + GeneratedAt int64 `json:"generatedAt"` + MatchType string `json:"matchType"` + MatchAvailable bool `json:"matchAvailable"` + HashSize int16 `json:"hashSize"` + PathBytes string `json:"pathBytes"` + NextPageCursor *string `json:"nextPageCursor,omitempty"` +} + +type RouteEvidenceQuery struct { + Since time.Time + Until time.Time + Cursor *RouteEvidenceCursor + Limit int32 +} + +// RouteEvidenceCursor binds the precise (heard_at DESC,id DESC) boundary to its route/window. +type RouteEvidenceCursor struct { + IATA string + PathKey string + Since time.Time + Until time.Time + HeardAt time.Time + ID int64 +} + +func ValidRouteEvidenceKey(iata, key string) bool { + if len(iata) != 3 || len(key) != 32 { + return false + } + for _, c := range iata { + if !(c >= 'A' && c <= 'Z') && !(c >= '0' && c <= '9') { + return false + } + } + _, err := hex.DecodeString(key) + return err == nil && key == strings.ToLower(key) +} + +func ValidRouteEvidenceWindow(since, until time.Time) bool { + return !since.Before(time.Unix(0, 0)) && until.After(since) && until.Sub(since) <= MaxRouteEvidenceWindow +} + +func (c RouteEvidenceCursor) String() string { + return fmt.Sprintf("v1:%s:%s:%d:%d:%d:%d", c.IATA, c.PathKey, c.Since.UnixMilli(), c.Until.UnixMilli(), c.HeardAt.UnixMicro(), c.ID) +} + +func ParseRouteEvidenceCursor(raw string) (*RouteEvidenceCursor, error) { + if len(raw) > 200 { + return nil, ErrRouteEvidenceInput + } + p := strings.Split(raw, ":") + if len(p) != 7 || p[0] != "v1" || !ValidRouteEvidenceKey(p[1], p[2]) { + return nil, ErrRouteEvidenceInput + } + values := [4]int64{} + for i, s := range p[3:] { + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || n < 0 || strconv.FormatInt(n, 10) != s { + return nil, ErrRouteEvidenceInput + } + values[i] = n + } + // Bound before converting; this also keeps millisecond/microsecond comparisons safe. + if values[0] > 253402300799999 || values[1] > 253402300799999 || values[2] > 253402300799999999 || values[3] <= 0 { + return nil, ErrRouteEvidenceInput + } + c := &RouteEvidenceCursor{IATA: p[1], PathKey: p[2], Since: time.UnixMilli(values[0]).UTC(), Until: time.UnixMilli(values[1]).UTC(), HeardAt: time.UnixMicro(values[2]).UTC(), ID: values[3]} + if !ValidRouteEvidenceWindow(c.Since, c.Until) || c.HeardAt.Before(c.Since) || !c.HeardAt.Before(c.Until) { + return nil, ErrRouteEvidenceInput + } + return c, nil +} diff --git a/internal/api/route_evidence_test.go b/internal/api/route_evidence_test.go new file mode 100644 index 0000000..42c9a73 --- /dev/null +++ b/internal/api/route_evidence_test.go @@ -0,0 +1,25 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package api + +import ( + "strings" + "testing" + "time" +) + +func TestRouteEvidenceCursor(t *testing.T) { + c := RouteEvidenceCursor{IATA: "YOW", PathKey: "f097439148601d9f3291c474f82fa64c", Since: time.UnixMilli(1000).UTC(), Until: time.UnixMilli(3000).UTC(), HeardAt: time.UnixMicro(2000123).UTC(), ID: 9223372036854775806} + got, err := ParseRouteEvidenceCursor(c.String()) + if err != nil || *got != c { + t.Fatalf("cursor round-trip: %+v %v", got, err) + } + if got.HeardAt.Nanosecond() != 123000 { + t.Fatal("lost microsecond precision") + } + for _, bad := range []string{"", strings.Repeat("x", 300), "v2:YOW:x:1:2:3:4", strings.Replace(c.String(), "YOW", "TOOLONG", 1), strings.Replace(c.String(), c.PathKey, "bad", 1), strings.Replace(c.String(), "2000123", "4000000", 1), strings.TrimSuffix(c.String(), "9223372036854775806") + "0"} { + if _, err := ParseRouteEvidenceCursor(bad); err == nil { + t.Errorf("accepted %q", bad) + } + } +} diff --git a/internal/api/routes.go b/internal/api/routes.go index 3bc7ece..3f1bee1 100644 --- a/internal/api/routes.go +++ b/internal/api/routes.go @@ -16,6 +16,7 @@ type RouteHop struct { // have been confirmed as high confidence. type KnownRoute struct { ID int64 `json:"id"` + PathKey string `json:"pathKey,omitempty"` // stable identity within this IATA; use for route evidence links IATA string `json:"iata"` HopCount int32 `json:"hopCount"` Hops []RouteHop `json:"hops"` diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index e58f054..8184a17 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -148,7 +148,7 @@ func (s *stubReader) GetObserverTelemetryBucketed(_ context.Context, _ uuid.UUID return nil, nil } -func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { +func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, nil } @@ -418,3 +418,7 @@ func TestCachedReader_InvalidateObserver(t *testing.T) { t.Error("expected observer scopes key to be deleted") } } + +func (s *stubReader) GetRouteEvidence(_ context.Context, _, _ string, _ api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + return nil, nil +} diff --git a/internal/cache/observer_window_test.go b/internal/cache/observer_window_test.go new file mode 100644 index 0000000..512e580 --- /dev/null +++ b/internal/cache/observer_window_test.go @@ -0,0 +1,47 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" +) + +type activityWindowReader struct { + stubReader + ends []time.Time +} + +func (r *activityWindowReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, until time.Time) (*api.ObserverActivity, error) { + r.ends = append(r.ends, until) + return &api.ObserverActivity{WindowEnd: until.UnixMilli()}, nil +} + +func TestObserverActivityCacheUsesEffectiveWindow(t *testing.T) { + client, redis := newTestClient(t) + inner := &activityWindowReader{} + reader := &CachedReader{inner: inner, c: client} + id := uuid.New() + anchor := time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC) + for _, offset := range []time.Duration{time.Second, time.Minute, 59 * time.Minute} { + if _, err := reader.GetObserverActivity(context.Background(), id, 24*time.Hour, time.Hour, anchor.Add(offset)); err != nil { + t.Fatal(err) + } + } + if len(inner.ends) != 1 || !inner.ends[0].Equal(anchor) || len(redis.Keys()) != 1 { + t.Fatalf("equivalent fixed windows missed the cache: %v, %v", inner.ends, redis.Keys()) + } + for _, until := range []time.Time{anchor.Add(time.Hour), {}, {}} { + if _, err := reader.GetObserverActivity(context.Background(), id, 24*time.Hour, time.Hour, until); err != nil { + t.Fatal(err) + } + } + if len(inner.ends) != 3 || !inner.ends[2].IsZero() || len(redis.Keys()) != 3 { + t.Fatalf("distinct/live windows aliased: %v", inner.ends) + } +} diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 65551da..515adc9 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -80,6 +80,11 @@ func NewCachedReader(inner api.Reader, c *Client, ttl CacheTTLs) api.Reader { } } +// InvalidateScopeNames makes newly committed catalogue names available to filters. +func (cr *CachedReader) InvalidateScopeNames(ctx context.Context) { + cr.c.del(ctx, keyScopeNames) +} + // InvalidateNode removes the cached entries for a node by UUID. // Should be called from the ingest path after a node upsert. func (cr *CachedReader) InvalidateNode(ctx context.Context, nodeID uuid.UUID) { @@ -372,10 +377,13 @@ func (cr *CachedReader) GetObserverTelemetryBucketed(ctx context.Context, observ } // GetObserverActivity implements [api.Reader]. -func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { - key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() +func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { + if !until.IsZero() { + until = until.UTC().Truncate(interval) + } + key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano) return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) { - return cr.inner.GetObserverActivity(ctx, observerID, window, interval) + return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until) }) } @@ -464,6 +472,11 @@ func (cr *CachedReader) SearchKnownRoutes(ctx context.Context, iata, fromHash, t return cr.inner.SearchKnownRoutes(ctx, iata, fromHash, toHash) } +// Precise cursor/window combinations are intentionally passed through, like route lists. +func (cr *CachedReader) GetRouteEvidence(ctx context.Context, iata, key string, query api.RouteEvidenceQuery) (*api.RouteEvidence, error) { + return cr.inner.GetRouteEvidence(ctx, iata, key, query) +} + // SearchCrossIATARoutes implements [api.Reader]. func (cr *CachedReader) SearchCrossIATARoutes(ctx context.Context, fromHash, fromIATA, toHash, toIATA string) ([]api.CrossIATARoute, error) { return cr.inner.SearchCrossIATARoutes(ctx, fromHash, fromIATA, toHash, toIATA) diff --git a/internal/config/config.go b/internal/config/config.go index b3c6cd3..b8b8fff 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -33,6 +33,7 @@ type Config struct { Routes RoutesConfig `yaml:"routes"` Ingest IngestFilterConfig `yaml:"ingest"` Scopes []ScopeConfig `yaml:"scopes"` + MeshMapper MeshMapperConfig `yaml:"meshmapper"` Cache CacheConfig `yaml:"cache"` CORS CORSConfig `yaml:"cors"` RateLimit RateLimitConfig `yaml:"ratelimit"` @@ -408,6 +409,9 @@ func Load(path string) (*Config, error) { } } configDir := filepath.Dir(path) + if err := cfg.validateMeshMapper(); err != nil { + return nil, err + } for iata, details := range cfg.IATAs { if details.BorderFile != "" && !filepath.IsAbs(details.BorderFile) { details.BorderFile = filepath.Join(configDir, details.BorderFile) diff --git a/internal/config/meshmapper.go b/internal/config/meshmapper.go new file mode 100644 index 0000000..5f36774 --- /dev/null +++ b/internal/config/meshmapper.go @@ -0,0 +1,64 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "fmt" + "net/url" + "regexp" + "time" +) + +// MaxScopeSources bounds both network refresh work and imported matching keys. +const MaxScopeSources = 16 + +type MeshMapperConfig struct { + Scopes MeshMapperScopesConfig `yaml:"scopes"` +} + +// MeshMapperScopesConfig augments, but never replaces, the manual scopes list. +type MeshMapperScopesConfig struct { + Enabled bool `yaml:"enabled"` + Sources map[string]string `yaml:"sources"` // configured IATA -> published regional endpoint + RefreshInterval duration `yaml:"refresh_interval"` +} + +func (c MeshMapperScopesConfig) Interval() time.Duration { + if c.RefreshInterval.Duration == 0 { + return time.Hour + } + return c.RefreshInterval.Duration +} + +var meshMapperHost = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*\.meshmapper\.net$`) + +func (c *Config) validateMeshMapper() error { + s := c.MeshMapper.Scopes + if !s.Enabled { + return nil + } + if len(s.Sources) == 0 || len(s.Sources) > MaxScopeSources { + return fmt.Errorf("meshmapper.scopes.sources must contain 1-%d regional endpoints", MaxScopeSources) + } + if s.Interval() < 5*time.Minute || s.Interval() > 24*time.Hour { + return fmt.Errorf("meshmapper.scopes.refresh_interval must be between 5m and 24h") + } + configured := map[string]bool{} + for _, region := range c.Regions { + for _, iata := range region.IATAs { + configured[iata] = true + } + } + for iata, endpoint := range s.Sources { + if !configured[iata] { + return fmt.Errorf("meshmapper.scopes source %q must belong to a configured region", iata) + } + u, err := url.Parse(endpoint) + if err != nil || u.Scheme != "https" || u.User != nil || !meshMapperHost.MatchString(u.Host) || + u.Path != "/get_scopes.php" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return fmt.Errorf("meshmapper.scopes source %q must be a published https://.meshmapper.net/get_scopes.php endpoint without credentials or parameters", iata) + } + } + return nil +} diff --git a/internal/config/meshmapper_test.go b/internal/config/meshmapper_test.go new file mode 100644 index 0000000..56a5634 --- /dev/null +++ b/internal/config/meshmapper_test.go @@ -0,0 +1,49 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestMeshMapperConfig(t *testing.T) { + valid := "regions:\n - slug: ottawa\n iatas: [YOW]\nmeshmapper:\n scopes:\n enabled: true\n sources:\n YOW: https://yow.meshmapper.net/get_scopes.php\n" + for _, tc := range []struct { + name, text string + bad bool + }{ + {"valid", valid, false}, + {"unknown region", strings.Replace(valid, "iatas: [YOW]", "iatas: [YVR]", 1), true}, + {"http", strings.Replace(valid, "https:", "http:", 1), true}, + {"private host", strings.Replace(valid, "yow.meshmapper.net", "127.0.0.1", 1), true}, + {"credentials", strings.Replace(valid, "https://", "https://secret@", 1), true}, + {"query", strings.Replace(valid, "get_scopes.php", "get_scopes.php?key=secret", 1), true}, + {"non API", strings.Replace(valid, "get_scopes.php", "index.php", 1), true}, + {"fast", valid + " refresh_interval: 1s\n", true}, + {"negative", valid + " refresh_interval: -1h\n", true}, + {"disabled", strings.Replace(valid, "enabled: true", "enabled: false", 1), false}, + } { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(p, []byte(tc.text), 0600); err != nil { + t.Fatal(err) + } + cfg, err := Load(p) + if (err != nil) != tc.bad { + t.Fatalf("error=%v", err) + } + if err == nil && cfg.MeshMapper.Scopes.Interval() != time.Hour { + t.Fatal("wrong default") + } + }) + } + cfg := Config{MeshMapper: MeshMapperConfig{Scopes: MeshMapperScopesConfig{Enabled: true}}} + if err := cfg.validateMeshMapper(); err == nil { + t.Fatal("empty sources accepted") + } +} diff --git a/internal/config/seed.go b/internal/config/seed.go index 53ea371..a419573 100644 --- a/internal/config/seed.go +++ b/internal/config/seed.go @@ -5,12 +5,12 @@ package config import ( "context" - "crypto/sha256" "encoding/json" "fmt" "log/slog" "os" - "strings" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) // Seeder is the database interface required to seed config data on startup. @@ -63,29 +63,10 @@ func Seed(ctx context.Context, cfg *Config, db Seeder) error { } // Transport Codes for _, s := range cfg.Scopes { - name := normalizeScopeName(s.Name) - key := deriveScopeKey(name) - h := sha256.Sum256(key) - fingerprint := h[:8] - if err := db.UpsertTransportScope(ctx, name, "", key, fingerprint); err != nil { + entry := scopestore.FromName(s.Name) + if err := db.UpsertTransportScope(ctx, entry.Name, "", entry.TransportKey, entry.KeyFingerprint); err != nil { return err } } return nil } - -// normalizeScopeName ensures the scope name has a # or $ prefix. -// Plain names get # prepended: "bc" → "#bc". -func normalizeScopeName(name string) string { - if strings.HasPrefix(name, "#") || strings.HasPrefix(name, "$") { - return name - } - return "#" + name -} - -// deriveScopeKey derives the 16-byte transport key from a normalized scope name. -// key = SHA256(name)[:16] -func deriveScopeKey(name string) []byte { - h := sha256.Sum256([]byte(name)) - return h[:16] -} diff --git a/internal/config/seed_test.go b/internal/config/seed_test.go index 4f2fa0c..04c8e63 100644 --- a/internal/config/seed_test.go +++ b/internal/config/seed_test.go @@ -10,6 +10,8 @@ import ( "errors" "os" "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" ) type stubSeeder struct { @@ -198,39 +200,39 @@ func TestSeed_DBError(t *testing.T) { } func TestNormalizeScopeName_WithHash(t *testing.T) { - if normalizeScopeName("#bc") != "#bc" { + if scopestore.FromName("#bc").Name != "#bc" { t.Error("expected #bc unchanged") } } func TestNormalizeScopeName_WithDollar(t *testing.T) { - if normalizeScopeName("$bc") != "$bc" { + if scopestore.FromName("$bc").Name != "$bc" { t.Error("expected $bc unchanged") } } func TestNormalizeScopeName_WithoutPrefix(t *testing.T) { - if normalizeScopeName("bc") != "#bc" { + if scopestore.FromName("bc").Name != "#bc" { t.Error("expected bc to become #bc") } } func TestNormalizeScopeName_Empty(t *testing.T) { - if normalizeScopeName("") != "#" { + if scopestore.FromName("").Name != "#" { t.Error("expected empty string to become #") } } func TestDeriveScopeKey_Length(t *testing.T) { - key := deriveScopeKey("#bc") + key := scopestore.FromName("#bc").TransportKey if len(key) != 16 { t.Errorf("expected 16 bytes, got %d", len(key)) } } func TestDeriveScopeKey_Deterministic(t *testing.T) { - a := deriveScopeKey("#bc") - b := deriveScopeKey("#bc") + a := scopestore.FromName("#bc").TransportKey + b := scopestore.FromName("#bc").TransportKey if hex.EncodeToString(a) != hex.EncodeToString(b) { t.Error("expected same key for same input") } @@ -238,7 +240,7 @@ func TestDeriveScopeKey_Deterministic(t *testing.T) { func TestDeriveScopeKey_KnownValue(t *testing.T) { // SHA256("#bc")[:16] — pin the exact derivation so changes are caught - key := deriveScopeKey("#bc") + key := scopestore.FromName("#bc").TransportKey got := hex.EncodeToString(key) // generate this once: echo -n "#bc" | sha256sum | cut -c1-32 const want = "84509cfe73d94f7f6a8299e6bcdb8a3c" @@ -248,8 +250,8 @@ func TestDeriveScopeKey_KnownValue(t *testing.T) { } func TestDeriveScopeKey_DifferentInputs(t *testing.T) { - a := deriveScopeKey("#bc") - b := deriveScopeKey("#other") + a := scopestore.FromName("#bc").TransportKey + b := scopestore.FromName("#other").TransportKey if hex.EncodeToString(a) == hex.EncodeToString(b) { t.Error("expected different keys for different inputs") } diff --git a/internal/ingest/backfill.go b/internal/ingest/backfill.go index f402a4d..ecb7ebb 100644 --- a/internal/ingest/backfill.go +++ b/internal/ingest/backfill.go @@ -29,9 +29,8 @@ type DecryptGroupTextResult struct { ChannelID int ChannelHash []byte Entry keystore.Entry - // NewMessage is false if InsertChannelMessage found the message already existed - // (packet_hash is unique per message) -- e.g. re-running the backfill twice. - NewMessage bool + // Message is nil when the packet already has a stored channel message. + Message *InsertedChannelMessage } // DecryptGroupText attempts to decrypt a GRP_TXT payload against keys and, on success, @@ -77,7 +76,7 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa if err != nil { return nil, fmt.Errorf("insert channel message: %w", err) } - if newMsg { + if newMsg != nil { // Non-fatal: the message is stored either way, so just log and continue -- matches // the live ingest path's existing behavior of not treating this as a hard failure. if err := db.SetPacketDecrypted(ctx, packetHash); err != nil { @@ -89,7 +88,7 @@ func DecryptGroupText(ctx context.Context, db DB, keys ChannelKeyStore, packetHa ChannelID: channelID, ChannelHash: channelHashBytes, Entry: usedEntry, - NewMessage: newMsg, + Message: newMsg, }, nil } @@ -115,7 +114,7 @@ func BackfillChannelMessages(ctx context.Context, db DB, keys ChannelKeyStore) ( slog.Error(fmt.Sprintf("ingest: backfill: decrypt failed for packet %s", hex.EncodeToString(p.PacketHash)), "component", "ingest", "error", err) continue } - if result != nil && result.NewMessage { + if result != nil && result.Message != nil { decrypted++ } } diff --git a/internal/ingest/backfill_test.go b/internal/ingest/backfill_test.go index 748332e..1bbeeef 100644 --- a/internal/ingest/backfill_test.go +++ b/internal/ingest/backfill_test.go @@ -49,8 +49,8 @@ func TestDecryptGroupText_Success(t *testing.T) { if result.Payload.Sender != "ded" || result.Payload.Text != "hello" { t.Errorf("expected decrypted sender=ded text=hello, got sender=%s text=%s", result.Payload.Sender, result.Payload.Text) } - if !result.NewMessage { - t.Error("expected NewMessage true when InsertChannelMessage reports a new insert") + if result.Message == nil { + t.Error("expected a message when InsertChannelMessage reports a new insert") } if db.upsertChannelCalls != 1 { t.Errorf("expected UpsertChannel to be called once, got %d", db.upsertChannelCalls) diff --git a/internal/ingest/channel_scope_test.go b/internal/ingest/channel_scope_test.go new file mode 100644 index 0000000..3ee6933 --- /dev/null +++ b/internal/ingest/channel_scope_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/MeshCore-Beacon/beacon-server/internal/hub" + "github.com/MeshCore-Beacon/beacon-server/internal/keystore" + "github.com/meshcore-go/meshcore-go" +) + +type scopeMessageDB struct { + *stubDB + next *InsertedChannelMessage +} + +func (s *scopeMessageDB) InsertChannelMessage(context.Context, InsertChannelMessageParams) (*InsertedChannelMessage, error) { + value := s.next + s.next = nil + return value, nil +} + +func TestChannelMessageScopeLive(t *testing.T) { + for _, status := range []string{"matched", "unscoped", "unknown", "unavailable"} { + t.Run(status, func(t *testing.T) { + w, base := newTestWorker() + stored := &InsertedChannelMessage{ID: 42, ScopeStatus: api.ChannelScopeStatus(status)} + name := "#stored" + if status == "matched" { + stored.Scope = &name + } + w.db = &scopeMessageDB{stubDB: base, next: stored} + psk := make([]byte, 16) + w.keys = &mapKeys{entries: map[byte][]keystore.Entry{0x11: {{Key: psk, Fingerprint: []byte{1}, Name: "fixture"}}}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + go w.hub.Run() + client := w.hub.NewClient() + defer w.hub.Remove(client) + w.hub.AddScope(client, "scope", hub.Scope{Events: []hub.EventType{hub.EventChannelMessage, hub.EventObserverStatus}}) + waitForSummarySubscriber(t, ctx, w.hub, client) + packet := &meshcore.Packet{Header: meshcore.MakeHeader(meshcore.RouteTypeTransportFlood, meshcore.PayloadTypeGrpTxt, 0), Payload: encryptedGroupText(t, 0x11, psk, "Fixture", "test")} + latest := "#different-later-reception" + w.handlePayloadTypeSideEffects(ctx, packet, "YOW", []byte{1}, RadioSettings{}, nil, &latest, nil, 0) + select { + case event := <-client.Send: + var got channelMessageEvent + if err := json.Unmarshal(event.Payload, &got); err != nil { + t.Fatal(err) + } + if got.ID != 42 || got.ScopeStatus != stored.ScopeStatus || (got.Scope == nil) != (stored.Scope == nil) || (got.Scope != nil && *got.Scope != name) { + t.Fatal("live scope did not use stored evidence", got) + } + case <-ctx.Done(): + t.Fatal("no live message") + } + w.handlePayloadTypeSideEffects(ctx, packet, "YOW", []byte{1}, RadioSettings{}, nil, &latest, nil, 0) + w.hub.Broadcast(hub.Event{Type: hub.EventObserverStatus}) + select { + case event := <-client.Send: + if event.Type == hub.EventChannelMessage { + t.Fatal("duplicate broadcast") + } + case <-ctx.Done(): + t.Fatal("no barrier") + } + }) + } +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 998c205..289a0de 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -89,7 +89,7 @@ type DB interface { UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) // UpsertObserverBroker records that this observer was seen on brokerName. - UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error + UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error // UpsertIATA auto-creates an iata_codes row if it doesn't exist yet. UpsertIATA(ctx context.Context, iata string) error @@ -131,8 +131,8 @@ type DB interface { // UpsertNodeShortID upserts a node_short_ids row for path resolution. UpsertNodeShortID(ctx context.Context, nodeID uuid.UUID, iata string, prefix4 []byte) error - // InsertChannelMessage stores a decrypted group text message. Returns insert success and an error. - InsertChannelMessage(ctx context.Context, m InsertChannelMessageParams) (bool, error) + // InsertChannelMessage stores a decrypted group text message. Returns the inserted message, nil for a duplicate, and an error. + InsertChannelMessage(ctx context.Context, m InsertChannelMessageParams) (*InsertedChannelMessage, error) // UpdateObserverStatus updates the observer row from a /status message. Returns the OberserID // and any error. diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index ea206c5..248d3a4 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -201,8 +201,10 @@ func (s *stubDB) SetNodeCapability(_ context.Context, nodeID uuid.UUID, paths, t func (s *stubDB) UpsertObserver(_ context.Context, _ []byte) (uuid.UUID, string, error) { return uuid.Nil, "", nil } -func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string) error { return nil } -func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } +func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, _ bool) error { + return nil +} +func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } func (s *stubDB) UpsertPacket(_ context.Context, _ UpsertPacketParams) (bool, error) { return false, nil } @@ -229,8 +231,11 @@ func (s *stubDB) GetNodesByIDs(_ context.Context, _ []uuid.UUID) (map[uuid.UUID] return nil, nil } -func (s *stubDB) InsertChannelMessage(_ context.Context, _ InsertChannelMessageParams) (bool, error) { - return s.insertChannelMessageResult, nil +func (s *stubDB) InsertChannelMessage(_ context.Context, _ InsertChannelMessageParams) (*InsertedChannelMessage, error) { + if !s.insertChannelMessageResult { + return nil, nil + } + return &InsertedChannelMessage{ID: 1, ScopeStatus: "unavailable"}, nil } func (s *stubDB) UpdateObserverStatus(_ context.Context, _ UpdateObserverStatusParams) (uuid.UUID, error) { diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go index f96a32a..3def0c9 100644 --- a/internal/ingest/neighbors.go +++ b/internal/ingest/neighbors.go @@ -48,7 +48,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra w.log.Error(fmt.Sprintf("db: upsert observer failed in neighbors from %s", pubkeyHex), "error", err) return } - if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName); err != nil { + if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName, false); err != nil { w.log.Error(fmt.Sprintf("db: upsert observer broker failed in neighbors from %s", pubkeyHex), "error", err) } diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 92f3da7..1a99b5e 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -11,11 +11,13 @@ import ( "encoding/hex" "encoding/json" "fmt" + "slices" "strings" "time" "github.com/MeshCore-Beacon/beacon-server/internal/api" "github.com/MeshCore-Beacon/beacon-server/internal/lora" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" "github.com/google/uuid" "github.com/meshcore-go/meshcore-go" ) @@ -324,7 +326,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ w.log.Error(fmt.Sprintf("db: upsert observer failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } - err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName) + err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, true) if err != nil { w.log.Error(fmt.Sprintf("db: update observer broker failed with packet from %s/%s", iata, pubkeyHex), "error", err) return @@ -694,14 +696,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ var matchedScope *string if packet.RouteType() == meshcore.RouteTypeTransportFlood || packet.RouteType() == meshcore.RouteTypeTransportDirect { - for _, entry := range w.scopes.Entries() { - code := computeTransportCode(entry.TransportKey, packet.PayloadType(), packet.Payload) - if code == packet.TransportCode1 { - s := entry.Name - matchedScope = &s - break - } - } + matchedScope = matchTransportScope(w.scopes.Entries(), iata, packet.PayloadType(), packet.Payload, packet.TransportCode1) } var scopeID *int32 if matchedScope != nil { @@ -930,6 +925,32 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ } } +// matchTransportScope preserves manual precedence; imported collisions remain unresolved. +func matchTransportScope(entries []scopestore.Entry, iata string, payloadType uint8, payload []byte, code uint16) *string { + var matched *string + ambiguous := false + for _, entry := range entries { + if entry.IATAs != nil && !slices.Contains(entry.IATAs, iata) { + continue + } + if computeTransportCode(entry.TransportKey, payloadType, payload) != code { + continue + } + name := entry.Name + if entry.IATAs == nil { + return &name + } + if matched != nil && *matched != name { + ambiguous = true + } + matched = &name + } + if ambiguous { + return nil + } + return matched +} + // computeTransportCode derives transport_code_1 from a transport key and packet payload. // code = HMAC-SHA256(key, payload_type_byte || payload)[0:2] as little-endian uint16. // Coerces reserved values 0x0000 → 0x0001 and 0xFFFF → 0xFFFE per §2.4. diff --git a/internal/ingest/scope_import_test.go b/internal/ingest/scope_import_test.go new file mode 100644 index 0000000..4474f13 --- /dev/null +++ b/internal/ingest/scope_import_test.go @@ -0,0 +1,113 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ingest + +import ( + "context" + "fmt" + "testing" + + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" + "github.com/meshcore-go/meshcore-go" +) + +func TestTransportScopeRegionalAndAmbiguousMatches(t *testing.T) { + entry := scopestore.FromName("yow") + entry.IATAs = []string{"YOW"} + payload := []byte{0xde, 0xad, 0xbe, 0xef} + // Fixed HMAC-SHA256 fixture: SHA256("#yow")[:16], payload type 4. + if got := matchTransportScope([]scopestore.Entry{entry}, "YOW", 4, payload, 21698); got == nil || *got != "#yow" { + t.Fatal(got) + } + if got := matchTransportScope([]scopestore.Entry{entry}, "YVR", 4, payload, 21698); got != nil { + t.Fatal("imported key escaped source region") + } + entry.IATAs = nil + if got := matchTransportScope([]scopestore.Entry{entry}, "YVR", 4, payload, 21698); got == nil { + t.Fatal("manual key was region-restricted") + } + // Distinct keys with the same short code for this payload must remain unknown. + a, b := scopestore.FromName("collision-320"), scopestore.FromName("collision-322") + a.IATAs, b.IATAs = []string{"YOW"}, []string{"YOW"} + for _, entries := range [][]scopestore.Entry{{a, b}, {b, a}} { + if computeTransportCode(entries[0].TransportKey, 4, payload) != 37018 { + t.Fatal("collision fixture changed") + } + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got != nil { + t.Fatal("ambiguous short code resolved", *got) + } + } +} + +type importCaptureDB struct{ *frameCaptureDB } + +func (s *importCaptureDB) GetTransportScopeByName(context.Context, string) (int32, error) { + return 123, nil +} + +func TestImportedScopeFlowsThroughPacketIngest(t *testing.T) { + for _, iata := range []string{"YOW", "YVR"} { + t.Run(iata, func(t *testing.T) { + w, base := newTestWorker() + capture := &importCaptureDB{&frameCaptureDB{stubDB: base}} + w.db = capture + key := scopestore.FromName("yow") + key.IATAs = []string{"YOW"} + store := scopestore.New() + store.Load([]scopestore.Entry{key}) + w.scopes = store + packet := buildGrpTxtPacket(t, 0x1a, make([]byte, 16)) + packet.Header = meshcore.MakeHeader(meshcore.RouteTypeTransportFlood, meshcore.PayloadTypeGrpTxt, 0) + packet.TransportCode1 = computeTransportCode(key.TransportKey, packet.PayloadType(), packet.Payload) + w.handlePacket(context.Background(), iata, "0102", packetEnvelope(t, packet)) + if len(capture.packets) != 1 { + t.Fatal("packet not stored") + } + got := capture.packets[0].ScopeID + if iata == "YOW" && (got == nil || *got != 123) { + t.Fatal("scope not persisted", got) + } + if iata == "YVR" && got != nil { + t.Fatal("scope assigned outside source region", got) + } + }) + } +} + +func BenchmarkTransportScopeCatalogue(b *testing.B) { + for _, count := range []int{0, 64, 1024} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + entries := []scopestore.Entry{scopestore.FromName("manual")} + for n := 0; n < count; n++ { + entry := scopestore.FromName(fmt.Sprintf("scope-%d", n)) + entry.IATAs = []string{fmt.Sprintf("R%02d", n/64)} + entries = append(entries, entry) + } + store := scopestore.New() + store.Load(entries) + payload := make([]byte, 128) + b.ReportAllocs() + for b.Loop() { + matchTransportScope(store.Entries(), "R00", 4, payload, 1234) + } + }) + } +} + +func TestManualScopeWinsTransportCodeCollision(t *testing.T) { + payload := []byte{0xde, 0xad, 0xbe, 0xef} + manual, imported := scopestore.FromName("collision-320"), scopestore.FromName("collision-322") + imported.IATAs = []string{"YOW"} + for _, entries := range [][]scopestore.Entry{{manual, imported}, {imported, manual}, {imported, scopestore.Entry{Name: "ambiguous", TransportKey: imported.TransportKey, IATAs: []string{"YOW"}}, manual}} { + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got == nil || *got != manual.Name { + t.Fatalf("manual match lost to imported collision: %v", got) + } + } + imported.IATAs = nil + for _, entries := range [][]scopestore.Entry{{manual, imported}, {imported, manual}} { + if got := matchTransportScope(entries, "YOW", 4, payload, 37018); got == nil || *got != entries[0].Name { + t.Fatalf("manual-only first-match behavior changed: %v", got) + } + } +} diff --git a/internal/ingest/side_effects.go b/internal/ingest/side_effects.go index 5d5c92c..0913242 100644 --- a/internal/ingest/side_effects.go +++ b/internal/ingest/side_effects.go @@ -37,14 +37,24 @@ type InsertChannelMessageParams struct { SentAt time.Time } +// InsertedChannelMessage is returned only for a new message, using its stored packet evidence. +type InsertedChannelMessage struct { + ID int64 + Scope *string + ScopeStatus api.ChannelScopeStatus +} + // channelMessageEvent is the JSON payload for a channelMessage WS event. type channelMessageEvent struct { - ChannelID int `json:"channelId"` - ChannelHash string `json:"channelHash"` // hex-encoded single byte - PacketHash string `json:"packetHash"` // hex-encoded - SenderName string `json:"senderName"` - Content string `json:"content"` - SentAt int64 `json:"sentAt"` // epoch ms + ID int64 `json:"id"` + ChannelID int `json:"channelId"` + ChannelHash string `json:"channelHash"` // hex-encoded single byte + PacketHash string `json:"packetHash"` // hex-encoded + SenderName string `json:"senderName"` + Content string `json:"content"` + SentAt int64 `json:"sentAt"` // epoch ms + Scope *string `json:"scope"` + ScopeStatus api.ChannelScopeStatus `json:"scopeStatus" enums:"matched,unscoped,unknown,unavailable"` } // nodeUpdateEvent is the JSON payload for a nodeUpdate WS event. @@ -205,8 +215,11 @@ func (w *Worker) handlePayloadTypeSideEffects(ctx context.Context, packet *meshc return } - if result.NewMessage { + if result.Message != nil { evt := channelMessageEvent{ + ID: result.Message.ID, + Scope: result.Message.Scope, + ScopeStatus: result.Message.ScopeStatus, ChannelID: result.ChannelID, ChannelHash: hex.EncodeToString(channelHashBytes), PacketHash: hex.EncodeToString(packetHash), diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 31c8064..3c693db 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -87,7 +87,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) if w.onObserverUpsert != nil { w.onObserverUpsert(ctx, id) } - if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName); err != nil { + if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, false); err != nil { w.log.Error(fmt.Sprintf("db: upsert observer broker failed in status from %s", pubkeyHex), "error", err) } params := UpdateObserverStatusParams{ diff --git a/internal/meshmapper/scopes.go b/internal/meshmapper/scopes.go new file mode 100644 index 0000000..23363e6 --- /dev/null +++ b/internal/meshmapper/scopes.go @@ -0,0 +1,302 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package meshmapper imports the published regional scope catalogue, off the ingest path. +package meshmapper + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "slices" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +const ( + MaxScopes = 64 // per source; at most 16 configured sources (1024 imported identities) + MaxBody = 64 << 10 + PollInterval = 15 * time.Second // one source per tick, at most four requests/minute +) + +// Cache stores source provenance and freshness separately from packet evidence. +type Cache struct { + Payload json.RawMessage + ETag string + CheckedAt, AttemptedAt, NextAttempt time.Time + LastError string +} + +type Store interface { + GetScopeCatalogue(context.Context, string, string) (*Cache, error) + SaveScopeCatalogue(context.Context, string, string, Cache, []scopestore.Entry) error +} + +type source struct { + iata, url string + cache Cache + entries []scopestore.Entry + generated time.Time +} + +// Importer is owned by one background task; ScopeStore synchronizes its consumers. +type Importer struct { + store Store + scopes *scopestore.ScopeStore + manual []scopestore.Entry + sources []source + interval time.Duration + client *http.Client + retryAfter time.Time + onChange func(context.Context) +} + +// SetCacheInvalidator is wired once at startup, before the background task starts. +func (i *Importer) SetCacheInvalidator(fn func(context.Context)) { i.onChange = fn } + +// New restores validated snapshots before ingestion, without making HTTP requests. +func New(ctx context.Context, cfg config.MeshMapperScopesConfig, store Store, scopes *scopestore.ScopeStore, manual []scopestore.Entry) (*Importer, error) { + i := &Importer{store: store, scopes: scopes, manual: manual, interval: cfg.Interval(), client: &http.Client{ + Timeout: 10 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }} + if !cfg.Enabled { + return i, nil + } + keys := make([]string, 0, len(cfg.Sources)) + for key := range cfg.Sources { + keys = append(keys, key) + } + slices.Sort(keys) + for _, key := range keys { + s := source{iata: key, url: cfg.Sources[key]} + cached, err := store.GetScopeCatalogue(ctx, s.iata, s.url) + if err != nil { + return nil, fmt.Errorf("restore scope catalogue %s: %w", key, err) + } + if cached != nil { + s.cache = *cached + if len(cached.Payload) > 0 { + s.entries, s.generated, err = decode(cached.Payload, key) + if err != nil { + s.entries = nil + s.cache.Payload, s.cache.ETag = nil, "" + s.cache.LastError = "invalid saved catalogue; awaiting refresh" + } + } + if cached.LastError == "HTTP 429" && cached.NextAttempt.After(i.retryAfter) { + i.retryAfter = cached.NextAttempt + } + } + i.sources = append(i.sources, s) + i.log(s, "restored") + } + i.publish() + return i, nil +} + +// Refresh checks only one due source. The scheduler serializes calls every 15s. +func (i *Importer) Refresh(ctx context.Context) (err error) { + parent := ctx + defer func() { + if parent.Err() != nil { + err = nil + } + }() + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + now := time.Now().UTC() + if now.Before(i.retryAfter) { + return nil + } + for n := range i.sources { + s := &i.sources[n] + if now.Before(s.cache.NextAttempt) { + continue + } + return i.refresh(ctx, s, now) + } + return nil +} + +func (i *Importer) refresh(ctx context.Context, s *source, now time.Time) error { + update := Cache{AttemptedAt: now, NextAttempt: now.Add(i.interval)} + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil) + if err != nil { + return err + } + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", "Beacon-MeshMapper-Scopes/1") + if len(s.cache.Payload) > 0 && s.cache.ETag != "" { + request.Header.Set("If-None-Match", s.cache.ETag) + } + response, err := i.client.Do(request) + var entries []scopestore.Entry + var generated time.Time + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + update.LastError = "request failed" // do not persist untrusted response bodies or URLs from errors + } else { + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + body, readErr := io.ReadAll(io.LimitReader(response.Body, MaxBody+1)) + if readErr == nil { + entries, generated, readErr = decode(body, s.iata) + } + if readErr != nil { + update.LastError = "invalid response" + } else { + update.Payload = body + } + case http.StatusNotModified: + if len(s.cache.Payload) == 0 || s.cache.ETag == "" { + update.LastError = "304 without cached catalogue" + } + default: + update.LastError = fmt.Sprintf("HTTP %d", response.StatusCode) + if response.StatusCode == http.StatusTooManyRequests || response.StatusCode == http.StatusServiceUnavailable { + // The published contract uses seconds. Ignore invalid/overflowing headers. + if seconds, parseErr := strconv.ParseInt(response.Header.Get("Retry-After"), 10, 64); parseErr == nil && seconds > 0 && seconds <= int64((1<<63-1)/time.Second) { + until := now.Add(time.Duration(seconds) * time.Second) + if until.After(update.NextAttempt) { + update.NextAttempt = until + } + } + } + } + if update.LastError == "" { + etag := response.Header.Get("ETag") + if len(etag) > 256 || strings.ContainsAny(etag, "\r\n") { + update.LastError = "invalid ETag" + update.Payload = nil + entries = nil + } else { + update.ETag = etag + if response.StatusCode == http.StatusNotModified && etag == "" { + update.ETag = s.cache.ETag + } + update.CheckedAt = now + } + } + } + if update.LastError == "HTTP 429" { + i.retryAfter = update.NextAttempt + } + if err := i.store.SaveScopeCatalogue(ctx, s.iata, s.url, update, entries); err != nil { + // Avoid a retry every scheduler tick during a DB outage; never publish uncommitted keys. + s.cache.NextAttempt = update.NextAttempt + return fmt.Errorf("persist scope catalogue %s: %w", s.iata, err) + } + if len(update.Payload) > 0 { + s.cache.Payload = update.Payload + s.entries = entries + s.generated = generated + } + if !update.CheckedAt.IsZero() { + s.cache.CheckedAt = update.CheckedAt + s.cache.ETag = update.ETag + } + s.cache.AttemptedAt = now + s.cache.NextAttempt = update.NextAttempt + s.cache.LastError = update.LastError + if len(update.Payload) > 0 { + i.publish() + if i.onChange != nil { + i.onChange(ctx) + } + } + i.log(*s, "checked") + return nil +} + +func (i *Importer) publish() { + byName := make(map[string]scopestore.Entry, len(i.manual)) + for _, e := range i.manual { + byName[e.Name] = e + } + for _, s := range i.sources { + for _, candidate := range s.entries { + entry, exists := byName[candidate.Name] + if exists && entry.IATAs == nil { + continue + } // manual metadata/keys win + if !exists { + entry = candidate + } + entry.IATAs = append(entry.IATAs, s.iata) + byName[entry.Name] = entry + } + } + names := make([]string, 0, len(byName)) + for name := range byName { + names = append(names, name) + } + slices.Sort(names) + entries := make([]scopestore.Entry, 0, len(names)) + for _, name := range names { + entries = append(entries, byName[name]) + } + i.scopes.Load(entries) +} + +func (i *Importer) log(s source, action string) { + level := slog.LevelInfo + if s.cache.LastError != "" { + level = slog.LevelWarn + } + slog.Log(context.Background(), level, "MeshMapper scopes "+action, "component", "meshmapper.scopes", "iata", s.iata, "source", s.url, + "names", len(s.entries), "generated_at", s.generated, "checked_at", s.cache.CheckedAt, "next_attempt", s.cache.NextAttempt, "last_error", s.cache.LastError) +} + +func decode(body []byte, iata string) ([]scopestore.Entry, time.Time, error) { + var document struct { + GeneratedAt time.Time `json:"generated_at"` + Region string `json:"region"` + Zones []string `json:"zones"` + Repeaters, Scoped *int + Scopes *[]struct { + Name string + Repeaters, Default *int + Monitored, Wardriving *bool + } + } + if len(body) > MaxBody || !utf8.Valid(body) { + return nil, time.Time{}, fmt.Errorf("invalid catalogue size/encoding") + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, time.Time{}, err + } + if document.GeneratedAt.IsZero() || document.Region != iata || len(document.Zones) != 1 || document.Zones[0] != iata || + document.Repeaters == nil || document.Scoped == nil || *document.Scoped < 0 || *document.Repeaters < *document.Scoped || + document.Scopes == nil || len(*document.Scopes) > MaxScopes { + return nil, time.Time{}, fmt.Errorf("invalid regional catalogue") + } + entries := make([]scopestore.Entry, 0, len(*document.Scopes)) + seen := map[string]bool{} + for _, s := range *document.Scopes { + if s.Name == "" || len(s.Name) > 128 || strings.TrimSpace(s.Name) != s.Name || strings.ContainsFunc(s.Name, unicode.IsControl) || + s.Repeaters == nil || s.Default == nil || s.Monitored == nil || s.Wardriving == nil || + *s.Default < 0 || *s.Repeaters < *s.Default || *s.Repeaters > *document.Scoped { + return nil, time.Time{}, fmt.Errorf("invalid scope entry") + } + entry := scopestore.FromName(s.Name) + if len(entry.Name) < 2 || seen[entry.Name] { + return nil, time.Time{}, fmt.Errorf("empty or duplicate scope name") + } + seen[entry.Name] = true + entries = append(entries, entry) + } + return entries, document.GeneratedAt, nil +} diff --git a/internal/meshmapper/scopes_test.go b/internal/meshmapper/scopes_test.go new file mode 100644 index 0000000..858ea58 --- /dev/null +++ b/internal/meshmapper/scopes_test.go @@ -0,0 +1,280 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package meshmapper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/config" + "github.com/MeshCore-Beacon/beacon-server/internal/scopestore" +) + +func catalogue(names ...string) []byte { + rows := make([]map[string]any, 0, len(names)) + for _, name := range names { + rows = append(rows, map[string]any{"name": name, "repeaters": 0, "default": 0, "monitored": true, "wardriving": false}) + } + b, _ := json.Marshal(map[string]any{"generated_at": "2026-09-27T16:58:49Z", "region": "YOW", "zones": []string{"YOW"}, "repeaters": 5, "scoped": 3, "scopes": rows}) + return b +} + +type memoryStore struct { + rows map[string]Cache + fail bool + saved []scopestore.Entry +} + +func (s *memoryStore) GetScopeCatalogue(_ context.Context, iata, url string) (*Cache, error) { + row, ok := s.rows[iata+url] + if !ok { + return nil, nil + } + return &row, nil +} +func (s *memoryStore) SaveScopeCatalogue(_ context.Context, iata, url string, next Cache, entries []scopestore.Entry) error { + if s.fail { + return errors.New("offline") + } + old := s.rows[iata+url] + if next.Payload == nil { + next.Payload = old.Payload + } + if next.CheckedAt.IsZero() { + next.CheckedAt = old.CheckedAt + next.ETag = old.ETag + } + s.rows[iata+url] = next + s.saved = append(s.saved, entries...) + return nil +} + +func names(store *scopestore.ScopeStore) []string { + out := []string{} + for _, e := range store.Entries() { + out = append(out, e.Name) + } + return out +} + +func TestCatalogueRefreshFallbackAndRestart(t *testing.T) { + ctx := context.Background() + status, body, etag, after := 200, catalogue("yow", "manual"), `"v1"`, "" + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls > 1 && r.Header.Get("If-None-Match") != `"v1"` { + t.Errorf("missing conditional refresh: %v", r.Header) + } + w.Header().Set("ETag", etag) + w.Header().Set("Retry-After", after) + w.WriteHeader(status) + if status != 304 { + _, _ = w.Write(body) + } + })) + defer server.Close() + cfg := config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}} + store := &memoryStore{rows: map[string]Cache{}} + scopes := scopestore.New() + manual := scopestore.FromName("manual") + manual.TransportKey = []byte("manual wins") + imp, err := New(ctx, cfg, store, scopes, []scopestore.Entry{manual}) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + check := func() { + t.Helper() + now = now.Add(time.Minute) + if err := imp.refresh(ctx, &imp.sources[0], now); err != nil { + t.Fatal(err) + } + } + check() + if got := names(scopes); !reflect.DeepEqual(got, []string{"#manual", "#yow"}) { + t.Fatal(got) + } + if string(scopes.Entries()[0].TransportKey) != "manual wins" || scopes.Entries()[0].IATAs != nil { + t.Fatal("manual key replaced/restricted") + } + if !reflect.DeepEqual(scopes.Entries()[1].IATAs, []string{"YOW"}) { + t.Fatal("import not bound to region") + } + generated := imp.sources[0].generated + status = 304 + check() + checked := imp.sources[0].cache.CheckedAt + if checked != now || imp.sources[0].generated != generated { + t.Fatal("304 lost freshness or changed generation time") + } + for _, failure := range []int{200, 404, 503, 302} { + status, body = failure, []byte(`{"scopes":null}`) + check() + if imp.sources[0].cache.CheckedAt != checked || len(scopes.Entries()) != 2 { + t.Fatal("failed refresh replaced known catalogue") + } + } + status, after = 429, "7200" + check() + if !imp.retryAfter.Equal(now.Add(2 * time.Hour)) { + t.Fatal("Retry-After ignored") + } + restored := scopestore.New() + restart, err := New(ctx, cfg, store, restored, []scopestore.Entry{manual}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(names(restored), names(scopes)) { + t.Fatal("outage restart lost cached names") + } + before := calls + if err := restart.Refresh(ctx); err != nil { + t.Fatal(err) + } + if calls != before { + t.Fatal("restart ignored shared rate-limit cooldown") + } + status, after, body = 200, "", catalogue() + check() + if got := names(scopes); !reflect.DeepEqual(got, []string{"#manual"}) { + t.Fatalf("empty snapshot: %v", got) + } + store.fail = true + body = catalogue("new") + now = now.Add(time.Hour) + if err := imp.refresh(ctx, &imp.sources[0], now); err == nil || len(scopes.Entries()) != 1 { + t.Fatal("uncommitted names became active") + } +} + +func TestDecodeCatalogueBoundaries(t *testing.T) { + valid := string(catalogue("yow", "YOW", "$private")) + bad := []string{ + `null`, `{}`, `[]`, valid + `{}`, + strings.Replace(valid, `"region":"YOW"`, `"region":"PNW"`, 1), + strings.Replace(valid, `"zones":["YOW"]`, `"zones":["YOW","YVR"]`, 1), + strings.Replace(valid, `"scoped":3`, `"scoped":6`, 1), + strings.Replace(valid, `"repeaters":0`, `"repeaters":-1`, 1), + strings.Replace(valid, `"monitored":true,`, ``, 1), + strings.Replace(valid, `"generated_at":"2026-09-27T16:58:49Z"`, `"generated_at":null`, 1), + string(catalogue("yow", "#yow")), string(catalogue("#")), string(catalogue(" yow")), string(catalogue("a\nb")), + strings.Repeat(" ", MaxBody+1), + } + tooMany := []string{} + for n := 0; n <= MaxScopes; n++ { + tooMany = append(tooMany, fmt.Sprint(n)) + } + bad = append(bad, string(catalogue(tooMany...))) + for n, raw := range bad { + if _, _, err := decode([]byte(raw), "YOW"); err == nil { + t.Errorf("invalid case %d accepted", n) + } + } + entries, _, err := decode([]byte(valid), "YOW") + if err != nil || len(entries) != 3 { + t.Fatal(entries, err) + } + if entries[0].Name != "#yow" || entries[1].Name != "#YOW" || entries[2].Name != "$private" { + t.Fatal("case/prefix changed", entries) + } +} + +func TestNoRedirectAndResponseBounds(t *testing.T) { + targetCalls := 0 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { targetCalls++ })) + defer target.Close() + var mode atomic.Value + mode.Store("redirect") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch mode.Load().(string) { + case "redirect": + http.Redirect(w, r, target.URL, 302) + case "oversize": + _, _ = w.Write([]byte(strings.Repeat("x", MaxBody+1))) + case "timeout": + select { + case <-r.Context().Done(): + case <-time.After(time.Second): + } + case "304": + w.WriteHeader(304) + } + })) + defer server.Close() + store := &memoryStore{rows: map[string]Cache{}} + imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopestore.New(), nil) + if err != nil { + t.Fatal(err) + } + imp.client.Timeout = 20 * time.Millisecond + for _, value := range []string{"redirect", "oversize", "timeout", "304"} { + mode.Store(value) + if err := imp.refresh(context.Background(), &imp.sources[0], time.Now()); err != nil { + t.Fatal(err) + } + if imp.sources[0].cache.LastError == "" || len(imp.sources[0].entries) != 0 { + t.Fatal("invalid response accepted", value) + } + } + if targetCalls != 0 { + t.Fatal("followed redirect") + } +} + +func TestImportedSourcesOverlapWithoutGlobalMembership(t *testing.T) { + ctx := context.Background() + store := &memoryStore{rows: map[string]Cache{}} + for _, iata := range []string{"YOW", "YVR"} { + store.rows[iata+iata] = Cache{Payload: []byte(strings.ReplaceAll(string(catalogue("can")), "YOW", iata))} + } + scopes := scopestore.New() + _, err := New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "YOW", "YVR": "YVR"}}, store, scopes, nil) + if err != nil || len(scopes.Entries()) != 1 || !reflect.DeepEqual(scopes.Entries()[0].IATAs, []string{"YOW", "YVR"}) { + t.Fatal(scopes.Entries(), err) + } + _, err = New(ctx, config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": "changed-source"}}, store, scopes, nil) + if err != nil || len(scopes.Entries()) != 0 { + t.Fatal("removed/changed source remained active", err) + } + // Disabled mode never calls the store, even if URLs remain configured. + _, err = New(ctx, config.MeshMapperScopesConfig{Sources: map[string]string{"YOW": "YOW"}}, nil, scopes, nil) + if err != nil { + t.Fatal(err) + } +} + +func TestInvalidSavedCatalogueDoesNotBlockStartupOrRefresh(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-None-Match") != "" { + t.Error("invalid saved catalogue must not accept a 304") + } + _, _ = w.Write(catalogue("yow")) + })) + defer server.Close() + store := &memoryStore{rows: map[string]Cache{"YOW" + server.URL: {Payload: []byte(`{"invalid":true}`), ETag: `"old"`}}} + scopes := scopestore.New() + imp, err := New(context.Background(), config.MeshMapperScopesConfig{Enabled: true, Sources: map[string]string{"YOW": server.URL}}, store, scopes, []scopestore.Entry{scopestore.FromName("manual")}) + if err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual"}) { + t.Fatal("invalid optional source blocked manual startup", err) + } + if err = imp.Refresh(context.Background()); err != nil || !reflect.DeepEqual(names(scopes), []string{"#manual", "#yow"}) { + t.Fatal("source did not recover on refresh", err, names(scopes)) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + imp.sources[0].cache.NextAttempt = time.Time{} + if err = imp.Refresh(ctx); err != nil { + t.Fatalf("normal shutdown is a task failure: %v", err) + } +} diff --git a/internal/presence/broker_freshness_test.go b/internal/presence/broker_freshness_test.go new file mode 100644 index 0000000..887a6e7 --- /dev/null +++ b/internal/presence/broker_freshness_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package presence + +import ( + "context" + "github.com/google/uuid" + "testing" + "time" +) + +type brokerTestStore struct { + Store + initial []bool + seen, packets []time.Time +} + +func (s *brokerTestStore) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, packet bool) error { + s.initial = append(s.initial, packet) + return nil +} +func (s *brokerTestStore) TouchObserverBrokers(_ context.Context, _ []uuid.UUID, _ []string, seen, packets []time.Time) error { + s.seen = seen + s.packets = packets + return nil +} + +func TestBrokerPresenceDoesNotInventPacketFreshness(t *testing.T) { + ctx := context.Background() + store := &brokerTestStore{} + c := New(store, time.Second, time.Minute) + now := time.Now() + c.now = func() time.Time { return now } + id := uuid.New() + c.UpsertObserverBroker(ctx, id, "one", false) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if len(store.initial) != 1 || store.initial[0] || !store.packets[0].IsZero() { + t.Fatal("status-only activity invented a packet") + } + now = now.Add(time.Minute) + packetTime := now + c.UpsertObserverBroker(ctx, id, "one", true) + now = now.Add(time.Minute) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if !store.seen[0].Equal(now) || !store.packets[0].Equal(packetTime) { + t.Fatal("later status changed packet freshness") + } +} diff --git a/internal/presence/coalescer.go b/internal/presence/coalescer.go index 9315f43..517f121 100644 --- a/internal/presence/coalescer.go +++ b/internal/presence/coalescer.go @@ -31,7 +31,7 @@ type Store interface { // TouchObserverBrokers applies coalesced last_seen/last_packet_at bumps // for the given (observer, broker) pairs in one statement. - TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error + TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error // TouchPackets applies coalesced last_heard_at bumps for the given packet // hashes in one statement. @@ -51,6 +51,8 @@ type observerBump struct { count int32 } +type brokerBump struct{ seen, packet time.Time } + type brokerKey struct { id uuid.UUID broker string @@ -68,7 +70,7 @@ type Coalescer struct { identities map[string]identity // pubkey -> observer row dirtyObservers map[uuid.UUID]observerBump knownBrokers map[brokerKey]struct{} - dirtyBrokers map[brokerKey]time.Time + dirtyBrokers map[brokerKey]brokerBump knownIATAs map[string]struct{} packetsSeen map[string]time.Time // hash -> last observation dirtyPackets map[string]time.Time @@ -86,7 +88,7 @@ func New(store Store, flushInterval, packetTTL time.Duration) *Coalescer { identities: make(map[string]identity), dirtyObservers: make(map[uuid.UUID]observerBump), knownBrokers: make(map[brokerKey]struct{}), - dirtyBrokers: make(map[brokerKey]time.Time), + dirtyBrokers: make(map[brokerKey]brokerBump), knownIATAs: make(map[string]struct{}), packetsSeen: make(map[string]time.Time), dirtyPackets: make(map[string]time.Time), @@ -122,17 +124,22 @@ func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUI // UpsertObserverBroker writes through the first time a pair is seen and // records a bump afterwards. -func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { key := brokerKey{id: observerID, broker: brokerName} c.mu.Lock() if _, ok := c.knownBrokers[key]; ok { - c.dirtyBrokers[key] = c.now() + bump := c.dirtyBrokers[key] + bump.seen = c.now() + if isPacket { + bump.packet = bump.seen + } + c.dirtyBrokers[key] = bump c.mu.Unlock() return nil } c.mu.Unlock() - if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName); err != nil { + if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName, isPacket); err != nil { return err } c.mu.Lock() @@ -248,7 +255,7 @@ func (c *Coalescer) Flush(ctx context.Context) { brokers := c.dirtyBrokers packets := c.dirtyPackets c.dirtyObservers = make(map[uuid.UUID]observerBump) - c.dirtyBrokers = make(map[brokerKey]time.Time) + c.dirtyBrokers = make(map[brokerKey]brokerBump) c.dirtyPackets = make(map[string]time.Time) cutoff := c.now().Add(-c.packetTTL) for key, seen := range c.packetsSeen { @@ -266,12 +273,14 @@ func (c *Coalescer) Flush(ctx context.Context) { ids := make([]uuid.UUID, 0, len(brokers)) names := make([]string, 0, len(brokers)) seen := make([]time.Time, 0, len(brokers)) + packetTimes := make([]time.Time, 0, len(brokers)) for key, ts := range brokers { ids = append(ids, key.id) names = append(names, key.broker) - seen = append(seen, ts) + seen = append(seen, ts.seen) + packetTimes = append(packetTimes, ts.packet) } - if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen); err != nil { + if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen, packetTimes); err != nil { slog.Error(fmt.Sprintf("presence: flush observer brokers failed (%d rows dropped)", len(ids)), "component", "presence", "error", err) } } diff --git a/internal/scopestore/scopestore.go b/internal/scopestore/scopestore.go index 8abbac3..da5c80f 100644 --- a/internal/scopestore/scopestore.go +++ b/internal/scopestore/scopestore.go @@ -5,13 +5,29 @@ // loaded from the database at startup. package scopestore -import "sync" +import ( + "crypto/sha256" + "strings" + "sync" +) // Entry holds a single transport scope key and its metadata. type Entry struct { Name string - TransportKey []byte // 16 bytes - KeyFingerprint []byte // 8 bytes + TransportKey []byte // 16 bytes + KeyFingerprint []byte // 8 bytes + IATAs []string // nil for manual keys; imported candidates are regional +} + +// FromName uses the same case-sensitive derivation for manual and imported scopes. +func FromName(name string) Entry { + if !strings.HasPrefix(name, "#") && !strings.HasPrefix(name, "$") { + name = "#" + name + } + h := sha256.Sum256([]byte(name)) + key := h[:16] + fingerprint := sha256.Sum256(key) + return Entry{Name: name, TransportKey: key, KeyFingerprint: fingerprint[:8]} } // ScopeStore holds all known transport scope keys in memory. @@ -25,18 +41,16 @@ func New() *ScopeStore { return &ScopeStore{} } -// Load replaces all entries — call on startup after DB seeding. +// Load publishes a new immutable snapshot. The caller must not mutate it after publication. func (s *ScopeStore) Load(entries []Entry) { s.mu.Lock() defer s.mu.Unlock() s.entries = entries } -// Entries returns a copy of all loaded entries. +// Entries returns the current immutable snapshot. Callers must not modify it. func (s *ScopeStore) Entries() []Entry { s.mu.RLock() defer s.mu.RUnlock() - result := make([]Entry, len(s.entries)) - copy(result, s.entries) - return result + return s.entries } diff --git a/internal/scopestore/scopestore_test.go b/internal/scopestore/scopestore_test.go index 8df3900..3e2f359 100644 --- a/internal/scopestore/scopestore_test.go +++ b/internal/scopestore/scopestore_test.go @@ -38,17 +38,19 @@ func TestLoad_ReplacesEntries(t *testing.T) { } } -func TestEntries_ReturnsCopy(t *testing.T) { +func TestEntriesRetainsImmutableSnapshotWithoutAllocation(t *testing.T) { s := New() - s.Load([]Entry{ - {Name: "#bc", TransportKey: []byte{0x01}, KeyFingerprint: []byte{0x02}}, - }) - entries := s.Entries() - entries[0].Name = "mutated" - - // original should be unchanged - original := s.Entries() - if original[0].Name != "#bc" { - t.Errorf("expected #bc, got %s after mutation of copy", original[0].Name) + s.Load([]Entry{{Name: "#bc"}}) + previous := s.Entries() + if allocations := testing.AllocsPerRun(100, func() { + if len(s.Entries()) != 1 { + t.Fatal("missing entries") + } + }); allocations != 0 { + t.Fatalf("snapshot read allocates: %v", allocations) + } + s.Load([]Entry{{Name: "#on"}}) + if previous[0].Name != "#bc" || s.Entries()[0].Name != "#on" { + t.Fatal("replacement mutated the previous reader snapshot") } } diff --git a/sqlc.yaml b/sqlc.yaml index 7943109..032c991 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -22,3 +22,11 @@ sql: go_type: "encoding/json.RawMessage" - db_type: "bpchar" go_type: "string" + + # Coalesced archive labels can still be NULL; preserve the API pointers. + - column: "mv_top_observers_by_iata.display_name" + go_type: {type: "string", pointer: true} + - column: "mv_top_observers_by_iata.observer_type" + go_type: {type: "string", pointer: true} + - column: "mv_top_advertisers_by_iata.name" + go_type: {type: "string", pointer: true}