From 70c5c389f9886cb8d8c9d9d34462c805cad6fc6b Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 03:46:29 -0400 Subject: [PATCH 01/11] feat(db): retain hourly analytics after raw packet expiry --- .github/workflows/ci.yml | 2 +- README.md | 15 + db/analytics_retention_integration_test.go | 261 +++++++++ db/migrations/039_analytics_retention.sql | 611 +++++++++++++++++++++ db/queries/queries.sql | 15 +- db/retention_integration_test.go | 5 +- db/sqlc/models.go | 158 ++++++ db/sqlc/querier.go | 2 +- db/sqlc/queries.sql.go | 24 +- sqlc.yaml | 8 + 10 files changed, 1067 insertions(+), 34 deletions(-) create mode 100644 db/analytics_retention_integration_test.go create mode 100644 db/migrations/039_analytics_retention.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beb7284..7d37492 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index 5418223..5243068 100644 --- a/README.md +++ b/README.md @@ -533,3 +533,18 @@ log: `LOG_LEVEL` and `LOG_FORMAT` override file settings; empty settings use `info` and `text`. Invalid values prevent startup. Configuration-loading failures can use the bootstrap text logger before file settings are available; failures after initialization retain error severity at every supported level. Records include a component field. Ingest workers also include their broker name, and HTTP completion records include the validated client address, route, status and duration. Query strings and protocol hello payloads are excluded. Expected ingest skips and routine WebSocket lifecycle details are debug-level. Changing the application's format does not change Caddy/Apache access logs or their fail2ban configuration. Collect/rotate stderr through Docker or systemd. + +### Analytics retention + +Hourly traffic, payload, observer activity, talker, advertiser, Signal and Paths +summaries retain 30 days independently of `packets.retention`. Cleanup saves only +aggregates before deleting each packet batch, in the same transaction. Raw +packets, observations and message bodies still expire under packet retention. +The summaries use UTC hourly buckets and appear on the normal view-refresh cycle. + +Migration 039 starts from data still present; previously deleted history cannot +be reconstructed. Telemetry has its own retention setting. Packet drill-down, +sub-hour observer activity, exact observer comparison and current entity/scope +counts continue to describe retained raw data or current entities, rather than +claiming archived packet detail. Archived summaries expire without waiting for +new packet deletions. A failed archive leaves its entire raw batch intact. diff --git a/db/analytics_retention_integration_test.go b/db/analytics_retention_integration_test.go new file mode 100644 index 0000000..2e0466c --- /dev/null +++ b/db/analytics_retention_integration_test.go @@ -0,0 +1,261 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "os" + "testing" + "time" +) + +var retainedViews = []string{"mv_hourly_iata_stats", "mv_payload_breakdown_by_iata", "mv_top_observers_by_iata", "mv_top_talkers_by_iata", "mv_top_advertisers_by_iata", "mv_observer_activity_hourly", "mv_signal_stats_hourly", "mv_path_stats_hourly"} + +func analyticsSnapshot(t *testing.T, ctx context.Context, tx pgx.Tx) map[string]string { + t.Helper() + result := map[string]string{} + for _, view := range retainedViews { + if _, err := tx.Exec(ctx, "REFRESH MATERIALIZED VIEW "+view); err != nil { + t.Fatal(err) + } + var rows string + if err := tx.QueryRow(ctx, "SELECT COALESCE(jsonb_agg(r ORDER BY r::text),'[]')::text FROM (SELECT to_jsonb(v) r FROM "+view+" v) s").Scan(&rows); err != nil { + t.Fatal(err) + } + result[view] = rows + } + return result +} + +func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + if _, err := tx.Exec(ctx, `INSERT INTO packets (packet_hash,last_heard_at) VALUES ('\x01',NOW()-interval '5 days'); INSERT INTO observers(id) VALUES ('00000000-0000-0000-0000-000000000001')`); err != nil { + t.Fatal(err) + } + var schema string + if err := tx.QueryRow(ctx, "SELECT current_schema()").Scan(&schema); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + conn := tx.Conn() + quoted := pgx.Identifier{schema}.Sanitize() + t.Cleanup(func() { + if _, err := conn.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error(err) + } + }) + if _, err := conn.Exec(ctx, "SET search_path TO "+quoted); err != nil { + t.Fatal(err) + } + writer, err := pgx.Connect(ctx, os.Getenv("BEACON_TEST_POSTGRES_DSN")) + if err != nil { + t.Fatal(err) + } + defer writer.Close(context.Background()) + if _, err = writer.Exec(ctx, "SET search_path TO "+quoted); err != nil { + t.Fatal(err) + } + ingest, err := writer.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer ingest.Rollback(context.Background()) + // An in-flight FK insert holds KEY SHARE on the packet until it commits. + if _, err = ingest.Exec(ctx, `INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\x01','00000000-0000-0000-0000-000000000001','YVR',NOW()-interval '5 days')`); err != nil { + t.Fatal(err) + } + q := sqlc.New(conn) + params := sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: time.Now().Add(-72 * time.Hour), Valid: true}, BatchSize: 1000} + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("must skip in-flight reception: %d %v", n, err) + } + if err := ingest.Commit(ctx); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 1 { + t.Fatalf("retry after ingestion: %d %v", n, err) + } + var count int + if err := conn.QueryRow(ctx, "SELECT observation_count FROM analytics_hourly_iata_stats").Scan(&count); err != nil || count != 1 { + t.Fatalf("committed reception not archived: %d %v", count, err) + } +} + +// Minimal real tables keep this regression runnable against an empty CI database. +func analyticsTables(t *testing.T, ctx context.Context, tx pgx.Tx) { + t.Helper() + isolateStatsSchema(t, ctx, tx) + _, err := tx.Exec(ctx, ` +CREATE TABLE packets (packet_hash bytea PRIMARY KEY, payload_type smallint, payload_version smallint, + route_type smallint, raw_payload bytea, raw_header bytea, origin_pubkey bytea, + first_heard_at timestamptz, last_heard_at timestamptz); +CREATE INDEX ON packets(last_heard_at); +CREATE TABLE packet_observations (id bigserial PRIMARY KEY, packet_hash bytea REFERENCES packets ON DELETE CASCADE, + observer_id uuid NOT NULL, iata char(3) NOT NULL, heard_at timestamptz NOT NULL, + path_length_byte smallint, hash_size smallint, hop_count smallint, path_bytes bytea, + snr real, rssi smallint, airtime_ms real, payload_type smallint, UNIQUE(packet_hash,observer_id)); +CREATE TABLE observers (id uuid PRIMARY KEY, public_key bytea, display_name text, observer_type text); +CREATE TABLE nodes (id uuid PRIMARY KEY, public_key bytea UNIQUE, node_type smallint, name text); +CREATE TABLE channel_messages (id bigserial PRIMARY KEY, channel_id integer, packet_hash bytea UNIQUE REFERENCES packets ON DELETE CASCADE, + sender_name text, content text, sent_at timestamptz); +`) + if err != nil { + t.Fatal(err) + } +} + +func TestAnalyticsRetentionPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + _, err := tx.Exec(ctx, ` + INSERT INTO observers (id,public_key,display_name,observer_type) SELECT ('00000000-0000-0000-0000-00000000000'||i)::uuid,int4send(i),CASE WHEN i=1 THEN NULL ELSE 'Observer '||i END,CASE WHEN i=1 THEN NULL ELSE 'test' END FROM generate_series(1,3) i; + INSERT INTO nodes (id,public_key,node_type,name) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2,'Repeater'); + INSERT INTO packets (packet_hash,payload_type,payload_version,route_type,raw_payload,raw_header,origin_pubkey,first_heard_at,last_heard_at) + SELECT int4send(i),4,0,CASE WHEN i%2=0 THEN 1 ELSE 2 END,'\x00','\x00','\x02',date_trunc('hour',NOW())-interval '5 days', + CASE WHEN i=4 THEN NOW() ELSE date_trunc('hour',NOW())-interval '5 days' END FROM generate_series(1,4) i; + INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,airtime_ms,payload_type) + SELECT packet_hash,('00000000-0000-0000-0000-00000000000'||n)::uuid,iata,first_heard_at+make_interval(secs=>n),0,1,0, + CASE WHEN n=1 THEN 0 ELSE NULL END,CASE WHEN n=1 THEN -100 ELSE NULL END,1.5,4 + FROM packets CROSS JOIN (VALUES ('YVR',1),('YVR',2),('YYZ',3)) v(iata,n); + INSERT INTO channel_messages (channel_id,packet_hash,sender_name,content,sent_at) + SELECT 1,packet_hash,'Sender','body that must expire',first_heard_at FROM packets; + CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS SELECT iata,date_trunc('hour',heard_at) AS hour,count(*) AS observation_count,count(DISTINCT packet_hash) AS unique_packets,count(DISTINCT observer_id) AS active_observers FROM packet_observations GROUP BY 1,2; + `) + if err != nil { + t.Fatal(err) + } + for _, migration := range []string{"016_mv_payload_breakdown.sql", "017_mv_top_observers.sql", "018_mv_top_talkers.sql", "019_mv_top_advertisers.sql", "021_mv_top_advertisers_route_type.sql", "032_mv_observer_activity.sql", "035_mv_signal_stats.sql", "036_mv_path_stats.sql"} { + applyStatsMigration(t, ctx, tx, migration) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + before := analyticsSnapshot(t, ctx, tx) + q := sqlc.New(tx) + cutoff := time.Now().Add(-72 * time.Hour) + // Failure in one archive must roll back every archive write and the raw deletion. + if _, err := tx.Exec(ctx, `SAVEPOINT archive_failure; ALTER TABLE analytics_signal_stats_hourly ADD CONSTRAINT fail_archive CHECK(receptions < 0) NOT VALID`); err != nil { + t.Fatal(err) + } + if _, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}); err == nil { + t.Fatal("expected archive failure") + } + if _, err := tx.Exec(ctx, "ROLLBACK TO archive_failure"); err != nil { + t.Fatal(err) + } + if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 4 { + t.Fatal("archive failure deleted raw packets") + } + for i := 0; i < 3; i++ { + n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1}) + if err != nil || n != 1 { + t.Fatalf("delete batch: %d %v", n, err) + } + } + if countRows(t, ctx, tx, "SELECT count(*) FROM packets") != 1 || countRows(t, ctx, tx, "SELECT count(*) FROM channel_messages") != 1 { + t.Fatal("raw packets/messages did not expire") + } + after := analyticsSnapshot(t, ctx, tx) + for _, view := range retainedViews { + if after[view] != before[view] { + t.Errorf("%s lost or double-counted history after raw expiry: before=%s after=%s", view, before[view], after[view]) + } + } + // Retry and migration journal retry preserve the exact summaries. + if n, err := q.DeleteOldPackets(ctx, sqlc.DeleteOldPacketsParams{Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true}, BatchSize: 1000}); err != nil || n != 0 { + t.Fatalf("retry: %d %v", n, err) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != before[view] { + t.Errorf("retry changed %s", view) + } + } + // Later receptions on a still-live packet must join its old bucket before expiry. + if _, err := tx.Exec(ctx, `INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000004'); INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi,payload_type) + SELECT packet_hash,'00000000-0000-0000-0000-000000000004','YVR',first_heard_at,0,1,0,'NaN',-80,NULL FROM packets; + UPDATE packets SET last_heard_at=first_heard_at`); err != nil { + t.Fatal(err) + } + late := analyticsSnapshot(t, ctx, tx) + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != late[view] { + t.Errorf("late reception changed %s: %s != %s", view, rows, late[view]) + } + } + // Archived labels follow a current rename; no entity FK may erase retained history. + if _, err := tx.Exec(ctx, `UPDATE nodes SET name='Renamed'; UPDATE observers SET display_name=NULL,observer_type=NULL;`); err != nil { + t.Fatal(err) + } + analyticsSnapshot(t, ctx, tx) + if countRows(t, ctx, tx, "SELECT count(*) FROM mv_top_advertisers_by_iata WHERE name='Renamed'") != 2 { + t.Fatal("archive name did not follow rename") + } + if _, err := q.GetStatsTopObservers(ctx, sqlc.GetStatsTopObserversParams{Column1: pgtype.Interval{Microseconds: int64(30 * 24 * time.Hour / time.Microsecond), Valid: true}, Limit: 10}); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, "DELETE FROM nodes; DELETE FROM observers"); err != nil { + t.Fatal(err) + } + if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_top_advertisers_by_iata") != 2 { + t.Fatal("entity deletion erased archive") + } + // Independent expiry runs even when there are no packets left to delete. + for _, view := range retainedViews { + table := "analytics_" + view[3:] + at := "bucket" + if view == "mv_hourly_iata_stats" || view == "mv_signal_stats_hourly" || view == "mv_path_stats_hourly" { + at = "hour" + } + if _, err := tx.Exec(ctx, "UPDATE "+table+" SET "+at+"="+at+"-interval '31 days'"); err != nil { + t.Fatal(err) + } + } + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + for _, view := range retainedViews { + if countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) != 0 { + t.Errorf("%s archive did not expire", view) + } + } + // More than one populated cohort, with shared hourly keys across the boundary. + if _, err := tx.Exec(ctx, ` +INSERT INTO observers (id) VALUES ('00000000-0000-0000-0000-000000000001'); +INSERT INTO nodes (id,public_key,node_type) VALUES ('00000000-0000-0000-0000-000000000002','\x02',2); +INSERT INTO packets (packet_hash,payload_type,route_type,origin_pubkey,first_heard_at,last_heard_at,raw_payload) +SELECT int4send(i),4,1,'\x02',date_trunc('hour',NOW())-interval '5 days',date_trunc('hour',NOW())-interval '5 days',decode(repeat('ab',1024),'hex') FROM generate_series(10000,11000) i; +INSERT INTO packet_observations (packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) +SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YVR',first_heard_at,0,1,0,4,0,-100 FROM packets; +INSERT INTO channel_messages (packet_hash,sender_name,sent_at) SELECT packet_hash,'Sender',first_heard_at FROM packets; +`); err != nil { + t.Fatal(err) + } + populated := analyticsSnapshot(t, ctx, tx) + started := time.Now() + if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { + t.Fatal(err) + } + t.Logf("1001 packets with bodies and observations archived/deleted in %s", time.Since(started)) + for view, rows := range analyticsSnapshot(t, ctx, tx) { + if rows != populated[view] { + t.Errorf("batch boundary changed %s", view) + } + } + var archiveRows int + for _, view := range retainedViews { + archiveRows += countRows(t, ctx, tx, "SELECT count(*) FROM analytics_"+view[3:]) + } + if archiveRows > 12 { + t.Fatalf("expected compact rollups, got %d archive rows for 1001 packets", archiveRows) + } +} diff --git a/db/migrations/039_analytics_retention.sql b/db/migrations/039_analytics_retention.sql new file mode 100644 index 0000000..75d5007 --- /dev/null +++ b/db/migrations/039_analytics_retention.sql @@ -0,0 +1,611 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Archive only aggregates of deleted packet cohorts, never packet/message bodies +-- or raw paths. Live and archived cohorts are disjoint in every refresh snapshot. +-- Thirty days matches the longest analytics window; raw retention is independent. + + +CREATE TABLE IF NOT EXISTS analytics_hourly_iata_stats ( + iata char(3), + hour timestamptz, + observation_count bigint, + unique_packets bigint, + PRIMARY KEY (iata,hour) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_hourly_iata_stats_time ON analytics_hourly_iata_stats (hour); + +CREATE TABLE IF NOT EXISTS analytics_payload_breakdown_by_iata ( + iata char(3), + payload_type smallint, + bucket timestamptz, + count bigint, + PRIMARY KEY (iata,payload_type,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_payload_breakdown_by_iata_time ON analytics_payload_breakdown_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_observers_by_iata ( + iata char(3), + observer_id uuid, + bucket timestamptz, + observation_count bigint, + display_name text, + observer_type text, + PRIMARY KEY (iata,observer_id,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_observers_by_iata_time ON analytics_top_observers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_talkers_by_iata ( + iata char(3), + sender_name text, + bucket timestamptz, + message_count bigint, + last_sent timestamptz, + PRIMARY KEY (iata,sender_name,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_talkers_by_iata_time ON analytics_top_talkers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_top_advertisers_by_iata ( + iata char(3), + node_id uuid, + bucket timestamptz, + advert_count bigint, + flood_advert_count bigint, + direct_advert_count bigint, + last_heard timestamptz, + name text, + node_type smallint, + PRIMARY KEY (iata,node_id,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_top_advertisers_by_iata_time ON analytics_top_advertisers_by_iata (bucket); + +CREATE TABLE IF NOT EXISTS analytics_observer_activity_hourly ( + observer_id uuid, + payload_type smallint, + bucket timestamptz, + observations bigint, + airtime_ms real, + airtime_n bigint, + snr_sum real, + snr_n bigint, + snr_min real, + rssi_sum bigint, + rssi_n bigint, + PRIMARY KEY (observer_id,payload_type,bucket) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_observer_activity_hourly_time ON analytics_observer_activity_hourly (bucket); + +CREATE TABLE IF NOT EXISTS analytics_signal_stats_hourly ( + iata char(3), + hour timestamptz, + kind integer, + snr_bin integer, + rssi_bin integer, + receptions bigint, + snr_samples bigint, + snr_sum double precision, + rssi_samples bigint, + rssi_sum double precision, + PRIMARY KEY (iata,hour,kind,snr_bin,rssi_bin) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_signal_stats_hourly_time ON analytics_signal_stats_hourly (hour); + +CREATE TABLE IF NOT EXISTS analytics_path_stats_hourly ( + iata char(3), + hour timestamptz, + category integer, + hash_bytes integer, + entries integer, + receptions bigint, + PRIMARY KEY (iata,hour,category,hash_bytes,entries) +); + +CREATE INDEX IF NOT EXISTS idx_analytics_path_stats_hourly_time ON analytics_path_stats_hourly (hour); + +DROP MATERIALIZED VIEW IF EXISTS mv_hourly_iata_stats; + +CREATE OR REPLACE VIEW analytics_live_hourly_iata_stats AS +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_hourly_iata_stats AS +WITH combined AS ( + SELECT iata, hour, observation_count, unique_packets FROM analytics_live_hourly_iata_stats + UNION ALL + SELECT iata, hour, observation_count, unique_packets FROM analytics_hourly_iata_stats + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observers_by_hour AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, observer_id + FROM packet_observations WHERE heard_at > NOW() - INTERVAL '30 days' + UNION + SELECT iata, bucket AS hour, observer_id FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), observer_counts AS (SELECT iata, hour, COUNT(*) AS n FROM observers_by_hour GROUP BY iata,hour) +SELECT combined.iata, combined.hour, SUM(observation_count)::bigint AS observation_count, SUM(unique_packets)::bigint AS unique_packets, MAX(o.n)::bigint AS active_observers +FROM combined JOIN observer_counts o USING (iata,hour) GROUP BY combined.iata,combined.hour; + +CREATE UNIQUE INDEX idx_analytics_hourly_iata_stats_view ON mv_hourly_iata_stats (iata,hour); + + +DROP MATERIALIZED VIEW IF EXISTS mv_payload_breakdown_by_iata; + +CREATE OR REPLACE VIEW analytics_live_payload_breakdown_by_iata AS +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_payload_breakdown_by_iata AS +WITH combined AS ( + SELECT iata, payload_type, bucket, count FROM analytics_live_payload_breakdown_by_iata + UNION ALL + SELECT iata, payload_type, bucket, count FROM analytics_payload_breakdown_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, payload_type, bucket, SUM(count)::bigint AS count +FROM combined GROUP BY iata,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_payload_breakdown_by_iata_view ON mv_payload_breakdown_by_iata (iata,payload_type,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_observers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_observers_by_iata AS +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM packet_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_observers_by_iata AS +WITH combined AS ( + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_live_top_observers_by_iata + UNION ALL + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM analytics_top_observers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, observer_id, bucket, SUM(observation_count)::bigint AS observation_count, MAX(display_name) AS display_name, MAX(observer_type) AS observer_type +FROM combined GROUP BY iata,observer_id,bucket) +SELECT t.iata, t.observer_id, COALESCE(o.display_name,t.display_name) AS display_name, + COALESCE(o.observer_type,t.observer_type) AS observer_type, t.bucket, t.observation_count +FROM totals t LEFT JOIN observers o ON o.id=t.observer_id; + +CREATE UNIQUE INDEX idx_analytics_top_observers_by_iata_view ON mv_top_observers_by_iata (iata,observer_id,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_talkers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_talkers_by_iata AS +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN packet_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_talkers_by_iata AS +WITH combined AS ( + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_live_top_talkers_by_iata + UNION ALL + SELECT iata, sender_name, bucket, message_count, last_sent FROM analytics_top_talkers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT iata, sender_name, bucket, SUM(message_count)::bigint AS message_count, MAX(last_sent) AS last_sent +FROM combined GROUP BY iata,sender_name,bucket; + +CREATE UNIQUE INDEX idx_analytics_top_talkers_by_iata_view ON mv_top_talkers_by_iata (iata,sender_name,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_top_advertisers_by_iata; + +CREATE OR REPLACE VIEW analytics_live_top_advertisers_by_iata AS +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN packet_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at > NOW() - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_top_advertisers_by_iata AS +WITH combined AS ( + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_live_top_advertisers_by_iata + UNION ALL + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM analytics_top_advertisers_by_iata + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +), totals AS (SELECT iata, node_id, bucket, SUM(advert_count)::bigint AS advert_count, SUM(flood_advert_count)::bigint AS flood_advert_count, SUM(direct_advert_count)::bigint AS direct_advert_count, MAX(last_heard) AS last_heard, MAX(name) AS name, MAX(node_type) AS node_type +FROM combined GROUP BY iata,node_id,bucket) +SELECT t.iata, t.node_id, COALESCE(n.name,t.name) AS name, COALESCE(n.node_type,t.node_type) AS node_type, + t.bucket, t.advert_count, t.flood_advert_count, t.direct_advert_count, t.last_heard +FROM totals t LEFT JOIN nodes n ON n.id=t.node_id; + +CREATE UNIQUE INDEX idx_analytics_top_advertisers_by_iata_view ON mv_top_advertisers_by_iata (iata,node_id,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_observer_activity_hourly; + +CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS +SELECT + observer_id, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY observer_id, payload_type, date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS +WITH combined AS ( + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly + UNION ALL + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n +FROM combined GROUP BY observer_id,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); + + +DROP MATERIALIZED VIEW IF EXISTS mv_signal_stats_hourly; + +CREATE OR REPLACE VIEW analytics_live_signal_stats_hourly AS +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)); + +CREATE MATERIALIZED VIEW mv_signal_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_live_signal_stats_hourly + UNION ALL + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM analytics_signal_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, kind, snr_bin, rssi_bin, SUM(receptions)::bigint AS receptions, SUM(snr_samples)::bigint AS snr_samples, SUM(snr_sum)::double precision AS snr_sum, SUM(rssi_samples)::bigint AS rssi_samples, SUM(rssi_sum)::double precision AS rssi_sum +FROM combined GROUP BY iata,hour,kind,snr_bin,rssi_bin; + +CREATE UNIQUE INDEX idx_analytics_signal_stats_hourly_view ON mv_signal_stats_hourly (iata,hour,kind,snr_bin,rssi_bin); + + +DROP MATERIALIZED VIEW IF EXISTS mv_path_stats_hourly; + +CREATE OR REPLACE VIEW analytics_live_path_stats_hourly AS +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM packet_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + AND heard_at < date_trunc('hour', NOW(), 'UTC') +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries; + +CREATE MATERIALIZED VIEW mv_path_stats_hourly AS +WITH combined AS ( + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_live_path_stats_hourly + UNION ALL + SELECT iata, hour, category, hash_bytes, entries, receptions FROM analytics_path_stats_hourly + WHERE hour >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND hour < date_trunc('hour', NOW(), 'UTC') +) +SELECT iata, hour, category, hash_bytes, entries, SUM(receptions)::bigint AS receptions +FROM combined GROUP BY iata,hour,category,hash_bytes,entries; + +CREATE UNIQUE INDEX idx_analytics_path_stats_hourly_view ON mv_path_stats_hourly (iata,hour,category,hash_bytes,entries); + + +-- VOLATILE gives the archive statement a new READ COMMITTED snapshot after +-- acquiring parent locks. FK inserts cannot race between that snapshot and the +-- cascade. SKIP LOCKED leaves packets being ingested for a later cleanup tick. +CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) +RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ +DECLARE hashes bytea[]; deleted bigint; +BEGIN + IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; + SELECT array_agg(p.packet_hash) INTO hashes FROM ( + SELECT ep.packet_hash FROM packets ep + WHERE ep.last_heard_at < cutoff + ORDER BY ep.last_heard_at, ep.packet_hash + LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED + ) p; + + WITH expired_observations AS MATERIALIZED ( + SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) + ), +archived_hourly_iata_stats AS ( + INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) + SELECT iata, hour, observation_count, unique_packets FROM ( +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,hour) DO UPDATE SET + observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, + unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets +), +archived_payload_breakdown_by_iata AS ( + INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) + SELECT iata, payload_type, bucket, count FROM ( +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET + count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count +), +archived_top_observers_by_iata AS ( + INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM expired_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET + observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, + display_name = EXCLUDED.display_name, + observer_type = EXCLUDED.observer_type +), +archived_top_talkers_by_iata AS ( + INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) + SELECT iata, sender_name, bucket, message_count, last_sent FROM ( +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN expired_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') + ) batch + ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET + message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, + last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) +), +archived_top_advertisers_by_iata AS ( + INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN expired_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,node_id,bucket) DO UPDATE SET + advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, + flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, + direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, + last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), + name = EXCLUDED.name, + node_type = EXCLUDED.node_type +), +archived_observer_activity_hourly AS ( + INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( +SELECT + observer_id, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY observer_id, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET + observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, + airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, + airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, + snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, + snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, + snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), + rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, + rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n +), +archived_signal_stats_hourly AS ( + INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) + ) batch + ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET + receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, + snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, + snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, + rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, + rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum +), +archived_path_stats_hourly AS ( + INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) + SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries + ) batch + ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET + receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions +) + DELETE FROM packets WHERE packet_hash = ANY(hashes); + GET DIAGNOSTICS deleted = ROW_COUNT; + + -- Once per completed cleanup, including when there were no raw packets to delete. + IF deleted < batch_size THEN + DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + END IF; + RETURN deleted; +END $$; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index ccc8492..29690a1 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -661,18 +661,9 @@ ORDER BY po.id ASC LIMIT $6; --- name: DeleteOldPackets :execrows --- One batch of expired packets; observations and channel messages cascade. -WITH expired AS ( - SELECT ep.packet_hash - FROM packets ep - WHERE ep.last_heard_at < @cutoff - ORDER BY ep.last_heard_at - LIMIT @batch_size - FOR UPDATE OF ep SKIP LOCKED -) -DELETE FROM packets p USING expired e -WHERE p.packet_hash = e.packet_hash; +-- name: DeleteOldPackets :one +-- Locks, archives and cascades one bounded packet cohort in a single transaction. +SELECT archive_delete_packets(@cutoff::timestamptz, @batch_size::integer)::bigint; -- name: DeleteOldNodes :exec -- Deletes nodes not seen since the given cutoff. node_iatas and node_neighbors cascade- diff --git a/db/retention_integration_test.go b/db/retention_integration_test.go index 5e40dbd..d630717 100644 --- a/db/retention_integration_test.go +++ b/db/retention_integration_test.go @@ -46,10 +46,9 @@ func countRows(t *testing.T, ctx context.Context, tx pgx.Tx, sql string) int { // More than one batch of old packets; observations go with them. func TestDeleteOldPacketsBatchesPostgres(t *testing.T) { ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") _, err := tx.Exec(ctx, ` -CREATE TEMP TABLE packets (LIKE public.packets INCLUDING ALL) ON COMMIT DROP; -CREATE TEMP TABLE packet_observations (LIKE public.packet_observations INCLUDING ALL) ON COMMIT DROP; -ALTER TABLE packet_observations ADD FOREIGN KEY (packet_hash) REFERENCES packets(packet_hash) ON DELETE CASCADE; INSERT INTO packets (packet_hash, payload_type, payload_version, route_type, raw_payload, raw_header, first_heard_at, last_heard_at) SELECT int4send(i), 4, 0, 1, '\x00', '\x00', CASE WHEN i <= 2500 THEN '2026-01-01'::timestamptz ELSE '2026-02-01'::timestamptz END, diff --git a/db/sqlc/models.go b/db/sqlc/models.go index b5ffb05..2c6e20f 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -16,6 +16,164 @@ type Account struct { DeactivatedAt pgtype.Timestamptz `json:"deactivated_at"` } +type AnalyticsHourlyIataStat struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount *int64 `json:"observation_count"` + UniquePackets *int64 `json:"unique_packets"` +} + +type AnalyticsLiveHourlyIataStat struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + ObservationCount int64 `json:"observation_count"` + UniquePackets int64 `json:"unique_packets"` +} + +type AnalyticsLiveObserverActivityHourly struct { + ObserverID uuid.UUID `json:"observer_id"` + PayloadType *int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Observations int64 `json:"observations"` + AirtimeMs float32 `json:"airtime_ms"` + AirtimeN int64 `json:"airtime_n"` + SnrSum float32 `json:"snr_sum"` + SnrN int64 `json:"snr_n"` + SnrMin float32 `json:"snr_min"` + RssiSum int64 `json:"rssi_sum"` + RssiN int64 `json:"rssi_n"` +} + +type AnalyticsLivePathStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions int64 `json:"receptions"` +} + +type AnalyticsLivePayloadBreakdownByIatum struct { + Iata string `json:"iata"` + PayloadType *int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Count int64 `json:"count"` +} + +type AnalyticsLiveSignalStatsHourly struct { + Iata string `json:"iata"` + Hour interface{} `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` + SnrSum float64 `json:"snr_sum"` + RssiSamples int64 `json:"rssi_samples"` + RssiSum float64 `json:"rssi_sum"` +} + +type AnalyticsLiveTopAdvertisersByIatum struct { + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Name *string `json:"name"` + NodeType int16 `json:"node_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` + LastHeard interface{} `json:"last_heard"` +} + +type AnalyticsLiveTopObserversByIatum struct { + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + ObservationCount int64 `json:"observation_count"` +} + +type AnalyticsLiveTopTalkersByIatum struct { + Iata string `json:"iata"` + SenderName *string `json:"sender_name"` + Bucket pgtype.Timestamptz `json:"bucket"` + MessageCount int64 `json:"message_count"` + LastSent interface{} `json:"last_sent"` +} + +type AnalyticsObserverActivityHourly struct { + ObserverID uuid.UUID `json:"observer_id"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Observations *int64 `json:"observations"` + AirtimeMs *float32 `json:"airtime_ms"` + AirtimeN *int64 `json:"airtime_n"` + SnrSum *float32 `json:"snr_sum"` + SnrN *int64 `json:"snr_n"` + SnrMin *float32 `json:"snr_min"` + RssiSum *int64 `json:"rssi_sum"` + RssiN *int64 `json:"rssi_n"` +} + +type AnalyticsPathStatsHourly struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + Category int32 `json:"category"` + HashBytes int32 `json:"hash_bytes"` + Entries int32 `json:"entries"` + Receptions *int64 `json:"receptions"` +} + +type AnalyticsPayloadBreakdownByIatum struct { + Iata string `json:"iata"` + PayloadType int16 `json:"payload_type"` + Bucket pgtype.Timestamptz `json:"bucket"` + Count *int64 `json:"count"` +} + +type AnalyticsSignalStatsHourly struct { + Iata string `json:"iata"` + Hour pgtype.Timestamptz `json:"hour"` + Kind int32 `json:"kind"` + SnrBin int32 `json:"snr_bin"` + RssiBin int32 `json:"rssi_bin"` + Receptions *int64 `json:"receptions"` + SnrSamples *int64 `json:"snr_samples"` + SnrSum *float64 `json:"snr_sum"` + RssiSamples *int64 `json:"rssi_samples"` + RssiSum *float64 `json:"rssi_sum"` +} + +type AnalyticsTopAdvertisersByIatum struct { + Iata string `json:"iata"` + NodeID uuid.UUID `json:"node_id"` + Bucket pgtype.Timestamptz `json:"bucket"` + AdvertCount *int64 `json:"advert_count"` + FloodAdvertCount *int64 `json:"flood_advert_count"` + DirectAdvertCount *int64 `json:"direct_advert_count"` + LastHeard pgtype.Timestamptz `json:"last_heard"` + Name *string `json:"name"` + NodeType *int16 `json:"node_type"` +} + +type AnalyticsTopObserversByIatum struct { + Iata string `json:"iata"` + ObserverID uuid.UUID `json:"observer_id"` + Bucket pgtype.Timestamptz `json:"bucket"` + ObservationCount *int64 `json:"observation_count"` + DisplayName *string `json:"display_name"` + ObserverType *string `json:"observer_type"` +} + +type AnalyticsTopTalkersByIatum struct { + Iata string `json:"iata"` + SenderName string `json:"sender_name"` + Bucket pgtype.Timestamptz `json:"bucket"` + MessageCount *int64 `json:"message_count"` + LastSent pgtype.Timestamptz `json:"last_sent"` +} + type Channel struct { ID int32 `json:"id"` ChannelHash []byte `json:"channel_hash"` diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 1547c1c..38e6c97 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -29,7 +29,7 @@ type Querier interface { // Opt-in age-out: preserve retained history and manually recorded ownership. // Bound deletions per cleanup tick and skip observers being updated by ingest. DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timestamptz) ([]uuid.UUID, error) - // One batch of expired packets; observations and channel messages cascade. + // Locks, archives and cascades one bounded packet cohort in a single transaction. DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) // One batch of routes past retention, or past grace with too few observations. // GREATEST keeps the scan on idx_known_routes_last_seen. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index c047b44..6a0eee1 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -124,17 +124,8 @@ func (q *Queries) DeleteOldObservers(ctx context.Context, lastSeen pgtype.Timest return items, nil } -const deleteOldPackets = `-- name: DeleteOldPackets :execrows -WITH expired AS ( - SELECT ep.packet_hash - FROM packets ep - WHERE ep.last_heard_at < $1 - ORDER BY ep.last_heard_at - LIMIT $2 - FOR UPDATE OF ep SKIP LOCKED -) -DELETE FROM packets p USING expired e -WHERE p.packet_hash = e.packet_hash +const deleteOldPackets = `-- name: DeleteOldPackets :one +SELECT archive_delete_packets($1::timestamptz, $2::integer)::bigint ` type DeleteOldPacketsParams struct { @@ -142,13 +133,12 @@ type DeleteOldPacketsParams struct { BatchSize int32 `json:"batch_size"` } -// One batch of expired packets; observations and channel messages cascade. +// Locks, archives and cascades one bounded packet cohort in a single transaction. func (q *Queries) DeleteOldPackets(ctx context.Context, arg DeleteOldPacketsParams) (int64, error) { - result, err := q.db.Exec(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) - if err != nil { - return 0, err - } - return result.RowsAffected(), nil + row := q.db.QueryRow(ctx, deleteOldPackets, arg.Cutoff, arg.BatchSize) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err } const deleteOldRoutes = `-- name: DeleteOldRoutes :execrows diff --git a/sqlc.yaml b/sqlc.yaml index 7943109..032c991 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -22,3 +22,11 @@ sql: go_type: "encoding/json.RawMessage" - db_type: "bpchar" go_type: "string" + + # Coalesced archive labels can still be NULL; preserve the API pointers. + - column: "mv_top_observers_by_iata.display_name" + go_type: {type: "string", pointer: true} + - column: "mv_top_observers_by_iata.observer_type" + go_type: {type: "string", pointer: true} + - column: "mv_top_advertisers_by_iata.name" + go_type: {type: "string", pointer: true} From 2ab36356fa023abb247db45c8a33de4410507f64 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 18:27:39 -0400 Subject: [PATCH 02/11] fix: address maintainer review for #167 --- db/analytics_retention_integration_test.go | 34 +++++++++++++++ db/migrations/039_analytics_retention.sql | 48 +++++++++++----------- db/sqlc/models.go | 34 +++++++-------- sqlc.yaml | 3 ++ 4 files changed, 79 insertions(+), 40 deletions(-) diff --git a/db/analytics_retention_integration_test.go b/db/analytics_retention_integration_test.go index 2e0466c..10e3333 100644 --- a/db/analytics_retention_integration_test.go +++ b/db/analytics_retention_integration_test.go @@ -9,6 +9,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "os" + "strings" "testing" "time" ) @@ -87,6 +88,30 @@ func TestAnalyticsRetentionConcurrentPostgres(t *testing.T) { if err := conn.QueryRow(ctx, "SELECT observation_count FROM analytics_hourly_iata_stats").Scan(&count); err != nil || count != 1 { t.Fatalf("committed reception not archived: %d %v", count, err) } + // A second cleanup runner must wait before touching shared aggregate keys. + first, err := writer.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer first.Rollback(context.Background()) + if _, err := sqlc.New(first).DeleteOldPackets(ctx, params); err != nil { + t.Fatal(err) + } + if _, err := conn.Exec(ctx, "SET statement_timeout='300ms'"); err != nil { + t.Fatal(err) + } + if _, err := q.DeleteOldPackets(ctx, params); err == nil { + t.Fatalf("overlapping cleanup did not wait for the first transaction: %v", err) + } + if _, err := conn.Exec(ctx, "SET statement_timeout=0"); err != nil { + t.Fatal(err) + } + if err := first.Commit(ctx); err != nil { + t.Fatal(err) + } + if n, err := q.DeleteOldPackets(ctx, params); err != nil || n != 0 { + t.Fatalf("cleanup lock was not released: %d %v", n, err) + } } // Minimal real tables keep this regression runnable against an empty CI database. @@ -136,6 +161,12 @@ func TestAnalyticsRetentionPostgres(t *testing.T) { applyStatsMigration(t, ctx, tx, migration) } applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + for _, name := range []string{"idx_mv_signal_stats_hourly_hour", "idx_mv_path_stats_hourly_hour"} { + var definition string + if err := tx.QueryRow(ctx, "SELECT pg_get_indexdef(to_regclass($1))", name).Scan(&definition); err != nil || !strings.Contains(definition, "(hour)") { + t.Fatalf("missing unfiltered time index %s: %s %v", name, definition, err) + } + } before := analyticsSnapshot(t, ctx, tx) q := sqlc.New(tx) cutoff := time.Now().Add(-72 * time.Hour) @@ -184,6 +215,9 @@ func TestAnalyticsRetentionPostgres(t *testing.T) { t.Fatal(err) } late := analyticsSnapshot(t, ctx, tx) + if countRows(t, ctx, tx, "SELECT COALESCE(SUM(observations),0) FROM mv_observer_activity_hourly WHERE payload_type=-1") != 1 { + t.Fatal("unknown-payload observation was omitted") + } if err := (&Store{q: q}).DeleteOldPackets(ctx, cutoff); err != nil { t.Fatal(err) } diff --git a/db/migrations/039_analytics_retention.sql b/db/migrations/039_analytics_retention.sql index 75d5007..069204a 100644 --- a/db/migrations/039_analytics_retention.sql +++ b/db/migrations/039_analytics_retention.sql @@ -9,8 +9,8 @@ CREATE TABLE IF NOT EXISTS analytics_hourly_iata_stats ( iata char(3), hour timestamptz, - observation_count bigint, - unique_packets bigint, + observation_count bigint NOT NULL DEFAULT 0, + unique_packets bigint NOT NULL DEFAULT 0, PRIMARY KEY (iata,hour) ); @@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS analytics_payload_breakdown_by_iata ( iata char(3), payload_type smallint, bucket timestamptz, - count bigint, + count bigint NOT NULL DEFAULT 0, PRIMARY KEY (iata,payload_type,bucket) ); @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS analytics_top_observers_by_iata ( iata char(3), observer_id uuid, bucket timestamptz, - observation_count bigint, + observation_count bigint NOT NULL DEFAULT 0, display_name text, observer_type text, PRIMARY KEY (iata,observer_id,bucket) @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS analytics_top_talkers_by_iata ( iata char(3), sender_name text, bucket timestamptz, - message_count bigint, + message_count bigint NOT NULL DEFAULT 0, last_sent timestamptz, PRIMARY KEY (iata,sender_name,bucket) ); @@ -53,9 +53,9 @@ CREATE TABLE IF NOT EXISTS analytics_top_advertisers_by_iata ( iata char(3), node_id uuid, bucket timestamptz, - advert_count bigint, - flood_advert_count bigint, - direct_advert_count bigint, + advert_count bigint NOT NULL DEFAULT 0, + flood_advert_count bigint NOT NULL DEFAULT 0, + direct_advert_count bigint NOT NULL DEFAULT 0, last_heard timestamptz, name text, node_type smallint, @@ -68,14 +68,14 @@ CREATE TABLE IF NOT EXISTS analytics_observer_activity_hourly ( observer_id uuid, payload_type smallint, bucket timestamptz, - observations bigint, + observations bigint NOT NULL DEFAULT 0, airtime_ms real, - airtime_n bigint, + airtime_n bigint NOT NULL DEFAULT 0, snr_sum real, - snr_n bigint, + snr_n bigint NOT NULL DEFAULT 0, snr_min real, rssi_sum bigint, - rssi_n bigint, + rssi_n bigint NOT NULL DEFAULT 0, PRIMARY KEY (observer_id,payload_type,bucket) ); @@ -87,10 +87,10 @@ CREATE TABLE IF NOT EXISTS analytics_signal_stats_hourly ( kind integer, snr_bin integer, rssi_bin integer, - receptions bigint, - snr_samples bigint, + receptions bigint NOT NULL DEFAULT 0, + snr_samples bigint NOT NULL DEFAULT 0, snr_sum double precision, - rssi_samples bigint, + rssi_samples bigint NOT NULL DEFAULT 0, rssi_sum double precision, PRIMARY KEY (iata,hour,kind,snr_bin,rssi_bin) ); @@ -103,7 +103,7 @@ CREATE TABLE IF NOT EXISTS analytics_path_stats_hourly ( category integer, hash_bytes integer, entries integer, - receptions bigint, + receptions bigint NOT NULL DEFAULT 0, PRIMARY KEY (iata,hour,category,hash_bytes,entries) ); @@ -260,11 +260,10 @@ CREATE UNIQUE INDEX idx_analytics_top_advertisers_by_iata_view ON mv_top_adverti DROP MATERIALIZED VIEW IF EXISTS mv_observer_activity_hourly; - CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS SELECT observer_id, - payload_type, + COALESCE(payload_type, -1)::smallint AS payload_type, date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, COUNT(*)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, @@ -276,8 +275,7 @@ SELECT COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM packet_observations WHERE heard_at > NOW() - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY observer_id, payload_type, date_trunc('hour', heard_at, 'UTC'); +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS WITH combined AS ( @@ -292,6 +290,7 @@ FROM combined GROUP BY observer_id,payload_type,bucket; CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); + DROP MATERIALIZED VIEW IF EXISTS mv_signal_stats_hourly; CREATE OR REPLACE VIEW analytics_live_signal_stats_hourly AS @@ -333,6 +332,7 @@ SELECT iata, hour, kind, snr_bin, rssi_bin, SUM(receptions)::bigint AS reception FROM combined GROUP BY iata,hour,kind,snr_bin,rssi_bin; CREATE UNIQUE INDEX idx_analytics_signal_stats_hourly_view ON mv_signal_stats_hourly (iata,hour,kind,snr_bin,rssi_bin); +CREATE INDEX idx_mv_signal_stats_hourly_hour ON mv_signal_stats_hourly (hour); DROP MATERIALIZED VIEW IF EXISTS mv_path_stats_hourly; @@ -376,6 +376,7 @@ SELECT iata, hour, category, hash_bytes, entries, SUM(receptions)::bigint AS rec FROM combined GROUP BY iata,hour,category,hash_bytes,entries; CREATE UNIQUE INDEX idx_analytics_path_stats_hourly_view ON mv_path_stats_hourly (iata,hour,category,hash_bytes,entries); +CREATE INDEX idx_mv_path_stats_hourly_hour ON mv_path_stats_hourly (hour); -- VOLATILE gives the archive statement a new READ COMMITTED snapshot after @@ -385,6 +386,8 @@ CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ DECLARE hashes bytea[]; deleted bigint; BEGIN + -- Serialize cleanup runners before overlapping aggregate upserts. + PERFORM pg_advisory_xact_lock(hashtext('beacon.archive_delete_packets')); IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; SELECT array_agg(p.packet_hash) INTO hashes FROM ( SELECT ep.packet_hash FROM packets ep @@ -500,7 +503,7 @@ archived_observer_activity_hourly AS ( SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( SELECT observer_id, - payload_type, + COALESCE(payload_type, -1)::smallint AS payload_type, date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, COUNT(*)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, @@ -512,8 +515,7 @@ SELECT COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM expired_observations WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY observer_id, payload_type, date_trunc('hour', heard_at, 'UTC') +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') ) batch ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 2c6e20f..8a24d53 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -19,8 +19,8 @@ type Account struct { type AnalyticsHourlyIataStat struct { Iata string `json:"iata"` Hour pgtype.Timestamptz `json:"hour"` - ObservationCount *int64 `json:"observation_count"` - UniquePackets *int64 `json:"unique_packets"` + ObservationCount int64 `json:"observation_count"` + UniquePackets int64 `json:"unique_packets"` } type AnalyticsLiveHourlyIataStat struct { @@ -32,7 +32,7 @@ type AnalyticsLiveHourlyIataStat struct { type AnalyticsLiveObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` - PayloadType *int16 `json:"payload_type"` + PayloadType int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs float32 `json:"airtime_ms"` @@ -106,14 +106,14 @@ type AnalyticsObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` PayloadType int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` - Observations *int64 `json:"observations"` + Observations int64 `json:"observations"` AirtimeMs *float32 `json:"airtime_ms"` - AirtimeN *int64 `json:"airtime_n"` + AirtimeN int64 `json:"airtime_n"` SnrSum *float32 `json:"snr_sum"` - SnrN *int64 `json:"snr_n"` + SnrN int64 `json:"snr_n"` SnrMin *float32 `json:"snr_min"` RssiSum *int64 `json:"rssi_sum"` - RssiN *int64 `json:"rssi_n"` + RssiN int64 `json:"rssi_n"` } type AnalyticsPathStatsHourly struct { @@ -122,14 +122,14 @@ type AnalyticsPathStatsHourly struct { Category int32 `json:"category"` HashBytes int32 `json:"hash_bytes"` Entries int32 `json:"entries"` - Receptions *int64 `json:"receptions"` + Receptions int64 `json:"receptions"` } type AnalyticsPayloadBreakdownByIatum struct { Iata string `json:"iata"` PayloadType int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` - Count *int64 `json:"count"` + Count int64 `json:"count"` } type AnalyticsSignalStatsHourly struct { @@ -138,10 +138,10 @@ type AnalyticsSignalStatsHourly struct { Kind int32 `json:"kind"` SnrBin int32 `json:"snr_bin"` RssiBin int32 `json:"rssi_bin"` - Receptions *int64 `json:"receptions"` - SnrSamples *int64 `json:"snr_samples"` + Receptions int64 `json:"receptions"` + SnrSamples int64 `json:"snr_samples"` SnrSum *float64 `json:"snr_sum"` - RssiSamples *int64 `json:"rssi_samples"` + RssiSamples int64 `json:"rssi_samples"` RssiSum *float64 `json:"rssi_sum"` } @@ -149,9 +149,9 @@ type AnalyticsTopAdvertisersByIatum struct { Iata string `json:"iata"` NodeID uuid.UUID `json:"node_id"` Bucket pgtype.Timestamptz `json:"bucket"` - AdvertCount *int64 `json:"advert_count"` - FloodAdvertCount *int64 `json:"flood_advert_count"` - DirectAdvertCount *int64 `json:"direct_advert_count"` + AdvertCount int64 `json:"advert_count"` + FloodAdvertCount int64 `json:"flood_advert_count"` + DirectAdvertCount int64 `json:"direct_advert_count"` LastHeard pgtype.Timestamptz `json:"last_heard"` Name *string `json:"name"` NodeType *int16 `json:"node_type"` @@ -161,7 +161,7 @@ type AnalyticsTopObserversByIatum struct { Iata string `json:"iata"` ObserverID uuid.UUID `json:"observer_id"` Bucket pgtype.Timestamptz `json:"bucket"` - ObservationCount *int64 `json:"observation_count"` + ObservationCount int64 `json:"observation_count"` DisplayName *string `json:"display_name"` ObserverType *string `json:"observer_type"` } @@ -170,7 +170,7 @@ type AnalyticsTopTalkersByIatum struct { Iata string `json:"iata"` SenderName string `json:"sender_name"` Bucket pgtype.Timestamptz `json:"bucket"` - MessageCount *int64 `json:"message_count"` + MessageCount int64 `json:"message_count"` LastSent pgtype.Timestamptz `json:"last_sent"` } diff --git a/sqlc.yaml b/sqlc.yaml index 032c991..b5e86b8 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -30,3 +30,6 @@ sql: go_type: {type: "string", pointer: true} - column: "mv_top_advertisers_by_iata.name" go_type: {type: "string", pointer: true} + + - column: "mv_observer_activity_hourly.payload_type" + go_type: {type: "int16", pointer: true} From 0c19e95132760eda1bdcf550ae1b01865bbf3c41 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:14:51 -0400 Subject: [PATCH 03/11] fix: address release review follow-ups for #167 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5243068..823c04b 100644 --- a/README.md +++ b/README.md @@ -541,6 +541,8 @@ summaries retain 30 days independently of `packets.retention`. Cleanup saves onl aggregates before deleting each packet batch, in the same transaction. Raw packets, observations and message bodies still expire under packet retention. The summaries use UTC hourly buckets and appear on the normal view-refresh cycle. +`mv_hourly_iata_stats` now covers 30 days instead of migration 001's seven days; +`/stats/observations?since=` can therefore return retained summaries older than a week. Migration 039 starts from data still present; previously deleted history cannot be reconstructed. Telemetry has its own retention setting. Packet drill-down, From 04c3285f31907f745674f158d8b53581534688dd Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 21:59:48 -0400 Subject: [PATCH 04/11] fix(observers): separate packet metrics from presence updates --- .github/workflows/ci.yml | 2 +- README.md | 18 ++ .../040_observer_unknown_activity.sql | 279 ++++++++++++++++++ db/observer_metrics_integration_test.go | 87 ++++++ db/observer_retention_integration_test.go | 4 +- db/observers.go | 62 ++-- db/observers_test.go | 35 ++- db/presence.go | 11 +- db/queries/observer_activity_summary.sql | 13 + db/queries/queries.sql | 19 +- db/sqlc/mock/querier.go | 17 +- db/sqlc/observer_activity_summary.sql.go | 52 ++++ db/sqlc/querier.go | 2 + db/sqlc/queries.sql.go | 52 +++- docs/docs.go | 46 +++ docs/swagger.json | 46 +++ docs/swagger.yaml | 33 +++ internal/api/handlers/observers.go | 13 +- internal/api/handlers/observers_test.go | 20 +- internal/api/handlers/stub_reader_test.go | 6 +- internal/api/observers.go | 19 +- internal/api/reader.go | 2 +- internal/cache/cache_test.go | 2 +- internal/cache/reader.go | 6 +- internal/ingest/ingest.go | 2 +- internal/ingest/ingest_test.go | 6 +- internal/ingest/neighbors.go | 2 +- internal/ingest/packet.go | 2 +- internal/ingest/status.go | 2 +- internal/presence/broker_freshness_test.go | 50 ++++ internal/presence/coalescer.go | 27 +- 31 files changed, 844 insertions(+), 93 deletions(-) create mode 100644 db/migrations/040_observer_unknown_activity.sql create mode 100644 db/observer_metrics_integration_test.go create mode 100644 db/queries/observer_activity_summary.sql create mode 100644 db/sqlc/observer_activity_summary.sql.go create mode 100644 internal/presence/broker_freshness_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d37492..c9d71ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: Verify stats and endpoint queries against PostgreSQL 16 env: BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable - run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v + run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches)Postgres$' -count=1 -v - name: Verify backup command against PostgreSQL 16 run: | diff --git a/README.md b/README.md index 823c04b..5f539b5 100644 --- a/README.md +++ b/README.md @@ -550,3 +550,21 @@ sub-hour observer activity, exact observer comparison and current entity/scope counts continue to describe retained raw data or current entities, rather than claiming archived packet detail. Archived summaries expire without waiting for new packet deletions. A failed archive leaves its entire raw batch intact. + +### Observer monitoring metrics + +Observer activity returns complete buckets in `[windowStart, windowEnd)`, plus +`generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the +sum of stored observations in those buckets; repeated broker delivery of the +same retained packet/observer pair counts once. Unknown payload types appear as +`-1` rather than disappearing. `lastCompleteHour` uses the previous complete UTC +hour and includes its own start/end; `latestRecordedAt` is the latest retained +reception timestamp. Missing records do not prove downtime. The optional `until` +(epoch milliseconds, within the last 30 days) aligns two observers' charts. + +The existing observer `observationCount` remains a legacy cumulative presence +counter for compatibility, including status/neighbour events. It is not a +period packet total. Broker presence and packet-arrival timestamps are now +updated separately; this cannot reconstruct previously overwritten timestamps. +Migration 040 repairs archived unknown-type counts from the all-payload observer +rollup; radio samples already discarded for those legacy rows remain unknown. diff --git a/db/migrations/040_observer_unknown_activity.sql b/db/migrations/040_observer_unknown_activity.sql new file mode 100644 index 0000000..6afab7e --- /dev/null +++ b/db/migrations/040_observer_unknown_activity.sql @@ -0,0 +1,279 @@ +-- Copyright 2026 Beacon Contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +-- Unknown legacy payloads are still recorded packets. Restore their archived +-- counts from the all-payload observer rollup; their discarded radio samples +-- cannot be recovered. Subtract all existing activity (including prior repairs) +-- so migration-journal retries do not double-count or erase newer samples. +INSERT INTO analytics_observer_activity_hourly +(observer_id,payload_type,bucket,observations,airtime_n,snr_n,rssi_n) +SELECT totals.observer_id, -1, totals.bucket, totals.n-COALESCE(activity.n,0),0,0,0 +FROM (SELECT observer_id,bucket,SUM(observation_count)::bigint AS n + FROM analytics_top_observers_by_iata GROUP BY observer_id,bucket) totals +LEFT JOIN (SELECT observer_id,bucket,SUM(observations)::bigint AS n + FROM analytics_observer_activity_hourly GROUP BY observer_id,bucket) activity +USING (observer_id,bucket) +WHERE totals.n>COALESCE(activity.n,0) +ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE +SET observations=analytics_observer_activity_hourly.observations+EXCLUDED.observations; + +DROP MATERIALIZED VIEW mv_observer_activity_hourly; +CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM packet_observations +WHERE heard_at > NOW() - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); + +CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS +WITH combined AS ( + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly + UNION ALL + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly + WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +) +SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n +FROM combined GROUP BY observer_id,payload_type,bucket; + +CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); + + + +-- Preserve the bounded atomic archive, including unknown payload activity. +CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) +RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ +DECLARE hashes bytea[]; deleted bigint; +BEGIN + IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; + SELECT array_agg(p.packet_hash) INTO hashes FROM ( + SELECT ep.packet_hash FROM packets ep + WHERE ep.last_heard_at < cutoff + ORDER BY ep.last_heard_at, ep.packet_hash + LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED + ) p; + + WITH expired_observations AS MATERIALIZED ( + SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) + ), +archived_hourly_iata_stats AS ( + INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) + SELECT iata, hour, observation_count, unique_packets FROM ( +SELECT + iata, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, + COUNT(*) AS observation_count, + COUNT(DISTINCT packet_hash) AS unique_packets +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY iata, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,hour) DO UPDATE SET + observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, + unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets +), +archived_payload_breakdown_by_iata AS ( + INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) + SELECT iata, payload_type, bucket, count FROM ( +SELECT + iata, + payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS count +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' + AND payload_type IS NOT NULL +GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET + count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count +), +archived_top_observers_by_iata AS ( + INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) + SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( +SELECT + po.iata, + po.observer_id, + o.display_name, + o.observer_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*) AS observation_count +FROM expired_observations po +JOIN observers o ON o.id = po.observer_id +WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET + observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, + display_name = EXCLUDED.display_name, + observer_type = EXCLUDED.observer_type +), +archived_top_talkers_by_iata AS ( + INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) + SELECT iata, sender_name, bucket, message_count, last_sent FROM ( +SELECT + po.iata, + cm.sender_name, + date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT cm.id) AS message_count, + MAX(cm.sent_at) AS last_sent +FROM channel_messages cm +JOIN expired_observations po ON po.packet_hash = cm.packet_hash +WHERE cm.sender_name IS NOT NULL + AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') + ) batch + ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET + message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, + last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) +), +archived_top_advertisers_by_iata AS ( + INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) + SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( +SELECT + po.iata, + n.id AS node_id, + n.name, + n.node_type, + date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, + COUNT(DISTINCT p.packet_hash) AS advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, + COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, + MAX(po.heard_at) AS last_heard +FROM packets p +JOIN expired_observations po ON po.packet_hash = p.packet_hash +JOIN nodes n ON n.public_key = p.origin_pubkey +WHERE p.payload_type = 4 -- ADVERT + AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') + ) batch + ON CONFLICT (iata,node_id,bucket) DO UPDATE SET + advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, + flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, + direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, + last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), + name = EXCLUDED.name, + node_type = EXCLUDED.node_type +), +archived_observer_activity_hourly AS ( + INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) + SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( +SELECT + observer_id, + COALESCE(payload_type, -1)::smallint AS payload_type, + date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, + COUNT(*)::bigint AS observations, + SUM(airtime_ms)::real AS airtime_ms, + COUNT(airtime_ms)::bigint AS airtime_n, + SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, + COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, + MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, + SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, + COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n +FROM expired_observations +WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' +GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') + ) batch + ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET + observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, + airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, + airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, + snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, + snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, + snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), + rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, + rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n +), +archived_signal_stats_hourly AS ( + INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) + SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( +WITH samples AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + AND snr > '-Infinity'::real AND snr < 'Infinity'::real + THEN snr::double precision END AS snr, + CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) + THEN rssi::double precision END AS rssi + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), binned AS ( + SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, + width_bucket(rssi, -140, 0, 14) AS rssi_bin + FROM samples +) +SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, + COALESCE(snr_bin, -1)::integer AS snr_bin, + COALESCE(rssi_bin, -1)::integer AS rssi_bin, + count(*)::bigint AS receptions, + count(snr)::bigint AS snr_samples, + COALESCE(sum(snr), 0)::double precision AS snr_sum, + count(rssi)::bigint AS rssi_samples, + COALESCE(sum(rssi), 0)::double precision AS rssi_sum +FROM binned +GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) + ) batch + ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET + receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, + snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, + snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, + rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, + rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum +), +archived_path_stats_hourly AS ( + INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) + SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( +WITH classified AS ( + SELECT iata, heard_at, hash_size, hop_count, + CASE WHEN payload_type = 9 THEN 2 + -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). + WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 + OR NOT (path_length_byte BETWEEN 0 AND 191 + AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 + AND hash_size = (path_length_byte >> 6) + 1 + AND hop_count = (path_length_byte & 63) + AND hash_size::integer * hop_count::integer <= 64 + AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) + THEN 3 + WHEN hop_count = 0 THEN 1 + ELSE 0 END::integer AS category + FROM expired_observations + WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' + +), buckets AS ( + SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, + CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, + CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries + FROM classified +) +SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions +FROM buckets GROUP BY iata, hour, category, hash_bytes, entries + ) batch + ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET + receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions +) + DELETE FROM packets WHERE packet_hash = ANY(hashes); + GET DIAGNOSTICS deleted = ROW_COUNT; + + -- Once per completed cleanup, including when there were no raw packets to delete. + IF deleted < batch_size THEN + DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; + END IF; + RETURN deleted; +END $$; diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go new file mode 100644 index 0000000..868ce6c --- /dev/null +++ b/db/observer_metrics_integration_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package db + +import ( + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "strings" + "testing" + "time" +) + +func TestObserverMetricsPostgres(t *testing.T) { + ctx, tx := retentionTx(t) + analyticsTables(t, ctx, tx) + // Use the real observer DDL without depending on a preinstalled public schema. + initial, err := migrationFiles.ReadFile("migrations/001_initial_schema.sql") + if err != nil { + t.Fatal(err) + } + ddl := string(initial) + start := strings.Index(ddl, "CREATE TABLE observers (") + end := strings.Index(ddl[start:], "\n);") + start + 3 + if _, err := tx.Exec(ctx, "DROP TABLE observers; "+ddl[start:end]+" ALTER TABLE observers ADD COLUMN region_scope TEXT;"); err != nil { + t.Fatal(err) + } + if _, err := tx.Exec(ctx, ` + CREATE TABLE observer_brokers (observer_id uuid,broker_name text,last_seen timestamptz,last_packet_at timestamptz,PRIMARY KEY(observer_id,broker_name)); + INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); + INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,0,-100 FROM packets; + `); err != nil { + t.Fatal(err) + } + applyStatsMigration(t, ctx, tx, "039_analytics_retention.sql") + store := &Store{q: sqlc.New(tx)} + id := uuid.MustParse("00000000-0000-0000-0000-000000000001") + if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { + t.Fatal(err) + } + // Migration repairs the two unknown counts that 039 did not archive. + applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") + applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") + until := time.Now().UTC().Truncate(time.Hour) + activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until) + if err != nil { + t.Fatal(err) + } + if activity.Summary.RecordedPackets != 3 || len(activity.PayloadTypes) != 2 || activity.Summary.LatestRecordedAt != nil || activity.Summary.LastCompleteHour != 0 { + t.Fatalf("archive summary: %+v types=%+v", activity.Summary, activity.PayloadTypes) + } + if activity.WindowEnd != until.UnixMilli() || activity.WindowStart != until.Add(-7*24*time.Hour).UnixMilli() || activity.Source != "hourly" { + t.Fatalf("window: %+v", activity) + } + if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,last_heard_at) VALUES ('\xaa',NOW()); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,0,-100); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW()) ON CONFLICT DO NOTHING;`); err != nil { + t.Fatal(err) + } + raw, err := store.GetObserverActivity(ctx, id, time.Hour, 15*time.Minute, until) + if err != nil { + t.Fatal(err) + } + if raw.Summary.RecordedPackets != 1 || raw.Summary.LastCompleteHour != 1 || raw.Summary.LatestRecordedAt == nil || raw.PayloadTypes[0].PayloadType != -1 { + t.Fatalf("raw/duplicate: %+v", raw) + } + if err := store.UpsertObserverBroker(ctx, id, "one", false); err != nil { + t.Fatal(err) + } + var packet pgtype.Timestamptz + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if packet.Valid { + t.Fatal("status created a packet timestamp") + } + if err := store.UpsertObserverBroker(ctx, id, "one", true); err != nil { + t.Fatal(err) + } + if err := store.TouchObserverBrokers(ctx, []uuid.UUID{id}, []string{"one"}, []time.Time{until.Add(time.Hour)}, []time.Time{{}}); err != nil { + t.Fatal(err) + } + tx.QueryRow(ctx, "SELECT last_packet_at FROM observer_brokers").Scan(&packet) + if !packet.Valid { + t.Fatal("status erased packet timestamp") + } +} diff --git a/db/observer_retention_integration_test.go b/db/observer_retention_integration_test.go index 8672ea4..8e64ea9 100644 --- a/db/observer_retention_integration_test.go +++ b/db/observer_retention_integration_test.go @@ -100,7 +100,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil { t.Fatal(err) } - if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture"); err != nil { + if err := coalescer.UpsertObserverBroker(ctx, oldID, "fixture", true); err != nil { t.Fatal(err) } // A future cutoff simulates this cached observer having aged past retention. @@ -112,7 +112,7 @@ SELECT id,1 FROM observers WHERE display_name='expired';`) if err != nil || newID == oldID { t.Fatalf("returning observer reused a deleted ID: %v", err) } - if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture"); err != nil { + if err := coalescer.UpsertObserverBroker(ctx, newID, "fixture", true); err != nil { t.Fatal(err) } var brokers int diff --git a/db/observers.go b/db/observers.go index a3448cf..1735042 100644 --- a/db/observers.go +++ b/db/observers.go @@ -230,12 +230,37 @@ func (s *Store) GetObserverTelemetryBucketed(ctx context.Context, observerID uui // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. // Buckets of an hour or coarser come from the hourly rollup; anything finer reads observations directly. // Range and Interval are left empty for the handler to fill. -func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { +func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { obs, err := s.q.GetObserverByID(ctx, observerID) if err != nil { return nil, err } - activity := &api.ObserverActivity{} + now := time.Now().UTC() + if until.IsZero() { + until = now + } + until = until.UTC().Truncate(interval) + since := until.Add(-window) + if !since.Equal(since.Truncate(interval)) { + since = since.Truncate(interval).Add(interval) + } + activity := &api.ObserverActivity{WindowStart: since.UnixMilli(), WindowEnd: until.UnixMilli(), GeneratedAt: now.UnixMilli(), Source: "raw", Summary: &api.ObserverActivitySummary{}} + if interval >= time.Hour { + activity.Source = "hourly" + } + live, err := s.q.GetObserverActivityLiveSummary(ctx, sqlc.GetObserverActivityLiveSummaryParams{ + ObserverID: observerID, GeneratedAt: pgtype.Timestamptz{Time: now, Valid: true}, + HourStart: pgtype.Timestamptz{Time: now.Truncate(time.Hour).Add(-time.Hour), Valid: true}, HourEnd: pgtype.Timestamptz{Time: now.Truncate(time.Hour), Valid: true}}) + if err != nil { + return nil, err + } + activity.Summary.LastCompleteHour = live.LastCompleteHour + if live.LatestRecordedAt.Valid { + v := live.LatestRecordedAt.Time.UnixMilli() + activity.Summary.LatestRecordedAt = &v + } + activity.Summary.LastCompleteHourStart = now.Truncate(time.Hour).Add(-time.Hour).UnixMilli() + activity.Summary.LastCompleteHourEnd = now.Truncate(time.Hour).UnixMilli() // radio is non-nil only when airtime is actually costable, so radio != null implies costed buckets if obs.RadioSf != nil && obs.RadioBwKhz != nil && obs.RadioCr != nil && *obs.RadioSf >= 7 && *obs.RadioSf <= 12 && *obs.RadioBwKhz > 0 && *obs.RadioCr > 0 { @@ -247,12 +272,6 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w PreambleSymbols: lora.PreambleSymbols(int(*obs.RadioSf)), } } - // round the window start up to a bucket boundary so the first bucket is never a partial one - start := time.Now().Add(-window).UTC() - since := start.Truncate(interval) - if since.Before(start) { - since = since.Add(interval) - } sinceTS := pgtype.Timestamptz{Time: since, Valid: true} binWidth := pgtype.Interval{Microseconds: interval.Microseconds(), Valid: true} @@ -261,6 +280,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w ObserverID: observerID, Column2: sinceTS, Column3: binWidth, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err @@ -268,6 +288,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) for _, r := range rows { p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations if r.AirtimeN > 0 { airtime := r.AirtimeMs p.AirtimeMs = &airtime @@ -286,18 +307,20 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityHourlyPayloadTypes(ctx, sqlc.GetObserverActivityHourlyPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - if v.PayloadType == nil { - continue + payloadType := int16(-1) + if v.PayloadType != nil { + payloadType = *v.PayloadType } activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), Count: v.Count, }) } @@ -308,6 +331,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w ObserverID: observerID, Column2: sinceTS, Column3: binWidth, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err @@ -315,6 +339,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w activity.Points = make([]api.ObserverActivityPoint, 0, len(rows)) for _, r := range rows { p := api.ObserverActivityPoint{T: r.Bucket.Time.UnixMilli(), Observations: r.Observations} + activity.Summary.RecordedPackets += r.Observations if r.AirtimeN > 0 { airtime := r.AirtimeMs p.AirtimeMs = &airtime @@ -332,18 +357,20 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w typeRows, err := s.q.GetObserverActivityRawPayloadTypes(ctx, sqlc.GetObserverActivityRawPayloadTypesParams{ ObserverID: observerID, Column2: sinceTS, + Until: pgtype.Timestamptz{Time: until, Valid: true}, }) if err != nil { return nil, err } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - if v.PayloadType == nil { - continue + payloadType := int16(-1) + if v.PayloadType != nil { + payloadType = *v.PayloadType } activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ - PayloadType: *v.PayloadType, - PayloadTypeName: api.PayloadTypeName(*v.PayloadType), + PayloadType: payloadType, + PayloadTypeName: api.PayloadTypeName(payloadType), Count: v.Count, }) } @@ -424,10 +451,11 @@ func (s *Store) GetObserverRadio(ctx context.Context, observerID uuid.UUID) (ing return settings, nil } -func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (s *Store) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { params := sqlc.UpsertObserverBrokerParams{ ObserverID: observerID, BrokerName: brokerName, + IsPacket: isPacket, } return s.q.UpsertObserverBroker(ctx, params) } diff --git a/db/observers_test.go b/db/observers_test.go index 76ccf54..31b562e 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -569,6 +569,7 @@ func f32(v float32) *float32 { return &v } func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -592,7 +593,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { }, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -615,8 +616,8 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { if p.RSSIAvg != nil { t.Errorf("expected nil RSSIAvg, got %v", *p.RSSIAvg) } - if len(got.PayloadTypes) != 1 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 { - t.Fatalf("expected one payload type 4 with count 9, got %+v", got.PayloadTypes) + if len(got.PayloadTypes) != 2 || got.PayloadTypes[0].PayloadType != 4 || got.PayloadTypes[0].Count != 9 || got.PayloadTypes[1].PayloadType != -1 || got.PayloadTypes[1].Count != 3 { + t.Fatalf("expected known and unknown payloads to reconcile, got %+v", got.PayloadTypes) } if got.PayloadTypes[0].PayloadTypeName != api.PayloadTypeName(4) { t.Errorf("expected payload type name %q, got %q", api.PayloadTypeName(4), got.PayloadTypes[0].PayloadTypeName) @@ -638,6 +639,7 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -658,7 +660,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 7*24*time.Hour, 6*time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -689,6 +691,7 @@ func TestGetObserverActivity_HourlyFoldWeightedAverages(t *testing.T) { func TestGetObserverActivity_RawPath(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -709,7 +712,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{{PayloadType: i16(1), Count: 5}}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -737,6 +740,7 @@ func TestGetObserverActivity_RawPath(t *testing.T) { func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -757,7 +761,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -779,6 +783,7 @@ func TestGetObserverActivity_RawFoldNoSignal(t *testing.T) { func TestGetObserverActivity_RawPathEmpty(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -789,7 +794,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) { Return([]sqlc.GetObserverActivityRawPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute) + got, err := store.GetObserverActivity(context.Background(), observerID, 6*time.Hour, 15*time.Minute, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -804,6 +809,7 @@ func TestGetObserverActivity_RawPathEmpty(t *testing.T) { func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -814,7 +820,7 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -826,13 +832,14 @@ func TestGetObserverActivity_HourlyPathAtOneHour(t *testing.T) { func TestGetObserverActivity_UnknownObserver(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000002") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). Return(sqlc.Observer{}, pgx.ErrNoRows) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, time.Hour, time.Hour, time.Time{}) if !errors.Is(err, pgx.ErrNoRows) { t.Fatalf("expected pgx.ErrNoRows, got %v", err) } @@ -858,6 +865,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { t.Run(tc.name, func(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") mock.EXPECT().GetObserverByID(gomock.Any(), observerID). @@ -868,7 +876,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) store := &Store{q: mock} - got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour) + got, err := store.GetObserverActivity(context.Background(), observerID, 24*time.Hour, time.Hour, time.Time{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -882,6 +890,7 @@ func TestGetObserverActivity_RadioNilWhenIncomplete(t *testing.T) { func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { ctrl := gomock.NewController(t) mock := mockdb.NewMockQuerier(ctrl) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil).AnyTimes() observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") window, interval := 24*time.Hour, 6*time.Hour @@ -897,12 +906,12 @@ func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{}, nil) - before := time.Now().Add(-window) + before := time.Now().UTC().Truncate(interval).Add(-window) store := &Store{q: mock} - if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval); err != nil { + if _, err := store.GetObserverActivity(context.Background(), observerID, window, interval, time.Time{}); err != nil { t.Fatalf("unexpected error: %v", err) } - after := time.Now().Add(-window) + after := time.Now().UTC().Truncate(interval).Add(-window) if gotSince.UnixNano()%int64(interval) != 0 { t.Errorf("since %s is not aligned to %s", gotSince, interval) diff --git a/db/presence.go b/db/presence.go index ea2d6fe..4122266 100644 --- a/db/presence.go +++ b/db/presence.go @@ -20,11 +20,12 @@ func (s *Store) TouchObservers(ctx context.Context, ids []uuid.UUID, seen []time }) } -func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error { +func (s *Store) TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error { return s.q.TouchObserverBrokers(ctx, sqlc.TouchObserverBrokersParams{ Column1: ids, Column2: brokers, Column3: toTimestamptzs(seen), + Column4: nullableTimes(packets), }) } @@ -42,3 +43,11 @@ func toTimestamptzs(ts []time.Time) []pgtype.Timestamptz { } return out } + +func nullableTimes(ts []time.Time) []pgtype.Timestamptz { + out := make([]pgtype.Timestamptz, len(ts)) + for i, t := range ts { + out[i] = pgtype.Timestamptz{Time: t, Valid: !t.IsZero()} + } + return out +} diff --git a/db/queries/observer_activity_summary.sql b/db/queries/observer_activity_summary.sql new file mode 100644 index 0000000..55b5aed --- /dev/null +++ b/db/queries/observer_activity_summary.sql @@ -0,0 +1,13 @@ +-- name: GetObserverActivityLiveSummary :one +-- Two indexed ranges, bounded to one observer; no legacy presence counters. +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = @observer_id::uuid + AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index 29690a1..aaf0be8 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -275,14 +275,14 @@ SELECT COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz GROUP BY bucket ORDER BY bucket; -- name: GetObserverActivityRawPayloadTypes :many SELECT payload_type, COUNT(*)::bigint AS count FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -299,14 +299,14 @@ SELECT COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz GROUP BY 1 ORDER BY 1; -- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < @until::timestamptz GROUP BY payload_type ORDER BY count DESC; @@ -363,19 +363,20 @@ RETURNING o.id; -- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN @is_packet::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW(); + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at); -- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name; diff --git a/db/sqlc/mock/querier.go b/db/sqlc/mock/querier.go index aeae62b..cf80ee9 100644 --- a/db/sqlc/mock/querier.go +++ b/db/sqlc/mock/querier.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -source=db/sqlc/querier.go -destination=db/sqlc/mock/querier.go -package=mockdb +// mockgen -source db/sqlc/querier.go -destination db/sqlc/mock/querier.go -package mockdb // // Package mockdb is a generated GoMock package. @@ -384,6 +384,21 @@ func (mr *MockQuerierMockRecorder) GetObserverActivityHourlyPayloadTypes(ctx, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityHourlyPayloadTypes", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityHourlyPayloadTypes), ctx, arg) } +// GetObserverActivityLiveSummary mocks base method. +func (m *MockQuerier) GetObserverActivityLiveSummary(ctx context.Context, arg db.GetObserverActivityLiveSummaryParams) (db.GetObserverActivityLiveSummaryRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetObserverActivityLiveSummary", ctx, arg) + ret0, _ := ret[0].(db.GetObserverActivityLiveSummaryRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetObserverActivityLiveSummary indicates an expected call of GetObserverActivityLiveSummary. +func (mr *MockQuerierMockRecorder) GetObserverActivityLiveSummary(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObserverActivityLiveSummary", reflect.TypeOf((*MockQuerier)(nil).GetObserverActivityLiveSummary), ctx, arg) +} + // GetObserverActivityRaw mocks base method. func (m *MockQuerier) GetObserverActivityRaw(ctx context.Context, arg db.GetObserverActivityRawParams) ([]db.GetObserverActivityRawRow, error) { m.ctrl.T.Helper() diff --git a/db/sqlc/observer_activity_summary.sql.go b/db/sqlc/observer_activity_summary.sql.go new file mode 100644 index 0000000..c6c97ac --- /dev/null +++ b/db/sqlc/observer_activity_summary.sql.go @@ -0,0 +1,52 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: observer_activity_summary.sql + +package db + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = $1::uuid + AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly +` + +type GetObserverActivityLiveSummaryParams struct { + ObserverID uuid.UUID `json:"observer_id"` + GeneratedAt pgtype.Timestamptz `json:"generated_at"` + HourStart pgtype.Timestamptz `json:"hour_start"` + HourEnd pgtype.Timestamptz `json:"hour_end"` +} + +type GetObserverActivityLiveSummaryRow struct { + LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"` + LastCompleteHour int64 `json:"last_complete_hour"` +} + +// Two indexed ranges, bounded to one observer; no legacy presence counters. +func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) { + row := q.db.QueryRow(ctx, getObserverActivityLiveSummary, + arg.ObserverID, + arg.GeneratedAt, + arg.HourStart, + arg.HourEnd, + ) + var i GetObserverActivityLiveSummaryRow + err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour) + return i, err +} diff --git a/db/sqlc/querier.go b/db/sqlc/querier.go index 38e6c97..8f96cbe 100644 --- a/db/sqlc/querier.go +++ b/db/sqlc/querier.go @@ -58,6 +58,8 @@ type Querier interface { // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) + // Two indexed ranges, bounded to one observer; no legacy presence counters. + GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) // Sub-hour activity buckets straight off idx_observations_observer; no join to packets. // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 6a0eee1..3dd6de8 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -650,7 +650,7 @@ SELECT COALESCE(SUM(rssi_sum), 0)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $4::timestamptz GROUP BY 1 ORDER BY 1 ` @@ -659,6 +659,7 @@ type GetObserverActivityHourlyParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityHourlyRow struct { @@ -675,7 +676,12 @@ type GetObserverActivityHourlyRow struct { // Hour-or-coarser buckets summed from the hourly rollup; same COALESCE-plus-count shape as the raw query. func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserverActivityHourlyParams) ([]GetObserverActivityHourlyRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityHourly, arg.ObserverID, arg.Column2, arg.Column3) + rows, err := q.db.Query(ctx, getObserverActivityHourly, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + ) if err != nil { return nil, err } @@ -707,7 +713,7 @@ func (q *Queries) GetObserverActivityHourly(ctx context.Context, arg GetObserver const getObserverActivityHourlyPayloadTypes = `-- name: GetObserverActivityHourlyPayloadTypes :many SELECT payload_type, SUM(observations)::bigint AS count FROM mv_observer_activity_hourly -WHERE observer_id = $1 AND bucket >= $2::timestamptz +WHERE observer_id = $1 AND bucket >= $2::timestamptz AND bucket < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -715,6 +721,7 @@ ORDER BY count DESC type GetObserverActivityHourlyPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityHourlyPayloadTypesRow struct { @@ -723,7 +730,7 @@ type GetObserverActivityHourlyPayloadTypesRow struct { } func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2) + rows, err := q.db.Query(ctx, getObserverActivityHourlyPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) if err != nil { return nil, err } @@ -754,7 +761,7 @@ SELECT COALESCE(AVG(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0)), 0)::real AS rssi_avg, COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $4::timestamptz GROUP BY bucket ORDER BY bucket ` @@ -763,6 +770,7 @@ type GetObserverActivityRawParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` Column3 pgtype.Interval `json:"column_3"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityRawRow struct { @@ -781,7 +789,12 @@ type GetObserverActivityRawRow struct { // Aggregates are COALESCEd and paired with a count column: sqlc types a cast expression as // NOT NULL, so the counts are what tell the store a bucket had no costed or no signal rows. func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverActivityRawParams) ([]GetObserverActivityRawRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityRaw, arg.ObserverID, arg.Column2, arg.Column3) + rows, err := q.db.Query(ctx, getObserverActivityRaw, + arg.ObserverID, + arg.Column2, + arg.Column3, + arg.Until, + ) if err != nil { return nil, err } @@ -813,7 +826,7 @@ func (q *Queries) GetObserverActivityRaw(ctx context.Context, arg GetObserverAct const getObserverActivityRawPayloadTypes = `-- name: GetObserverActivityRawPayloadTypes :many SELECT payload_type, COUNT(*)::bigint AS count FROM packet_observations -WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND payload_type IS NOT NULL +WHERE observer_id = $1 AND heard_at >= $2::timestamptz AND heard_at < $3::timestamptz GROUP BY payload_type ORDER BY count DESC ` @@ -821,6 +834,7 @@ ORDER BY count DESC type GetObserverActivityRawPayloadTypesParams struct { ObserverID uuid.UUID `json:"observer_id"` Column2 pgtype.Timestamptz `json:"column_2"` + Until pgtype.Timestamptz `json:"until"` } type GetObserverActivityRawPayloadTypesRow struct { @@ -829,7 +843,7 @@ type GetObserverActivityRawPayloadTypesRow struct { } func (q *Queries) GetObserverActivityRawPayloadTypes(ctx context.Context, arg GetObserverActivityRawPayloadTypesParams) ([]GetObserverActivityRawPayloadTypesRow, error) { - rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2) + rows, err := q.db.Query(ctx, getObserverActivityRawPayloadTypes, arg.ObserverID, arg.Column2, arg.Until) if err != nil { return nil, err } @@ -4422,11 +4436,12 @@ func (q *Queries) SetPacketDecrypted(ctx context.Context, packetHash []byte) err const touchObserverBrokers = `-- name: TouchObserverBrokers :exec UPDATE observer_brokers ob SET last_seen = GREATEST(ob.last_seen, v.seen), - last_packet_at = GREATEST(ob.last_packet_at, v.seen) + last_packet_at = GREATEST(ob.last_packet_at, v.packet) FROM ( SELECT unnest($1::uuid[]) AS observer_id, unnest($2::text[]) AS broker_name, - unnest($3::timestamptz[]) AS seen + unnest($3::timestamptz[]) AS seen, + unnest($4::timestamptz[]) AS packet ) v WHERE ob.observer_id = v.observer_id AND ob.broker_name = v.broker_name ` @@ -4435,10 +4450,16 @@ type TouchObserverBrokersParams struct { Column1 []uuid.UUID `json:"column_1"` Column2 []string `json:"column_2"` Column3 []pgtype.Timestamptz `json:"column_3"` + Column4 []pgtype.Timestamptz `json:"column_4"` } func (q *Queries) TouchObserverBrokers(ctx context.Context, arg TouchObserverBrokersParams) error { - _, err := q.db.Exec(ctx, touchObserverBrokers, arg.Column1, arg.Column2, arg.Column3) + _, err := q.db.Exec(ctx, touchObserverBrokers, + arg.Column1, + arg.Column2, + arg.Column3, + arg.Column4, + ) return err } @@ -4943,22 +4964,23 @@ func (q *Queries) UpsertObserver(ctx context.Context, publicKey []byte) (Observe const upsertObserverBroker = `-- name: UpsertObserverBroker :exec INSERT INTO observer_brokers (observer_id, broker_name, last_seen, last_packet_at) -VALUES ($1, $2, NOW(), NOW()) +VALUES ($1, $2, NOW(), CASE WHEN $3::boolean THEN NOW() END) ON CONFLICT (observer_id, broker_name) DO UPDATE SET - last_seen = NOW(), - last_packet_at = NOW() + last_seen = NOW(), + last_packet_at = COALESCE(EXCLUDED.last_packet_at, observer_brokers.last_packet_at) ` type UpsertObserverBrokerParams struct { ObserverID uuid.UUID `json:"observer_id"` BrokerName string `json:"broker_name"` + IsPacket bool `json:"is_packet"` } // ============================================================ // OBSERVER BROKERS // ============================================================ func (q *Queries) UpsertObserverBroker(ctx context.Context, arg UpsertObserverBrokerParams) error { - _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName) + _, err := q.db.Exec(ctx, upsertObserverBroker, arg.ObserverID, arg.BrokerName, arg.IsPacket) return err } diff --git a/docs/docs.go b/docs/docs.go index 43ed843..bc88fbf 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1468,6 +1468,12 @@ const docTemplate = `{ "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", "name": "interval", "in": "query" + }, + { + "type": "integer", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "name": "until", + "in": "query" } ], "responses": { @@ -3924,6 +3930,7 @@ const docTemplate = `{ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -3979,6 +3986,10 @@ const docTemplate = `{ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { "type": "object", "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, "interval": { "type": "string" }, @@ -3999,6 +4010,21 @@ const docTemplate = `{ }, "range": { "type": "string" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive complete-bucket end, epoch ms", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" } } }, @@ -4046,6 +4072,26 @@ const docTemplate = `{ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 8a83f4c..75a8e5a 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1466,6 +1466,12 @@ "description": "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)", "name": "interval", "in": "query" + }, + { + "type": "integer", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "name": "until", + "in": "query" } ], "responses": { @@ -3922,6 +3928,7 @@ "type": "integer" }, "observationCount": { + "description": "legacy cumulative presence counter; includes non-packet events", "type": "integer" }, "observerType": { @@ -3977,6 +3984,10 @@ "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity": { "type": "object", "properties": { + "generatedAt": { + "description": "response computation time, not proof of continuous coverage", + "type": "integer" + }, "interval": { "type": "string" }, @@ -3997,6 +4008,21 @@ }, "range": { "type": "string" + }, + "source": { + "description": "raw or hourly; missing records do not prove an outage", + "type": "string" + }, + "summary": { + "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" + }, + "windowEnd": { + "description": "exclusive complete-bucket end, epoch ms", + "type": "integer" + }, + "windowStart": { + "description": "inclusive complete-bucket start, epoch ms", + "type": "integer" } } }, @@ -4044,6 +4070,26 @@ } } }, + "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary": { + "type": "object", + "properties": { + "lastCompleteHour": { + "type": "integer" + }, + "lastCompleteHourEnd": { + "type": "integer" + }, + "lastCompleteHourStart": { + "type": "integer" + }, + "latestRecordedAt": { + "type": "integer" + }, + "recordedPackets": { + "type": "integer" + } + } + }, "github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f03b978..cab6df7 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -521,6 +521,7 @@ definitions: description: epoch ms type: integer observationCount: + description: legacy cumulative presence counter; includes non-packet events type: integer observerType: description: e.g. "meshcoretomqtt", "meshcoreha" @@ -560,6 +561,9 @@ definitions: type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivity: properties: + generatedAt: + description: response computation time, not proof of continuous coverage + type: integer interval: type: string payloadTypes: @@ -574,6 +578,17 @@ definitions: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityRadio' range: type: string + source: + description: raw or hourly; missing records do not prove an outage + type: string + summary: + $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary' + windowEnd: + description: exclusive complete-bucket end, epoch ms + type: integer + windowStart: + description: inclusive complete-bucket start, epoch ms + type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivityPoint: properties: @@ -604,6 +619,19 @@ definitions: sf: type: integer type: object + github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary: + properties: + lastCompleteHour: + type: integer + lastCompleteHourEnd: + type: integer + lastCompleteHourStart: + type: integer + latestRecordedAt: + type: integer + recordedPackets: + type: integer + type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker: properties: lastPacketAt: @@ -2401,6 +2429,11 @@ paths: in: query name: interval type: string + - description: Optional exclusive end in epoch milliseconds; aligned down to + a complete bucket, at most 30 days old + in: query + name: until + type: integer produces: - application/json responses: diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 43147b5..15d52d6 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -256,6 +256,7 @@ var activityIntervals = map[string]time.Duration{ // @Param observerId path string true "Observer UUID" // @Param range query string false "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h" // @Param interval query string false "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)" +// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old" // @Success 200 {object} api.ObserverActivity // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError @@ -295,7 +296,17 @@ func getObserverActivity(reader api.Reader) http.HandlerFunc { respondError(w, http.StatusBadRequest, "range/interval exceeds 1000 buckets") return } - activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval) + until := time.Time{} + if values, ok := r.URL.Query()["until"]; ok { + n, err := strconv.ParseInt(values[0], 10, 64) + now := time.Now() + if len(values) != 1 || err != nil || n < 0 || n > now.UnixMilli() || n < now.Add(-720*time.Hour).UnixMilli() { + respondError(w, http.StatusBadRequest, "until must be one past epoch-millisecond timestamp within 30 days") + return + } + until = time.UnixMilli(n) + } + activity, err := reader.GetObserverActivity(r.Context(), observerID, window, interval, until) if err != nil { if errors.Is(err, pgx.ErrNoRows) { respondError(w, http.StatusNotFound, "observer not found") diff --git a/internal/api/handlers/observers_test.go b/internal/api/handlers/observers_test.go index 1bf865f..cfef1c9 100644 --- a/internal/api/handlers/observers_test.go +++ b/internal/api/handlers/observers_test.go @@ -200,7 +200,7 @@ func TestGetObserverActivity_Defaults(t *testing.T) { var gotWindow, gotInterval time.Duration r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -232,7 +232,7 @@ func TestGetObserverActivity_CustomRangeAndInterval(t *testing.T) { var gotWindow, gotInterval time.Duration r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -264,7 +264,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { t.Fatal("reader should not be called") return nil, nil }, @@ -286,7 +286,7 @@ func TestGetObserverActivity_SubHourRangeLimit(t *testing.T) { var gotWindow, gotInterval time.Duration ok := chi.NewRouter() ok.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, window, interval time.Duration, _ time.Time) (*api.ObserverActivity, error) { gotWindow, gotInterval = window, interval return &api.ObserverActivity{Points: []api.ObserverActivityPoint{}}, nil }, @@ -309,6 +309,10 @@ func TestGetObserverActivity_BadRequests(t *testing.T) { query string }{ {"invalid uuid", "not-a-uuid", ""}, + {"bad end", "00000000-0000-0000-0000-000000000001", "?until=banana"}, + {"negative end", "00000000-0000-0000-0000-000000000001", "?until=-1"}, + {"duplicate end", "00000000-0000-0000-0000-000000000001", "?until=1&until=2"}, + {"future end", "00000000-0000-0000-0000-000000000001", "?until=253402300799999"}, {"unparseable range", "00000000-0000-0000-0000-000000000001", "?range=banana"}, {"range too long", "00000000-0000-0000-0000-000000000001", "?range=721h"}, {"zero range", "00000000-0000-0000-0000-000000000001", "?range=0"}, @@ -319,7 +323,7 @@ func TestGetObserverActivity_BadRequests(t *testing.T) { t.Run(tt.name, func(t *testing.T) { r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { t.Fatal("reader should not be called") return nil, nil }, @@ -338,7 +342,7 @@ func TestGetObserverActivity_NotFound(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, pgx.ErrNoRows }, })) @@ -364,7 +368,7 @@ func TestGetObserverActivity_ReaderError(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, errors.New("boom") }, })) @@ -381,7 +385,7 @@ func TestGetObserverActivity_NullsSerialise(t *testing.T) { observerID := uuid.MustParse("00000000-0000-0000-0000-000000000001") r := chi.NewRouter() r.Get("/observers/{observerId}/activity", getObserverActivity(stubReader{ - getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { + getObserverActivity: func(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return &api.ObserverActivity{ PayloadTypes: []api.PayloadBreakdownItem{}, Points: []api.ObserverActivityPoint{{T: 1757376000000, Observations: 3}}, diff --git a/internal/api/handlers/stub_reader_test.go b/internal/api/handlers/stub_reader_test.go index 42200c1..004d94c 100644 --- a/internal/api/handlers/stub_reader_test.go +++ b/internal/api/handlers/stub_reader_test.go @@ -32,7 +32,7 @@ type stubReader struct { getObserver func(ctx context.Context, observerID uuid.UUID) (*api.Observer, error) getObserverTelemetry func(ctx context.Context, observerID uuid.UUID, since, until time.Time, afterID int64) (*api.ObserverTelemetry, error) getObserverTelemetryBucketed func(ctx context.Context, observerID uuid.UUID, since, until time.Time, bucketHours int32) ([]api.ObserverTelemetryPoint, error) - getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) + getObserverActivity func(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) getObserverScopes func(ctx context.Context, observerID uuid.UUID) ([]string, error) listObserverAdverts func(ctx context.Context, observerID uuid.UUID, cursor int64, limit int32) (api.Page[api.AdvertObservation], error) listNodes func(ctx context.Context, nodeType int16, iatas []string, supportsMultibytePaths, supportsMultibyteTraces *bool, pubkey []byte, pubkeyPrefix, name, scope string, cursor int64, limit int32, includeNeighbors bool) (api.Page[api.NodeSummary], error) @@ -186,9 +186,9 @@ func (s stubReader) GetObserverTelemetryBucketed(ctx context.Context, observerID return nil, nil } -func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { +func (s stubReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { if s.getObserverActivity != nil { - return s.getObserverActivity(ctx, observerID, window, interval) + return s.getObserverActivity(ctx, observerID, window, interval, until) } return nil, nil } diff --git a/internal/api/observers.go b/internal/api/observers.go index 206387b..d16fb13 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -44,8 +44,8 @@ type Observer struct { LastStatusAt *int64 `json:"lastStatusAt,omitempty"` // epoch ms FirstSeen int64 `json:"firstSeen"` // epoch ms LastSeen int64 `json:"lastSeen"` // epoch ms - ObservationCount int64 `json:"observationCount"` - Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on + ObservationCount int64 `json:"observationCount"` // legacy cumulative presence counter; includes non-packet events + Brokers []ObserverBroker `json:"brokers"` // broker names this observer has been seen on } // ObserverTelemetryPoint is a single telemetry snapshot for an observer. @@ -89,7 +89,22 @@ type ObserverActivityPoint struct { } // ObserverActivity is the per-observer heard-activity response. +// ObserverActivitySummary describes stored packet records, never MQTT presence events. +type ObserverActivitySummary struct { + RecordedPackets int64 `json:"recordedPackets"` + LastCompleteHour int64 `json:"lastCompleteHour"` + LastCompleteHourStart int64 `json:"lastCompleteHourStart"` + LastCompleteHourEnd int64 `json:"lastCompleteHourEnd"` + LatestRecordedAt *int64 `json:"latestRecordedAt"` +} + type ObserverActivity struct { + WindowStart int64 `json:"windowStart"` // inclusive complete-bucket start, epoch ms + WindowEnd int64 `json:"windowEnd"` // exclusive complete-bucket end, epoch ms + GeneratedAt int64 `json:"generatedAt"` // response computation time, not proof of continuous coverage + Source string `json:"source"` // raw or hourly; missing records do not prove an outage + Summary *ObserverActivitySummary `json:"summary,omitempty"` + Range string `json:"range"` Interval string `json:"interval"` Radio *ObserverActivityRadio `json:"radio"` diff --git a/internal/api/reader.go b/internal/api/reader.go index 8aa3f1a..c9a424d 100644 --- a/internal/api/reader.go +++ b/internal/api/reader.go @@ -100,7 +100,7 @@ type Reader interface { // GetObserverActivity returns bucketed heard-activity for an observer over the trailing window. // interval >= 1h is served from the hourly rollup. Returns pgx.ErrNoRows for an unknown observer. // Range and Interval on the result are left empty for the handler to fill. - GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*ObserverActivity, error) + GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*ObserverActivity, error) // GetObserverScopes returns the names of all transport scopes an observer has // been seen forwarding packets for, ordered alphabetically. diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index e58f054..21d6f3d 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -148,7 +148,7 @@ func (s *stubReader) GetObserverTelemetryBucketed(_ context.Context, _ uuid.UUID return nil, nil } -func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration) (*api.ObserverActivity, error) { +func (s *stubReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, _ time.Time) (*api.ObserverActivity, error) { return nil, nil } diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 65551da..2d31ba4 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -372,10 +372,10 @@ func (cr *CachedReader) GetObserverTelemetryBucketed(ctx context.Context, observ } // GetObserverActivity implements [api.Reader]. -func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration) (*api.ObserverActivity, error) { - key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() +func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { + key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano) return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) { - return cr.inner.GetObserverActivity(ctx, observerID, window, interval) + return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until) }) } diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 998c205..0c4d7f8 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -89,7 +89,7 @@ type DB interface { UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUID, string, error) // UpsertObserverBroker records that this observer was seen on brokerName. - UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error + UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error // UpsertIATA auto-creates an iata_codes row if it doesn't exist yet. UpsertIATA(ctx context.Context, iata string) error diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index ea206c5..2090f0e 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -201,8 +201,10 @@ func (s *stubDB) SetNodeCapability(_ context.Context, nodeID uuid.UUID, paths, t func (s *stubDB) UpsertObserver(_ context.Context, _ []byte) (uuid.UUID, string, error) { return uuid.Nil, "", nil } -func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string) error { return nil } -func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } +func (s *stubDB) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, _ bool) error { + return nil +} +func (s *stubDB) UpsertIATA(_ context.Context, _ string) error { return nil } func (s *stubDB) UpsertPacket(_ context.Context, _ UpsertPacketParams) (bool, error) { return false, nil } diff --git a/internal/ingest/neighbors.go b/internal/ingest/neighbors.go index f96a32a..3def0c9 100644 --- a/internal/ingest/neighbors.go +++ b/internal/ingest/neighbors.go @@ -48,7 +48,7 @@ func (w *Worker) handleNeighbors(ctx context.Context, iata, pubkeyHex string, ra w.log.Error(fmt.Sprintf("db: upsert observer failed in neighbors from %s", pubkeyHex), "error", err) return } - if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName); err != nil { + if err := w.db.UpsertObserverBroker(ctx, observerID, w.cfg.BrokerName, false); err != nil { w.log.Error(fmt.Sprintf("db: upsert observer broker failed in neighbors from %s", pubkeyHex), "error", err) } diff --git a/internal/ingest/packet.go b/internal/ingest/packet.go index 92f3da7..bcc9ca5 100644 --- a/internal/ingest/packet.go +++ b/internal/ingest/packet.go @@ -324,7 +324,7 @@ func (w *Worker) handlePacket(ctx context.Context, iata, pubkeyHex string, raw [ w.log.Error(fmt.Sprintf("db: upsert observer failed with packet from %s/%s", iata, pubkeyHex), "error", err) return } - err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName) + err = w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, true) if err != nil { w.log.Error(fmt.Sprintf("db: update observer broker failed with packet from %s/%s", iata, pubkeyHex), "error", err) return diff --git a/internal/ingest/status.go b/internal/ingest/status.go index 31c8064..3c693db 100644 --- a/internal/ingest/status.go +++ b/internal/ingest/status.go @@ -87,7 +87,7 @@ func (w *Worker) handleStatus(ctx context.Context, pubkeyHex string, raw []byte) if w.onObserverUpsert != nil { w.onObserverUpsert(ctx, id) } - if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName); err != nil { + if err := w.db.UpsertObserverBroker(ctx, id, w.cfg.BrokerName, false); err != nil { w.log.Error(fmt.Sprintf("db: upsert observer broker failed in status from %s", pubkeyHex), "error", err) } params := UpdateObserverStatusParams{ diff --git a/internal/presence/broker_freshness_test.go b/internal/presence/broker_freshness_test.go new file mode 100644 index 0000000..887a6e7 --- /dev/null +++ b/internal/presence/broker_freshness_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later +package presence + +import ( + "context" + "github.com/google/uuid" + "testing" + "time" +) + +type brokerTestStore struct { + Store + initial []bool + seen, packets []time.Time +} + +func (s *brokerTestStore) UpsertObserverBroker(_ context.Context, _ uuid.UUID, _ string, packet bool) error { + s.initial = append(s.initial, packet) + return nil +} +func (s *brokerTestStore) TouchObserverBrokers(_ context.Context, _ []uuid.UUID, _ []string, seen, packets []time.Time) error { + s.seen = seen + s.packets = packets + return nil +} + +func TestBrokerPresenceDoesNotInventPacketFreshness(t *testing.T) { + ctx := context.Background() + store := &brokerTestStore{} + c := New(store, time.Second, time.Minute) + now := time.Now() + c.now = func() time.Time { return now } + id := uuid.New() + c.UpsertObserverBroker(ctx, id, "one", false) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if len(store.initial) != 1 || store.initial[0] || !store.packets[0].IsZero() { + t.Fatal("status-only activity invented a packet") + } + now = now.Add(time.Minute) + packetTime := now + c.UpsertObserverBroker(ctx, id, "one", true) + now = now.Add(time.Minute) + c.UpsertObserverBroker(ctx, id, "one", false) + c.Flush(ctx) + if !store.seen[0].Equal(now) || !store.packets[0].Equal(packetTime) { + t.Fatal("later status changed packet freshness") + } +} diff --git a/internal/presence/coalescer.go b/internal/presence/coalescer.go index 9315f43..517f121 100644 --- a/internal/presence/coalescer.go +++ b/internal/presence/coalescer.go @@ -31,7 +31,7 @@ type Store interface { // TouchObserverBrokers applies coalesced last_seen/last_packet_at bumps // for the given (observer, broker) pairs in one statement. - TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen []time.Time) error + TouchObserverBrokers(ctx context.Context, ids []uuid.UUID, brokers []string, seen, packets []time.Time) error // TouchPackets applies coalesced last_heard_at bumps for the given packet // hashes in one statement. @@ -51,6 +51,8 @@ type observerBump struct { count int32 } +type brokerBump struct{ seen, packet time.Time } + type brokerKey struct { id uuid.UUID broker string @@ -68,7 +70,7 @@ type Coalescer struct { identities map[string]identity // pubkey -> observer row dirtyObservers map[uuid.UUID]observerBump knownBrokers map[brokerKey]struct{} - dirtyBrokers map[brokerKey]time.Time + dirtyBrokers map[brokerKey]brokerBump knownIATAs map[string]struct{} packetsSeen map[string]time.Time // hash -> last observation dirtyPackets map[string]time.Time @@ -86,7 +88,7 @@ func New(store Store, flushInterval, packetTTL time.Duration) *Coalescer { identities: make(map[string]identity), dirtyObservers: make(map[uuid.UUID]observerBump), knownBrokers: make(map[brokerKey]struct{}), - dirtyBrokers: make(map[brokerKey]time.Time), + dirtyBrokers: make(map[brokerKey]brokerBump), knownIATAs: make(map[string]struct{}), packetsSeen: make(map[string]time.Time), dirtyPackets: make(map[string]time.Time), @@ -122,17 +124,22 @@ func (c *Coalescer) UpsertObserver(ctx context.Context, pubkey []byte) (uuid.UUI // UpsertObserverBroker writes through the first time a pair is seen and // records a bump afterwards. -func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string) error { +func (c *Coalescer) UpsertObserverBroker(ctx context.Context, observerID uuid.UUID, brokerName string, isPacket bool) error { key := brokerKey{id: observerID, broker: brokerName} c.mu.Lock() if _, ok := c.knownBrokers[key]; ok { - c.dirtyBrokers[key] = c.now() + bump := c.dirtyBrokers[key] + bump.seen = c.now() + if isPacket { + bump.packet = bump.seen + } + c.dirtyBrokers[key] = bump c.mu.Unlock() return nil } c.mu.Unlock() - if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName); err != nil { + if err := c.Store.UpsertObserverBroker(ctx, observerID, brokerName, isPacket); err != nil { return err } c.mu.Lock() @@ -248,7 +255,7 @@ func (c *Coalescer) Flush(ctx context.Context) { brokers := c.dirtyBrokers packets := c.dirtyPackets c.dirtyObservers = make(map[uuid.UUID]observerBump) - c.dirtyBrokers = make(map[brokerKey]time.Time) + c.dirtyBrokers = make(map[brokerKey]brokerBump) c.dirtyPackets = make(map[string]time.Time) cutoff := c.now().Add(-c.packetTTL) for key, seen := range c.packetsSeen { @@ -266,12 +273,14 @@ func (c *Coalescer) Flush(ctx context.Context) { ids := make([]uuid.UUID, 0, len(brokers)) names := make([]string, 0, len(brokers)) seen := make([]time.Time, 0, len(brokers)) + packetTimes := make([]time.Time, 0, len(brokers)) for key, ts := range brokers { ids = append(ids, key.id) names = append(names, key.broker) - seen = append(seen, ts) + seen = append(seen, ts.seen) + packetTimes = append(packetTimes, ts.packet) } - if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen); err != nil { + if err := c.Store.TouchObserverBrokers(ctx, ids, names, seen, packetTimes); err != nil { slog.Error(fmt.Sprintf("presence: flush observer brokers failed (%d rows dropped)", len(ids)), "component", "presence", "error", err) } } From 7e0c48201544d1effe411f1edcdce139536a4daa Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 22:34:58 -0400 Subject: [PATCH 05/11] fix(observers): keep non-finite signal samples out of JSON --- db/observer_metrics_integration_test.go | 6 +++++- db/observers.go | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 868ce6c..f4e114e 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -3,6 +3,7 @@ package db import ( + "encoding/json" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -30,7 +31,7 @@ func TestObserverMetricsPostgres(t *testing.T) { INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) - SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,0,-100 FROM packets; + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash=int4send(2) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; `); err != nil { t.Fatal(err) } @@ -48,6 +49,9 @@ func TestObserverMetricsPostgres(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := json.Marshal(activity); err != nil { + t.Fatalf("non-finite sample leaked into JSON: %v", err) + } if activity.Summary.RecordedPackets != 3 || len(activity.PayloadTypes) != 2 || activity.Summary.LatestRecordedAt != nil || activity.Summary.LastCompleteHour != 0 { t.Fatalf("archive summary: %+v types=%+v", activity.Summary, activity.PayloadTypes) } diff --git a/db/observers.go b/db/observers.go index 1735042..09a377a 100644 --- a/db/observers.go +++ b/db/observers.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "fmt" "log/slog" + "math" "time" sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" @@ -296,7 +297,12 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w if r.SnrN > 0 { avg := r.SnrSum / float32(r.SnrN) min := r.SnrMin - p.SNRAvg, p.SNRMin = &avg, &min + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } } if r.RssiN > 0 { avg := float32(r.RssiSum) / float32(r.RssiN) @@ -346,7 +352,12 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w } if r.SnrN > 0 { avg, min := r.SnrAvg, r.SnrMin - p.SNRAvg, p.SNRMin = &avg, &min + if !math.IsNaN(float64(avg)) && !math.IsInf(float64(avg), 0) { + p.SNRAvg = &avg + } + if !math.IsNaN(float64(min)) && !math.IsInf(float64(min), 0) { + p.SNRMin = &min + } } if r.RssiN > 0 { avg := r.RssiAvg From 729d2bc8a72c99954bc7e99fd8478ce728a1d09f Mon Sep 17 00:00:00 2001 From: n30nex Date: Sat, 26 Sep 2026 22:39:21 -0400 Subject: [PATCH 06/11] test(observers): exercise non-finite raw and archived samples --- db/observer_metrics_integration_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index f4e114e..6266dff 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -31,7 +31,7 @@ func TestObserverMetricsPostgres(t *testing.T) { INSERT INTO observers (id,public_key) VALUES ('00000000-0000-0000-0000-000000000001','\x01'); INSERT INTO packets(packet_hash,payload_type,last_heard_at,first_heard_at) SELECT int4send(i),4,NOW()-interval '5 days',NOW()-interval '5 days' FROM generate_series(1,3) i; INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,payload_type,snr,rssi) - SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash=int4send(2) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; + SELECT packet_hash,'00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '5 days',0,1,0,CASE WHEN packet_hash=int4send(1) THEN 4 END,CASE WHEN packet_hash IN (int4send(1),int4send(2)) THEN 'NaN'::real ELSE 0 END,-100 FROM packets; `); err != nil { t.Fatal(err) } @@ -59,7 +59,7 @@ func TestObserverMetricsPostgres(t *testing.T) { t.Fatalf("window: %+v", activity) } if _, err := tx.Exec(ctx, `INSERT INTO packets(packet_hash,last_heard_at) VALUES ('\xaa',NOW()); - INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,0,-100); + INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at,path_length_byte,hash_size,hop_count,snr,rssi) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',date_trunc('hour',NOW())-interval '30 minutes',0,1,0,'NaN'::real,-100); INSERT INTO packet_observations(packet_hash,observer_id,iata,heard_at) VALUES ('\xaa','00000000-0000-0000-0000-000000000001','YOW',NOW()) ON CONFLICT DO NOTHING;`); err != nil { t.Fatal(err) } @@ -67,6 +67,9 @@ func TestObserverMetricsPostgres(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err := json.Marshal(raw); err != nil { + t.Fatalf("raw non-finite sample leaked into JSON: %v", err) + } if raw.Summary.RecordedPackets != 1 || raw.Summary.LastCompleteHour != 1 || raw.Summary.LatestRecordedAt == nil || raw.PayloadTypes[0].PayloadType != -1 { t.Fatalf("raw/duplicate: %+v", raw) } From 64d718220ec96f3dfb70fa334b7ed367a7a9312c Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 18:27:40 -0400 Subject: [PATCH 07/11] fix: address maintainer review for #169 --- README.md | 9 +- .../040_observer_unknown_activity.sql | 279 ------------------ db/observer_metrics_integration_test.go | 7 +- db/observer_window_test.go | 55 ++++ db/observers.go | 3 +- db/observers_test.go | 2 +- db/queries/observer_activity_summary.sql | 13 - db/queries/queries.sql | 14 + db/sqlc/models.go | 2 +- db/sqlc/observer_activity_summary.sql.go | 52 ---- db/sqlc/queries.sql.go | 39 +++ docs/docs.go | 3 +- docs/swagger.json | 3 +- docs/swagger.yaml | 4 +- internal/api/observers.go | 8 +- internal/cache/observer_window_test.go | 47 +++ internal/cache/reader.go | 3 + 17 files changed, 183 insertions(+), 360 deletions(-) delete mode 100644 db/migrations/040_observer_unknown_activity.sql create mode 100644 db/observer_window_test.go delete mode 100644 db/queries/observer_activity_summary.sql delete mode 100644 db/sqlc/observer_activity_summary.sql.go create mode 100644 internal/cache/observer_window_test.go diff --git a/README.md b/README.md index 5f539b5..773cfd9 100644 --- a/README.md +++ b/README.md @@ -553,11 +553,11 @@ new packet deletions. A failed archive leaves its entire raw batch intact. ### Observer monitoring metrics -Observer activity returns complete buckets in `[windowStart, windowEnd)`, plus +Observer activity returns buckets in `[windowStart, windowEnd)`, including the current partial bucket for live requests, plus `generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the sum of stored observations in those buckets; repeated broker delivery of the same retained packet/observer pair counts once. Unknown payload types appear as -`-1` rather than disappearing. `lastCompleteHour` uses the previous complete UTC +`-1` rather than disappearing. Freshness fields are measured at `generatedAt` even for historical `until` requests; `recordedPackets` alone follows the selected window. `lastCompleteHour` uses the previous complete UTC hour and includes its own start/end; `latestRecordedAt` is the latest retained reception timestamp. Missing records do not prove downtime. The optional `until` (epoch milliseconds, within the last 30 days) aligns two observers' charts. @@ -566,5 +566,6 @@ The existing observer `observationCount` remains a legacy cumulative presence counter for compatibility, including status/neighbour events. It is not a period packet total. Broker presence and packet-arrival timestamps are now updated separately; this cannot reconstruct previously overwritten timestamps. -Migration 040 repairs archived unknown-type counts from the all-payload observer -rollup; radio samples already discarded for those legacy rows remain unknown. +The initial analytics archive migration includes unknown-type activity. Existing +development previews that used the earlier draft require a separate operator repair; +radio samples already discarded for those legacy rows cannot be recovered. diff --git a/db/migrations/040_observer_unknown_activity.sql b/db/migrations/040_observer_unknown_activity.sql deleted file mode 100644 index 6afab7e..0000000 --- a/db/migrations/040_observer_unknown_activity.sql +++ /dev/null @@ -1,279 +0,0 @@ --- Copyright 2026 Beacon Contributors --- SPDX-License-Identifier: AGPL-3.0-or-later - --- Unknown legacy payloads are still recorded packets. Restore their archived --- counts from the all-payload observer rollup; their discarded radio samples --- cannot be recovered. Subtract all existing activity (including prior repairs) --- so migration-journal retries do not double-count or erase newer samples. -INSERT INTO analytics_observer_activity_hourly -(observer_id,payload_type,bucket,observations,airtime_n,snr_n,rssi_n) -SELECT totals.observer_id, -1, totals.bucket, totals.n-COALESCE(activity.n,0),0,0,0 -FROM (SELECT observer_id,bucket,SUM(observation_count)::bigint AS n - FROM analytics_top_observers_by_iata GROUP BY observer_id,bucket) totals -LEFT JOIN (SELECT observer_id,bucket,SUM(observations)::bigint AS n - FROM analytics_observer_activity_hourly GROUP BY observer_id,bucket) activity -USING (observer_id,bucket) -WHERE totals.n>COALESCE(activity.n,0) -ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE -SET observations=analytics_observer_activity_hourly.observations+EXCLUDED.observations; - -DROP MATERIALIZED VIEW mv_observer_activity_hourly; -CREATE OR REPLACE VIEW analytics_live_observer_activity_hourly AS -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM packet_observations -WHERE heard_at > NOW() - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC'); - -CREATE MATERIALIZED VIEW mv_observer_activity_hourly AS -WITH combined AS ( - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_live_observer_activity_hourly - UNION ALL - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM analytics_observer_activity_hourly - WHERE bucket >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -) -SELECT observer_id, payload_type, bucket, SUM(observations)::bigint AS observations, SUM(airtime_ms)::real AS airtime_ms, SUM(airtime_n)::bigint AS airtime_n, SUM(snr_sum)::real AS snr_sum, SUM(snr_n)::bigint AS snr_n, MIN(snr_min)::real AS snr_min, SUM(rssi_sum)::bigint AS rssi_sum, SUM(rssi_n)::bigint AS rssi_n -FROM combined GROUP BY observer_id,payload_type,bucket; - -CREATE UNIQUE INDEX idx_analytics_observer_activity_hourly_view ON mv_observer_activity_hourly (observer_id,payload_type,bucket); - - - --- Preserve the bounded atomic archive, including unknown payload activity. -CREATE OR REPLACE FUNCTION archive_delete_packets(cutoff timestamptz, batch_size integer) -RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ -DECLARE hashes bytea[]; deleted bigint; -BEGIN - IF batch_size < 1 THEN RAISE EXCEPTION 'batch_size must be positive'; END IF; - SELECT array_agg(p.packet_hash) INTO hashes FROM ( - SELECT ep.packet_hash FROM packets ep - WHERE ep.last_heard_at < cutoff - ORDER BY ep.last_heard_at, ep.packet_hash - LIMIT batch_size FOR UPDATE OF ep SKIP LOCKED - ) p; - - WITH expired_observations AS MATERIALIZED ( - SELECT po.* FROM packet_observations po WHERE po.packet_hash = ANY(hashes) - ), -archived_hourly_iata_stats AS ( - INSERT INTO analytics_hourly_iata_stats (iata, hour, observation_count, unique_packets) - SELECT iata, hour, observation_count, unique_packets FROM ( -SELECT - iata, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS hour, - COUNT(*) AS observation_count, - COUNT(DISTINCT packet_hash) AS unique_packets -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY iata, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,hour) DO UPDATE SET - observation_count = analytics_hourly_iata_stats.observation_count + EXCLUDED.observation_count, - unique_packets = analytics_hourly_iata_stats.unique_packets + EXCLUDED.unique_packets -), -archived_payload_breakdown_by_iata AS ( - INSERT INTO analytics_payload_breakdown_by_iata (iata, payload_type, bucket, count) - SELECT iata, payload_type, bucket, count FROM ( -SELECT - iata, - payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS count -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' - AND payload_type IS NOT NULL -GROUP BY iata, payload_type, date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (iata,payload_type,bucket) DO UPDATE SET - count = analytics_payload_breakdown_by_iata.count + EXCLUDED.count -), -archived_top_observers_by_iata AS ( - INSERT INTO analytics_top_observers_by_iata (iata, observer_id, bucket, observation_count, display_name, observer_type) - SELECT iata, observer_id, bucket, observation_count, display_name, observer_type FROM ( -SELECT - po.iata, - po.observer_id, - o.display_name, - o.observer_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*) AS observation_count -FROM expired_observations po -JOIN observers o ON o.id = po.observer_id -WHERE po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, po.observer_id, o.display_name, o.observer_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,observer_id,bucket) DO UPDATE SET - observation_count = analytics_top_observers_by_iata.observation_count + EXCLUDED.observation_count, - display_name = EXCLUDED.display_name, - observer_type = EXCLUDED.observer_type -), -archived_top_talkers_by_iata AS ( - INSERT INTO analytics_top_talkers_by_iata (iata, sender_name, bucket, message_count, last_sent) - SELECT iata, sender_name, bucket, message_count, last_sent FROM ( -SELECT - po.iata, - cm.sender_name, - date_trunc('hour', cm.sent_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT cm.id) AS message_count, - MAX(cm.sent_at) AS last_sent -FROM channel_messages cm -JOIN expired_observations po ON po.packet_hash = cm.packet_hash -WHERE cm.sender_name IS NOT NULL - AND cm.sent_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, cm.sender_name, date_trunc('hour', cm.sent_at, 'UTC') - ) batch - ON CONFLICT (iata,sender_name,bucket) DO UPDATE SET - message_count = analytics_top_talkers_by_iata.message_count + EXCLUDED.message_count, - last_sent = GREATEST(analytics_top_talkers_by_iata.last_sent, EXCLUDED.last_sent) -), -archived_top_advertisers_by_iata AS ( - INSERT INTO analytics_top_advertisers_by_iata (iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type) - SELECT iata, node_id, bucket, advert_count, flood_advert_count, direct_advert_count, last_heard, name, node_type FROM ( -SELECT - po.iata, - n.id AS node_id, - n.name, - n.node_type, - date_trunc('hour', po.heard_at, 'UTC')::timestamptz AS bucket, - COUNT(DISTINCT p.packet_hash) AS advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (0, 1)) AS flood_advert_count, - COUNT(DISTINCT p.packet_hash) FILTER (WHERE p.route_type IN (2, 3)) AS direct_advert_count, - MAX(po.heard_at) AS last_heard -FROM packets p -JOIN expired_observations po ON po.packet_hash = p.packet_hash -JOIN nodes n ON n.public_key = p.origin_pubkey -WHERE p.payload_type = 4 -- ADVERT - AND po.heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY po.iata, n.id, n.name, n.node_type, date_trunc('hour', po.heard_at, 'UTC') - ) batch - ON CONFLICT (iata,node_id,bucket) DO UPDATE SET - advert_count = analytics_top_advertisers_by_iata.advert_count + EXCLUDED.advert_count, - flood_advert_count = analytics_top_advertisers_by_iata.flood_advert_count + EXCLUDED.flood_advert_count, - direct_advert_count = analytics_top_advertisers_by_iata.direct_advert_count + EXCLUDED.direct_advert_count, - last_heard = GREATEST(analytics_top_advertisers_by_iata.last_heard, EXCLUDED.last_heard), - name = EXCLUDED.name, - node_type = EXCLUDED.node_type -), -archived_observer_activity_hourly AS ( - INSERT INTO analytics_observer_activity_hourly (observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n) - SELECT observer_id, payload_type, bucket, observations, airtime_ms, airtime_n, snr_sum, snr_n, snr_min, rssi_sum, rssi_n FROM ( -SELECT - observer_id, - COALESCE(payload_type, -1)::smallint AS payload_type, - date_trunc('hour', heard_at, 'UTC')::timestamptz AS bucket, - COUNT(*)::bigint AS observations, - SUM(airtime_ms)::real AS airtime_ms, - COUNT(airtime_ms)::bigint AS airtime_n, - SUM(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_sum, - COUNT(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS snr_n, - MIN(snr) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::real AS snr_min, - SUM(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_sum, - COUNT(rssi) FILTER (WHERE NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0))::bigint AS rssi_n -FROM expired_observations -WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days' -GROUP BY observer_id, COALESCE(payload_type, -1), date_trunc('hour', heard_at, 'UTC') - ) batch - ON CONFLICT (observer_id,payload_type,bucket) DO UPDATE SET - observations = analytics_observer_activity_hourly.observations + EXCLUDED.observations, - airtime_ms = CASE WHEN analytics_observer_activity_hourly.airtime_ms IS NULL AND EXCLUDED.airtime_ms IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.airtime_ms, 0) + COALESCE(EXCLUDED.airtime_ms, 0) END, - airtime_n = analytics_observer_activity_hourly.airtime_n + EXCLUDED.airtime_n, - snr_sum = CASE WHEN analytics_observer_activity_hourly.snr_sum IS NULL AND EXCLUDED.snr_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.snr_sum, 0) + COALESCE(EXCLUDED.snr_sum, 0) END, - snr_n = analytics_observer_activity_hourly.snr_n + EXCLUDED.snr_n, - snr_min = LEAST(analytics_observer_activity_hourly.snr_min, EXCLUDED.snr_min), - rssi_sum = CASE WHEN analytics_observer_activity_hourly.rssi_sum IS NULL AND EXCLUDED.rssi_sum IS NULL THEN NULL ELSE COALESCE(analytics_observer_activity_hourly.rssi_sum, 0) + COALESCE(EXCLUDED.rssi_sum, 0) END, - rssi_n = analytics_observer_activity_hourly.rssi_n + EXCLUDED.rssi_n -), -archived_signal_stats_hourly AS ( - INSERT INTO analytics_signal_stats_hourly (iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum) - SELECT iata, hour, kind, snr_bin, rssi_bin, receptions, snr_samples, snr_sum, rssi_samples, rssi_sum FROM ( -WITH samples AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - AND snr > '-Infinity'::real AND snr < 'Infinity'::real - THEN snr::double precision END AS snr, - CASE WHEN NOT (COALESCE(rssi, 0) = 0 AND COALESCE(snr, 0) = 0) - THEN rssi::double precision END AS rssi - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), binned AS ( - SELECT *, width_bucket(snr, -30, 30, 12) AS snr_bin, - width_bucket(rssi, -140, 0, 14) AS rssi_bin - FROM samples -) -SELECT iata, hour, grouping(snr_bin, rssi_bin)::integer AS kind, - COALESCE(snr_bin, -1)::integer AS snr_bin, - COALESCE(rssi_bin, -1)::integer AS rssi_bin, - count(*)::bigint AS receptions, - count(snr)::bigint AS snr_samples, - COALESCE(sum(snr), 0)::double precision AS snr_sum, - count(rssi)::bigint AS rssi_samples, - COALESCE(sum(rssi), 0)::double precision AS rssi_sum -FROM binned -GROUP BY GROUPING SETS ((iata, hour), (iata, hour, snr_bin), (iata, hour, rssi_bin)) - ) batch - ON CONFLICT (iata,hour,kind,snr_bin,rssi_bin) DO UPDATE SET - receptions = analytics_signal_stats_hourly.receptions + EXCLUDED.receptions, - snr_samples = analytics_signal_stats_hourly.snr_samples + EXCLUDED.snr_samples, - snr_sum = analytics_signal_stats_hourly.snr_sum + EXCLUDED.snr_sum, - rssi_samples = analytics_signal_stats_hourly.rssi_samples + EXCLUDED.rssi_samples, - rssi_sum = analytics_signal_stats_hourly.rssi_sum + EXCLUDED.rssi_sum -), -archived_path_stats_hourly AS ( - INSERT INTO analytics_path_stats_hourly (iata, hour, category, hash_bytes, entries, receptions) - SELECT iata, hour, category, hash_bytes, entries, receptions FROM ( -WITH classified AS ( - SELECT iata, heard_at, hash_size, hop_count, - CASE WHEN payload_type = 9 THEN 2 - -- Match meshcore-go IsValidPathLen (1/2/3-byte hashes, max 64 path bytes). - WHEN payload_type IS NULL OR payload_type NOT BETWEEN 0 AND 15 - OR NOT (path_length_byte BETWEEN 0 AND 191 - AND hash_size BETWEEN 1 AND 3 AND hop_count BETWEEN 0 AND 63 - AND hash_size = (path_length_byte >> 6) + 1 - AND hop_count = (path_length_byte & 63) - AND hash_size::integer * hop_count::integer <= 64 - AND COALESCE(octet_length(path_bytes), 0) = hash_size::integer * hop_count::integer) - THEN 3 - WHEN hop_count = 0 THEN 1 - ELSE 0 END::integer AS category - FROM expired_observations - WHERE heard_at >= date_trunc('hour', NOW(), 'UTC') - INTERVAL '720 hours' - -), buckets AS ( - SELECT iata, date_trunc('hour', heard_at, 'UTC') AS hour, category, - CASE WHEN category = 0 THEN hash_size ELSE 0 END::integer AS hash_bytes, - CASE WHEN category = 0 THEN hop_count ELSE 0 END::integer AS entries - FROM classified -) -SELECT iata, hour, category, hash_bytes, entries, count(*)::bigint AS receptions -FROM buckets GROUP BY iata, hour, category, hash_bytes, entries - ) batch - ON CONFLICT (iata,hour,category,hash_bytes,entries) DO UPDATE SET - receptions = analytics_path_stats_hourly.receptions + EXCLUDED.receptions -) - DELETE FROM packets WHERE packet_hash = ANY(hashes); - GET DIAGNOSTICS deleted = ROW_COUNT; - - -- Once per completed cleanup, including when there were no raw packets to delete. - IF deleted < batch_size THEN - DELETE FROM analytics_hourly_iata_stats WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_payload_breakdown_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_observers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_talkers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_top_advertisers_by_iata WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_observer_activity_hourly WHERE bucket < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_signal_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - DELETE FROM analytics_path_stats_hourly WHERE hour < date_trunc('hour', NOW(), 'UTC') - INTERVAL '30 days'; - END IF; - RETURN deleted; -END $$; diff --git a/db/observer_metrics_integration_test.go b/db/observer_metrics_integration_test.go index 6266dff..91dbeca 100644 --- a/db/observer_metrics_integration_test.go +++ b/db/observer_metrics_integration_test.go @@ -41,9 +41,10 @@ func TestObserverMetricsPostgres(t *testing.T) { if err := store.DeleteOldPackets(ctx, time.Now().Add(-72*time.Hour)); err != nil { t.Fatal(err) } - // Migration repairs the two unknown counts that 039 did not archive. - applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") - applyStatsMigration(t, ctx, tx, "040_observer_unknown_activity.sql") + // The initial archive migration retains unknown payloads without a second rebuild. + if err := store.RefreshObserverActivity(ctx); err != nil { + t.Fatal(err) + } until := time.Now().UTC().Truncate(time.Hour) activity, err := store.GetObserverActivity(ctx, id, 7*24*time.Hour, time.Hour, until) if err != nil { diff --git a/db/observer_window_test.go b/db/observer_window_test.go new file mode 100644 index 0000000..205d1ef --- /dev/null +++ b/db/observer_window_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package db + +import ( + "context" + "testing" + "time" + + sqlc "github.com/MeshCore-Beacon/beacon-server/db/sqlc" + mockdb "github.com/MeshCore-Beacon/beacon-server/db/sqlc/mock" + "github.com/google/uuid" + "go.uber.org/mock/gomock" +) + +func TestObserverActivityLiveAndFixedWindow(t *testing.T) { + for _, interval := range []time.Duration{15 * time.Minute, time.Hour, 6 * time.Hour} { + for _, fixed := range []bool{false, true} { + t.Run(interval.String()+"/"+map[bool]string{false: "live", true: "fixed"}[fixed], func(t *testing.T) { + mock := mockdb.NewMockQuerier(gomock.NewController(t)) + id := uuid.New() + mock.EXPECT().GetObserverByID(gomock.Any(), id).Return(sqlc.Observer{}, nil) + mock.EXPECT().GetObserverActivityLiveSummary(gomock.Any(), gomock.Any()).Return(sqlc.GetObserverActivityLiveSummaryRow{}, nil) + if interval < time.Hour { + mock.EXPECT().GetObserverActivityRaw(gomock.Any(), gomock.Any()).Return(nil, nil) + mock.EXPECT().GetObserverActivityRawPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) + } else { + mock.EXPECT().GetObserverActivityHourly(gomock.Any(), gomock.Any()).Return(nil, nil) + mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()).Return(nil, nil) + } + before := time.Now().UTC() + var until time.Time + if fixed { + until = before.Add(-24*time.Hour + 123*time.Millisecond) + } + got, err := (&Store{q: mock}).GetObserverActivity(context.Background(), id, 48*time.Hour, interval, until) + if err != nil { + t.Fatal(err) + } + if fixed { + if got.WindowEnd != until.Truncate(interval).UnixMilli() { + t.Fatalf("fixed end was not aligned: %d", got.WindowEnd) + } + } else if got.WindowEnd < before.UnixMilli() || got.WindowEnd > time.Now().UnixMilli() { + t.Fatalf("live end dropped the partial bucket: %d", got.WindowEnd) + } + // Freshness is always live, even when the activity window is historical. + if got.Summary.LastCompleteHourEnd != time.UnixMilli(got.GeneratedAt).UTC().Truncate(time.Hour).UnixMilli() { + t.Fatal("freshness was anchored to the historical activity window") + } + }) + } + } +} diff --git a/db/observers.go b/db/observers.go index 09a377a..36316db 100644 --- a/db/observers.go +++ b/db/observers.go @@ -239,8 +239,9 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w now := time.Now().UTC() if until.IsZero() { until = now + } else { + until = until.UTC().Truncate(interval) } - until = until.UTC().Truncate(interval) since := until.Add(-window) if !since.Equal(since.Truncate(interval)) { since = since.Truncate(interval).Add(interval) diff --git a/db/observers_test.go b/db/observers_test.go index 31b562e..44ed23c 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -919,7 +919,7 @@ func TestGetObserverActivity_SinceAlignedToInterval(t *testing.T) { if gotSince.Before(before) { t.Errorf("since %s is before the window start %s", gotSince, before) } - if !gotSince.Before(after.Add(interval)) { + if gotSince.After(after.Add(interval)) { t.Errorf("since %s is more than one interval past the window start %s", gotSince, after) } if !gotInterval.Valid || gotInterval.Microseconds != interval.Microseconds() { diff --git a/db/queries/observer_activity_summary.sql b/db/queries/observer_activity_summary.sql deleted file mode 100644 index 55b5aed..0000000 --- a/db/queries/observer_activity_summary.sql +++ /dev/null @@ -1,13 +0,0 @@ --- name: GetObserverActivityLiveSummary :one --- Two indexed ranges, bounded to one observer; no legacy presence counters. -WITH latest AS ( - SELECT heard_at FROM packet_observations - WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz - ORDER BY heard_at DESC LIMIT 1 -), hourly AS ( - SELECT COUNT(*)::bigint AS n FROM packet_observations - WHERE observer_id = @observer_id::uuid - AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz -) -SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, - hourly.n AS last_complete_hour FROM hourly; diff --git a/db/queries/queries.sql b/db/queries/queries.sql index aaf0be8..05ceacd 100644 --- a/db/queries/queries.sql +++ b/db/queries/queries.sql @@ -1572,3 +1572,17 @@ WITH target AS MATERIALIZED ( ) SELECT EXISTS(SELECT 1 FROM target) AS found, EXISTS(SELECT 1 FROM changed) AS deactivated; + +-- name: GetObserverActivityLiveSummary :one +-- Two indexed ranges, bounded to one observer; no legacy presence counters. +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = @observer_id::uuid AND heard_at <= @generated_at::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = @observer_id::uuid + AND heard_at >= @hour_start::timestamptz AND heard_at < @hour_end::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly; diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 8a24d53..378fb60 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -32,7 +32,7 @@ type AnalyticsLiveHourlyIataStat struct { type AnalyticsLiveObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` - PayloadType int16 `json:"payload_type"` + PayloadType *int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs float32 `json:"airtime_ms"` diff --git a/db/sqlc/observer_activity_summary.sql.go b/db/sqlc/observer_activity_summary.sql.go deleted file mode 100644 index c6c97ac..0000000 --- a/db/sqlc/observer_activity_summary.sql.go +++ /dev/null @@ -1,52 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.31.1 -// source: observer_activity_summary.sql - -package db - -import ( - "context" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" -) - -const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one -WITH latest AS ( - SELECT heard_at FROM packet_observations - WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz - ORDER BY heard_at DESC LIMIT 1 -), hourly AS ( - SELECT COUNT(*)::bigint AS n FROM packet_observations - WHERE observer_id = $1::uuid - AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz -) -SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, - hourly.n AS last_complete_hour FROM hourly -` - -type GetObserverActivityLiveSummaryParams struct { - ObserverID uuid.UUID `json:"observer_id"` - GeneratedAt pgtype.Timestamptz `json:"generated_at"` - HourStart pgtype.Timestamptz `json:"hour_start"` - HourEnd pgtype.Timestamptz `json:"hour_end"` -} - -type GetObserverActivityLiveSummaryRow struct { - LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"` - LastCompleteHour int64 `json:"last_complete_hour"` -} - -// Two indexed ranges, bounded to one observer; no legacy presence counters. -func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) { - row := q.db.QueryRow(ctx, getObserverActivityLiveSummary, - arg.ObserverID, - arg.GeneratedAt, - arg.HourStart, - arg.HourEnd, - ) - var i GetObserverActivityLiveSummaryRow - err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour) - return i, err -} diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 3dd6de8..489094c 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -749,6 +749,45 @@ func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg return items, nil } +const getObserverActivityLiveSummary = `-- name: GetObserverActivityLiveSummary :one +WITH latest AS ( + SELECT heard_at FROM packet_observations + WHERE observer_id = $1::uuid AND heard_at <= $2::timestamptz + ORDER BY heard_at DESC LIMIT 1 +), hourly AS ( + SELECT COUNT(*)::bigint AS n FROM packet_observations + WHERE observer_id = $1::uuid + AND heard_at >= $3::timestamptz AND heard_at < $4::timestamptz +) +SELECT (SELECT heard_at FROM latest)::timestamptz AS latest_recorded_at, + hourly.n AS last_complete_hour FROM hourly +` + +type GetObserverActivityLiveSummaryParams struct { + ObserverID uuid.UUID `json:"observer_id"` + GeneratedAt pgtype.Timestamptz `json:"generated_at"` + HourStart pgtype.Timestamptz `json:"hour_start"` + HourEnd pgtype.Timestamptz `json:"hour_end"` +} + +type GetObserverActivityLiveSummaryRow struct { + LatestRecordedAt pgtype.Timestamptz `json:"latest_recorded_at"` + LastCompleteHour int64 `json:"last_complete_hour"` +} + +// Two indexed ranges, bounded to one observer; no legacy presence counters. +func (q *Queries) GetObserverActivityLiveSummary(ctx context.Context, arg GetObserverActivityLiveSummaryParams) (GetObserverActivityLiveSummaryRow, error) { + row := q.db.QueryRow(ctx, getObserverActivityLiveSummary, + arg.ObserverID, + arg.GeneratedAt, + arg.HourStart, + arg.HourEnd, + ) + var i GetObserverActivityLiveSummaryRow + err := row.Scan(&i.LatestRecordedAt, &i.LastCompleteHour) + return i, err +} + const getObserverActivityRaw = `-- name: GetObserverActivityRaw :many SELECT date_bin($3::interval, heard_at, TIMESTAMPTZ 'epoch')::timestamptz AS bucket, diff --git a/docs/docs.go b/docs/docs.go index bc88fbf..1b27101 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -4019,7 +4019,7 @@ const docTemplate = `{ "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" }, "windowEnd": { - "description": "exclusive complete-bucket end, epoch ms", + "description": "exclusive end, epoch ms; live requests include the current partial bucket", "type": "integer" }, "windowStart": { @@ -4088,6 +4088,7 @@ const docTemplate = `{ "type": "integer" }, "recordedPackets": { + "description": "stored observations within windowStart/windowEnd", "type": "integer" } } diff --git a/docs/swagger.json b/docs/swagger.json index 75a8e5a..52df28f 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -4017,7 +4017,7 @@ "$ref": "#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary" }, "windowEnd": { - "description": "exclusive complete-bucket end, epoch ms", + "description": "exclusive end, epoch ms; live requests include the current partial bucket", "type": "integer" }, "windowStart": { @@ -4086,6 +4086,7 @@ "type": "integer" }, "recordedPackets": { + "description": "stored observations within windowStart/windowEnd", "type": "integer" } } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index cab6df7..4fda3a4 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -584,7 +584,8 @@ definitions: summary: $ref: '#/definitions/github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverActivitySummary' windowEnd: - description: exclusive complete-bucket end, epoch ms + description: exclusive end, epoch ms; live requests include the current partial + bucket type: integer windowStart: description: inclusive complete-bucket start, epoch ms @@ -630,6 +631,7 @@ definitions: latestRecordedAt: type: integer recordedPackets: + description: stored observations within windowStart/windowEnd type: integer type: object github_com_MeshCore-Beacon_beacon-server_internal_api.ObserverBroker: diff --git a/internal/api/observers.go b/internal/api/observers.go index d16fb13..2da680c 100644 --- a/internal/api/observers.go +++ b/internal/api/observers.go @@ -88,19 +88,21 @@ type ObserverActivityPoint struct { RSSIAvg *float32 `json:"rssiAvg"` } -// ObserverActivity is the per-observer heard-activity response. // ObserverActivitySummary describes stored packet records, never MQTT presence events. +// Freshness and lastCompleteHour are measured at generatedAt, even with an explicit until; +// only recordedPackets follows the selected activity window. type ObserverActivitySummary struct { - RecordedPackets int64 `json:"recordedPackets"` + RecordedPackets int64 `json:"recordedPackets"` // stored observations within windowStart/windowEnd LastCompleteHour int64 `json:"lastCompleteHour"` LastCompleteHourStart int64 `json:"lastCompleteHourStart"` LastCompleteHourEnd int64 `json:"lastCompleteHourEnd"` LatestRecordedAt *int64 `json:"latestRecordedAt"` } +// ObserverActivity is the per-observer heard-activity response. type ObserverActivity struct { WindowStart int64 `json:"windowStart"` // inclusive complete-bucket start, epoch ms - WindowEnd int64 `json:"windowEnd"` // exclusive complete-bucket end, epoch ms + WindowEnd int64 `json:"windowEnd"` // exclusive end, epoch ms; live requests include the current partial bucket GeneratedAt int64 `json:"generatedAt"` // response computation time, not proof of continuous coverage Source string `json:"source"` // raw or hourly; missing records do not prove an outage Summary *ObserverActivitySummary `json:"summary,omitempty"` diff --git a/internal/cache/observer_window_test.go b/internal/cache/observer_window_test.go new file mode 100644 index 0000000..512e580 --- /dev/null +++ b/internal/cache/observer_window_test.go @@ -0,0 +1,47 @@ +// Copyright 2026 Beacon Contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cache + +import ( + "context" + "testing" + "time" + + "github.com/MeshCore-Beacon/beacon-server/internal/api" + "github.com/google/uuid" +) + +type activityWindowReader struct { + stubReader + ends []time.Time +} + +func (r *activityWindowReader) GetObserverActivity(_ context.Context, _ uuid.UUID, _, _ time.Duration, until time.Time) (*api.ObserverActivity, error) { + r.ends = append(r.ends, until) + return &api.ObserverActivity{WindowEnd: until.UnixMilli()}, nil +} + +func TestObserverActivityCacheUsesEffectiveWindow(t *testing.T) { + client, redis := newTestClient(t) + inner := &activityWindowReader{} + reader := &CachedReader{inner: inner, c: client} + id := uuid.New() + anchor := time.Date(2026, 9, 29, 12, 0, 0, 0, time.UTC) + for _, offset := range []time.Duration{time.Second, time.Minute, 59 * time.Minute} { + if _, err := reader.GetObserverActivity(context.Background(), id, 24*time.Hour, time.Hour, anchor.Add(offset)); err != nil { + t.Fatal(err) + } + } + if len(inner.ends) != 1 || !inner.ends[0].Equal(anchor) || len(redis.Keys()) != 1 { + t.Fatalf("equivalent fixed windows missed the cache: %v, %v", inner.ends, redis.Keys()) + } + for _, until := range []time.Time{anchor.Add(time.Hour), {}, {}} { + if _, err := reader.GetObserverActivity(context.Background(), id, 24*time.Hour, time.Hour, until); err != nil { + t.Fatal(err) + } + } + if len(inner.ends) != 3 || !inner.ends[2].IsZero() || len(redis.Keys()) != 3 { + t.Fatalf("distinct/live windows aliased: %v", inner.ends) + } +} diff --git a/internal/cache/reader.go b/internal/cache/reader.go index 2d31ba4..2cec676 100644 --- a/internal/cache/reader.go +++ b/internal/cache/reader.go @@ -373,6 +373,9 @@ func (cr *CachedReader) GetObserverTelemetryBucketed(ctx context.Context, observ // GetObserverActivity implements [api.Reader]. func (cr *CachedReader) GetObserverActivity(ctx context.Context, observerID uuid.UUID, window, interval time.Duration, until time.Time) (*api.ObserverActivity, error) { + if !until.IsZero() { + until = until.UTC().Truncate(interval) + } key := keyObserverActivityPrefix + observerID.String() + ":" + window.String() + ":" + interval.String() + ":" + until.UTC().Format(time.RFC3339Nano) return getOrSet(ctx, cr.c, key, observerActivityTTL, func() (*api.ObserverActivity, error) { return cr.inner.GetObserverActivity(ctx, observerID, window, interval, until) From 3fc5600a3b3a9633bae7c8764e9b0721339b796e Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:14:51 -0400 Subject: [PATCH 08/11] fix: address release review follow-ups for #169 --- db/observers.go | 5 +---- db/sqlc/models.go | 4 ++-- db/sqlc/queries.sql.go | 4 ++-- internal/api/handlers/observers.go | 2 +- sqlc.yaml | 2 -- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/db/observers.go b/db/observers.go index 36316db..fc10efa 100644 --- a/db/observers.go +++ b/db/observers.go @@ -321,10 +321,7 @@ func (s *Store) GetObserverActivity(ctx context.Context, observerID uuid.UUID, w } activity.PayloadTypes = make([]api.PayloadBreakdownItem, 0, len(typeRows)) for _, v := range typeRows { - payloadType := int16(-1) - if v.PayloadType != nil { - payloadType = *v.PayloadType - } + payloadType := v.PayloadType activity.PayloadTypes = append(activity.PayloadTypes, api.PayloadBreakdownItem{ PayloadType: payloadType, PayloadTypeName: api.PayloadTypeName(payloadType), diff --git a/db/sqlc/models.go b/db/sqlc/models.go index 378fb60..47cce44 100644 --- a/db/sqlc/models.go +++ b/db/sqlc/models.go @@ -32,7 +32,7 @@ type AnalyticsLiveHourlyIataStat struct { type AnalyticsLiveObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` - PayloadType *int16 `json:"payload_type"` + PayloadType int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs float32 `json:"airtime_ms"` @@ -244,7 +244,7 @@ type MvHourlyIataStat struct { type MvObserverActivityHourly struct { ObserverID uuid.UUID `json:"observer_id"` - PayloadType *int16 `json:"payload_type"` + PayloadType int16 `json:"payload_type"` Bucket pgtype.Timestamptz `json:"bucket"` Observations int64 `json:"observations"` AirtimeMs float32 `json:"airtime_ms"` diff --git a/db/sqlc/queries.sql.go b/db/sqlc/queries.sql.go index 489094c..054bc00 100644 --- a/db/sqlc/queries.sql.go +++ b/db/sqlc/queries.sql.go @@ -725,8 +725,8 @@ type GetObserverActivityHourlyPayloadTypesParams struct { } type GetObserverActivityHourlyPayloadTypesRow struct { - PayloadType *int16 `json:"payload_type"` - Count int64 `json:"count"` + PayloadType int16 `json:"payload_type"` + Count int64 `json:"count"` } func (q *Queries) GetObserverActivityHourlyPayloadTypes(ctx context.Context, arg GetObserverActivityHourlyPayloadTypesParams) ([]GetObserverActivityHourlyPayloadTypesRow, error) { diff --git a/internal/api/handlers/observers.go b/internal/api/handlers/observers.go index 15d52d6..66bddd8 100644 --- a/internal/api/handlers/observers.go +++ b/internal/api/handlers/observers.go @@ -256,7 +256,7 @@ var activityIntervals = map[string]time.Duration{ // @Param observerId path string true "Observer UUID" // @Param range query string false "Trailing window as a Go duration, max 720h (default 24h); max 48h when interval is under 1h" // @Param interval query string false "Bucket size: 5m, 15m, 1h, 6h or 24h (default 15m)" -// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old" +// @Param until query int false "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until." // @Success 200 {object} api.ObserverActivity // @Failure 400 {object} handlers.APIError // @Failure 404 {object} handlers.APIError diff --git a/sqlc.yaml b/sqlc.yaml index b5e86b8..9210892 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -31,5 +31,3 @@ sql: - column: "mv_top_advertisers_by_iata.name" go_type: {type: "string", pointer: true} - - column: "mv_observer_activity_hourly.payload_type" - go_type: {type: "int16", pointer: true} From 4886e56ec37821f7d013d901c5b986a9752e98ce Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:23:16 -0400 Subject: [PATCH 09/11] docs(api): refresh generated contract after stack update --- docs/docs.go | 2 +- docs/swagger.json | 2 +- docs/swagger.yaml | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 1b27101..15be624 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1471,7 +1471,7 @@ const docTemplate = `{ }, { "type": "integer", - "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until.", "name": "until", "in": "query" } diff --git a/docs/swagger.json b/docs/swagger.json index 52df28f..96ef7bd 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1469,7 +1469,7 @@ }, { "type": "integer", - "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old", + "description": "Optional exclusive end in epoch milliseconds; aligned down to a complete bucket, at most 30 days old. Activity uses this window; summary freshness/latestRecordedAt and the last complete hour are measured at response generation time, independently of until.", "name": "until", "in": "query" } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 4fda3a4..144e621 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2432,7 +2432,9 @@ paths: name: interval type: string - description: Optional exclusive end in epoch milliseconds; aligned down to - a complete bucket, at most 30 days old + a complete bucket, at most 30 days old. Activity uses this window; summary + freshness/latestRecordedAt and the last complete hour are measured at response + generation time, independently of until. in: query name: until type: integer From ba861b8d291c33e364686785ae22e8c0a3938e11 Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:28:23 -0400 Subject: [PATCH 10/11] style: trim regenerated sqlc configuration --- sqlc.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlc.yaml b/sqlc.yaml index 9210892..032c991 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -30,4 +30,3 @@ sql: go_type: {type: "string", pointer: true} - column: "mv_top_advertisers_by_iata.name" go_type: {type: "string", pointer: true} - From 9bdc6b48042a18b2e2b39c0824ae3681a42f726c Mon Sep 17 00:00:00 2001 From: n30nex Date: Tue, 29 Sep 2026 21:35:13 -0400 Subject: [PATCH 11/11] test: mirror coalesced observer payload types --- db/observers_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/observers_test.go b/db/observers_test.go index 44ed23c..f6c67d3 100644 --- a/db/observers_test.go +++ b/db/observers_test.go @@ -588,8 +588,8 @@ func TestGetObserverActivity_HourlyFoldNoSignal(t *testing.T) { }}, nil) mock.EXPECT().GetObserverActivityHourlyPayloadTypes(gomock.Any(), gomock.Any()). Return([]sqlc.GetObserverActivityHourlyPayloadTypesRow{ - {PayloadType: i16(4), Count: 9}, - {PayloadType: nil, Count: 3}, + {PayloadType: 4, Count: 9}, + {PayloadType: -1, Count: 3}, }, nil) store := &Store{q: mock}