feat(scopes): import cached regional MeshMapper catalogues

.github/workflows/ci.yml

@@ -47,6 +47,9 @@ jobs:
       - name: Test
         run: go test ./...
 
+      - name: Verify scope refresh concurrency
+        run: go test -race ./internal/meshmapper ./internal/scopestore
+
       - name: Verify backup startup against PostgreSQL 16
         env:
           BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable
@@ -55,7 +58,7 @@ jobs:
       - name: Verify stats and endpoint queries against PostgreSQL 16
         env:
           BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable
-        run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive)Postgres$' -count=1 -v
+        run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue)Postgres$' -count=1 -v
 
       - name: Verify backup command against PostgreSQL 16
         run: |

README.md

@@ -533,3 +533,46 @@ log:
 `LOG_LEVEL` and `LOG_FORMAT` override file settings; empty settings use `info` and `text`. Invalid values prevent startup. Configuration-loading failures can use the bootstrap text logger before file settings are available; failures after initialization retain error severity at every supported level.
 
 Records include a component field. Ingest workers also include their broker name, and HTTP completion records include the validated client address, route, status and duration. Query strings and protocol hello payloads are excluded. Expected ingest skips and routine WebSocket lifecycle details are debug-level. Changing the application's format does not change Caddy/Apache access logs or their fail2ban configuration. Collect/rotate stderr through Docker or systemd.
+
+### Analytics retention
+
+Hourly traffic, payload, observer activity, talker, advertiser, Signal and Paths
+summaries retain 30 days independently of `packets.retention`. Cleanup saves only
+aggregates before deleting each packet batch, in the same transaction. Raw
+packets, observations and message bodies still expire under packet retention.
+The summaries use UTC hourly buckets and appear on the normal view-refresh cycle.
+
+Migration 039 starts from data still present; previously deleted history cannot
+be reconstructed. Telemetry has its own retention setting. Packet drill-down,
+sub-hour observer activity, exact observer comparison and current entity/scope
+counts continue to describe retained raw data or current entities, rather than
+claiming archived packet detail. Archived summaries expire without waiting for
+new packet deletions. A failed archive leaves its entire raw batch intact.
+
+### Observer monitoring metrics
+
+Observer activity returns complete buckets in `[windowStart, windowEnd)`, plus
+`generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the
+sum of stored observations in those buckets; repeated broker delivery of the
+same retained packet/observer pair counts once. Unknown payload types appear as
+`-1` rather than disappearing. `lastCompleteHour` uses the previous complete UTC
+hour and includes its own start/end; `latestRecordedAt` is the latest retained
+reception timestamp. Missing records do not prove downtime. The optional `until`
+(epoch milliseconds, within the last 30 days) aligns two observers' charts.
+
+The existing observer `observationCount` remains a legacy cumulative presence
+counter for compatibility, including status/neighbour events. It is not a
+period packet total. Broker presence and packet-arrival timestamps are now
+updated separately; this cannot reconstruct previously overwritten timestamps.
+Migration 040 repairs archived unknown-type counts from the all-payload observer
+rollup; radio samples already discarded for those legacy rows remain unknown.
+
+### Saved-route observation evidence
+
+Known-route responses include `pathKey`, a stable identity within the route's IATA. Use `GET /api/v1/routes/{iata}/{pathKey}/observations` to fetch the **full saved route** and retained report references. Search results can contain a subsegment while sharing the full route's key; the evidence response always describes the complete saved sequence.
+
+The match requires the complete saved `pathBytes`, `hashSize`, hop count and IATA. A compact digest index narrows candidates, but full bytes are still compared. Other hash widths, TRACE readings/intended routes and unclassified legacy observations are excluded. Matching short prefixes does not confirm historical node identities, forwarding or delivery. The stored route counter can include repeated processing and outlive raw reports; it is not a retained-result total.
+
+`range` defaults to `24h` and accepts durations up to `720h`, anchored on the server. Alternatively supply both `since` and exclusive `until` in epoch milliseconds, with a maximum 30-day span and no future end. `limit` defaults to 50 and is capped at 200. Follow `nextPageCursor` as `pageCursor`; its route, window and microsecond/ID boundary are pinned. Do not combine it with another range, or change its explicit window. Numeric legacy `cursor` is unsupported. Responses include effective window bounds, `matchAvailable`, an empty `items` array when no matching raw evidence remains, and `hasMore`; no total-count scan or packet/message body is added. Malformed saved path metadata is explicitly unavailable, and missing routes return 404.
+
+Migration 041 builds the compact observation index concurrently. Keep it as a single statement outside a transaction; the existing runner handles an interrupted or already-built index before recording completion. It does not alter retained rows or expiry configuration. Native PostgreSQL tests cover ties below millisecond precision, cursor scope, different widths/sites, TRACE/unknown exclusions, raw expiry, index retry and custom/generic indexed plans.

cmd/beacon/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/MeshCore-Beacon/beacon-server/internal/ingest"
 	"github.com/MeshCore-Beacon/beacon-server/internal/keystore"
 	"github.com/MeshCore-Beacon/beacon-server/internal/logging"
+	"github.com/MeshCore-Beacon/beacon-server/internal/meshmapper"
 	"github.com/MeshCore-Beacon/beacon-server/internal/presence"
 	"github.com/MeshCore-Beacon/beacon-server/internal/scopestore"
 
@@ -189,6 +190,16 @@ func main() {
 	}
 	scopes.Load(scopeEntries)
 	slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup")
+	var scopeImporter *meshmapper.Importer
+	if cfg.MeshMapper.Scopes.Enabled {
+		restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second)
+		scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, scopes, scopeEntries)
+		cancelRestore()
+		if err != nil {
+			slog.Error("failed to restore MeshMapper scope catalogues", "component", "startup", "error", err)
+			os.Exit(1)
+		}
+	}
 
 	// ── Build channel keystore ──────────────────────────────────────────────
 	entries := make(map[string][]keystore.Entry)
@@ -281,6 +292,9 @@ func main() {
 	)
 
 	if cr, ok := reader.(*cache.CachedReader); ok {
+		if scopeImporter != nil {
+			scopeImporter.SetCacheInvalidator(cr.InvalidateScopeNames)
+		}
 		broker1.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver)
 		broker2.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver)
 	}
@@ -301,6 +315,9 @@ func main() {
 		}
 		tasks = append(tasks, background.ObserverCleanupTask(coalescer, resolved.ObserverDeleteAfter, resolved.CleanupInterval, onDelete))
 	}
+	if scopeImporter != nil {
+		tasks = append(tasks, background.Task{Name: "meshmapper.scopes", Interval: meshmapper.PollInterval, Run: scopeImporter.Refresh})
+	}
 	scheduler := background.New(tasks)
 	go scheduler.Start(ctx)
 

config.yaml.example

@@ -214,6 +214,21 @@ cache:
 #  allow_countries: [CA, US]
 #  allow_continents: [NA]
 
+# Optional public MeshMapper scope-name discovery. Manual scopes remain authoritative.
+# No API key is needed. Sources must belong to a configured region's IATAs.
+# Group endpoints are not supported: their names cannot be attributed to each IATA.
+# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping.
+# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names.
+# Refresh success/failure/freshness is logged under component=meshmapper.scopes.
+# Removing/turning off a source deactivates its imported matching keys on restart;
+# historical scope identities and recorded evidence are retained.
+#meshmapper:
+#  scopes:
+#    enabled: false
+#    refresh_interval: 1h  # 5m-24h; one source checked per 15s tick
+#    sources:
+#      YOW: https://yow.meshmapper.net/get_scopes.php
+
 # Background task intervals.
 # Shorter intervals are useful during initial deployment to confirm data is
 # flowing. Back off to 1h or more once stable.