feat(channels): expose recorded packet scope in message delivery

.github/workflows/ci.yml

@@ -47,6 +47,9 @@ jobs:
       - name: Test
         run: go test ./...
 
+      - name: Verify scope refresh concurrency
+        run: go test -race ./internal/meshmapper ./internal/scopestore
+
       - name: Verify backup startup against PostgreSQL 16
         env:
           BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable
@@ -55,7 +58,7 @@ jobs:
       - name: Verify stats and endpoint queries against PostgreSQL 16
         env:
           BEACON_TEST_POSTGRES_DSN: postgres://postgres:backup-ci-only@127.0.0.1:5432/postgres?sslmode=disable
-        run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive)Postgres$' -count=1 -v
+        run: go test ./db -run '^Test(Signal|Paths|PacketEndpointsResolveLive|ObserverMetrics|RouteEvidence|RouteEvidenceIndex|AnalyticsRetention|AnalyticsRetentionConcurrent|DeleteOldPacketsBatches|MeshMapperCatalogue|ChannelMessageScopes)Postgres$' -count=1 -v
 
       - name: Verify backup command against PostgreSQL 16
         run: |

README.md

@@ -533,3 +533,41 @@ log:
 `LOG_LEVEL` and `LOG_FORMAT` override file settings; empty settings use `info` and `text`. Invalid values prevent startup. Configuration-loading failures can use the bootstrap text logger before file settings are available; failures after initialization retain error severity at every supported level.
 
 Records include a component field. Ingest workers also include their broker name, and HTTP completion records include the validated client address, route, status and duration. Query strings and protocol hello payloads are excluded. Expected ingest skips and routine WebSocket lifecycle details are debug-level. Changing the application's format does not change Caddy/Apache access logs or their fail2ban configuration. Collect/rotate stderr through Docker or systemd.
+
+### Analytics retention
+
+Hourly traffic, payload, observer activity, talker, advertiser, Signal and Paths
+summaries retain 30 days independently of `packets.retention`. Cleanup saves only
+aggregates before deleting each packet batch, in the same transaction. Raw
+packets, observations and message bodies still expire under packet retention.
+The summaries use UTC hourly buckets and appear on the normal view-refresh cycle.
+`mv_hourly_iata_stats` now covers 30 days instead of migration 001's seven days;
+`/stats/observations?since=` can therefore return retained summaries older than a week.
+
+Migration 039 starts from data still present; previously deleted history cannot
+be reconstructed. Telemetry has its own retention setting. Packet drill-down,
+sub-hour observer activity, exact observer comparison and current entity/scope
+counts continue to describe retained raw data or current entities, rather than
+claiming archived packet detail. Archived summaries expire without waiting for
+new packet deletions. A failed archive leaves its entire raw batch intact.
+
+### Observer monitoring metrics
+
+Observer activity returns buckets in `[windowStart, windowEnd)`, including the current partial bucket for live requests, plus
+`generatedAt`, `source` (`raw` or `hourly`) and `summary`. `recordedPackets` is the
+sum of stored observations in those buckets; repeated broker delivery of the
+same retained packet/observer pair counts once. Unknown payload types appear as
+`-1` rather than disappearing. Freshness fields are measured at `generatedAt` even for historical `until` requests; `recordedPackets` alone follows the selected window. `lastCompleteHour` uses the previous complete UTC
+hour and includes its own start/end; `latestRecordedAt` is the latest retained
+reception timestamp. Missing records do not prove downtime. The optional `until`
+(epoch milliseconds, within the last 30 days) aligns two observers' charts.
+
+The existing observer `observationCount` remains a legacy cumulative presence
+counter for compatibility, including status/neighbour events. It is not a
+period packet total. Broker presence and packet-arrival timestamps are now
+updated separately; this cannot reconstruct previously overwritten timestamps.
+The initial analytics archive migration includes unknown-type activity. Existing
+development previews that used the earlier draft require a separate operator repair;
+radio samples already discarded for those legacy rows cannot be recovered.
+
+Saved-route evidence: see [the operator guide](https://github.com/MeshCore-Beacon/beacon-docs/blob/main/app_documentation/saved-route-evidence.md).

cmd/beacon/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/MeshCore-Beacon/beacon-server/internal/ingest"
 	"github.com/MeshCore-Beacon/beacon-server/internal/keystore"
 	"github.com/MeshCore-Beacon/beacon-server/internal/logging"
+	"github.com/MeshCore-Beacon/beacon-server/internal/meshmapper"
 	"github.com/MeshCore-Beacon/beacon-server/internal/presence"
 	"github.com/MeshCore-Beacon/beacon-server/internal/scopestore"
 
@@ -189,6 +190,16 @@ func main() {
 	}
 	scopes.Load(scopeEntries)
 	slog.Info(fmt.Sprintf("loaded %d transport scopes", len(scopeEntries)), "component", "startup")
+	var scopeImporter *meshmapper.Importer
+	if cfg.MeshMapper.Scopes.Enabled {
+		restoreCtx, cancelRestore := context.WithTimeout(ctx, 10*time.Second)
+		scopeImporter, err = meshmapper.New(restoreCtx, cfg.MeshMapper.Scopes, store, scopes, scopeEntries)
+		cancelRestore()
+		if err != nil {
+			slog.Error("failed to restore MeshMapper scope catalogues", "component", "startup", "error", err)
+			os.Exit(1)
+		}
+	}
 
 	// ── Build channel keystore ──────────────────────────────────────────────
 	entries := make(map[string][]keystore.Entry)
@@ -281,6 +292,9 @@ func main() {
 	)
 
 	if cr, ok := reader.(*cache.CachedReader); ok {
+		if scopeImporter != nil {
+			scopeImporter.SetCacheInvalidator(cr.InvalidateScopeNames)
+		}
 		broker1.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver)
 		broker2.SetCacheInvalidators(cr.InvalidateNode, cr.InvalidateObserver)
 	}
@@ -301,6 +315,9 @@ func main() {
 		}
 		tasks = append(tasks, background.ObserverCleanupTask(coalescer, resolved.ObserverDeleteAfter, resolved.CleanupInterval, onDelete))
 	}
+	if scopeImporter != nil {
+		tasks = append(tasks, background.Task{Name: "meshmapper.scopes", Interval: meshmapper.PollInterval, Run: scopeImporter.Refresh})
+	}
 	scheduler := background.New(tasks)
 	go scheduler.Start(ctx)
 

config.yaml.example

@@ -214,6 +214,21 @@ cache:
 #  allow_countries: [CA, US]
 #  allow_continents: [NA]
 
+# Optional public MeshMapper scope-name discovery. Manual scopes remain authoritative.
+# No API key is needed. Sources must belong to a configured region's IATAs.
+# Group endpoints are not supported: their names cannot be attributed to each IATA.
+# At most 16 sources, 64 names per source, 64 KiB per response; no page scraping.
+# Catalogues and ETags persist in PostgreSQL. Failed refreshes retain known names.
+# Refresh success/failure/freshness is logged under component=meshmapper.scopes.
+# Removing/turning off a source deactivates its imported matching keys on restart;
+# historical scope identities and recorded evidence are retained.
+#meshmapper:
+#  scopes:
+#    enabled: false
+#    refresh_interval: 1h  # 5m-24h; one source checked per 15s tick
+#    sources:
+#      YOW: https://yow.meshmapper.net/get_scopes.php
+
 # Background task intervals.
 # Shorter intervals are useful during initial deployment to confirm data is
 # flowing. Back off to 1h or more once stable.