From 319b3d3e7a1ab2974b532ac54ae18f030ce61ae5 Mon Sep 17 00:00:00 2001 From: dborup Date: Thu, 8 Oct 2026 13:41:57 +0200 Subject: [PATCH 1/2] Add observer declaration scope audit --- cmd/server/openapi.go | 1 + cmd/server/routes.go | 2 + cmd/server/scope_audit.go | 440 +++++++++++++++++++++++++ cmd/server/scope_audit_test.go | 359 ++++++++++++++++++++ docs/user-guide/analytics.md | 18 + public/analytics.js | 58 +++- public/index.html | 1 + public/scope-audit.js | 151 +++++++++ public/style.css | 15 + test-all.sh | 1 + test-analytics-subtab-deeplinks-205.js | 30 ++ test-e2e-playwright.js | 2 + test-scope-audit.js | 77 +++++ tools/scope-audit-e2e.js | 82 +++++ 14 files changed, 1232 insertions(+), 5 deletions(-) create mode 100644 cmd/server/scope_audit.go create mode 100644 cmd/server/scope_audit_test.go create mode 100644 public/scope-audit.js create mode 100644 test-scope-audit.js create mode 100644 tools/scope-audit-e2e.js diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 5525579bd..3da8503fc 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -147,6 +147,7 @@ func routeDescriptions() map[string]routeMeta { {Name: "from", Description: "RFC3339 start; if from or to is given, window is ignored (an unparseable value leaves that bound open)", Type: "string"}, {Name: "to", Description: "RFC3339 end; if from or to is given, window is ignored (an unparseable value leaves that bound open)", Type: "string"}, }}, + "GET /api/scope-audit": {Summary: "Declared versus observed forwarding scopes", Description: "Observer-confirmed scope declarations compared with collision-safe flood path evidence. Missing observations are not proof of a configuration error. Unknown scopes, ambiguous hops, absent observations, and declarations older than seven days make evidence incomplete. Observer scope lists may be truncated without a completeness flag. Cached 30 seconds for each of three supported windows; identity visibility is applied live.", Tag: "analytics", QueryParams: []paramMeta{{Name: "window", Description: "Observation window: 1h, 24h (default), or 7d", Type: "string"}}}, "GET /api/analytics/wardriving": {Summary: "Wardriving channel analytics", Description: "Activity/entry-point/coverage/signal/session analytics for the #wardriving channel (or another channel via ?channel=): message volume over time, top senders, path[0] entry-point hash-prefix tallies (resolve names via /api/resolve-hops), per-observer coverage (observer's known IATA-derived coordinates, not the sender's — MeshMapper's wardriving messages normally carry an anonymous session token, not live GPS), average SNR/RSSI over the same time buckets as the activity series, each sender's messages grouped into distinct sessions/runs (split on a 15-minute gap, each with an AirtimeMs field — LoRa Time-on-Air × distinct relaying repeaters, same formula as the Overview tab's Relay Airtime Share, omitted in DB-only mode), and any senders who explicitly shared their own position (some clients append plaintext \",\" after the token — a deliberate choice by that sender, confirmed empirically, not something CoreScope infers). Cached 30s per window+channel.", Tag: "analytics", QueryParams: []paramMeta{ {Name: "window", Description: "Time window: 1h, 24h (default), or 7d", Type: "string"}, diff --git a/cmd/server/routes.go b/cmd/server/routes.go index c67e817ba..0eb450dcf 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -79,6 +79,7 @@ type Server struct { advertRoutes nodeAdvertRouteCache // Cached /api/scope-stats response — per-window, recomputed at most once every 30s + scopeAudit scopeAuditCache scopeStatsMu sync.Mutex scopeStatsCache map[string]*ScopeStatsResponse scopeStatsCachedAt map[string]time.Time @@ -319,6 +320,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/health", s.handleHealth).Methods("GET") r.HandleFunc("/api/stats", s.handleStats).Methods("GET") r.HandleFunc("/api/scope-stats", s.handleScopeStats).Methods("GET") + r.HandleFunc("/api/scope-audit", s.handleScopeAudit).Methods("GET") r.HandleFunc("/api/analytics/hop-depth", s.handleHopDepthAnalytics).Methods("GET") r.HandleFunc("/api/analytics/wardriving", s.handleWardrivingStats).Methods("GET") r.HandleFunc("/api/analytics/wardriving/sender-messages", s.handleWardrivingSenderMessages).Methods("GET") diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go new file mode 100644 index 000000000..c2b2e3578 --- /dev/null +++ b/cmd/server/scope_audit.go @@ -0,0 +1,440 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "net/http" + "sort" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +// The observer supplies a snapshot, not a continuously verified declaration. +// Keep the initial evidence-age threshold explicit; a future customizer can +// expose it without changing the findings themselves. +const scopeAuditDeclarationMaxAge = 7 * 24 * time.Hour +const scopeAuditTTL = 30 * time.Second +const scopeAuditQueryTimeout = 20 * time.Second + +type ScopeAuditCount struct { + Name string `json:"name"` + Count int `json:"count"` +} +type ScopeAuditRow struct { + PublicKey string `json:"publicKey"` + Name string `json:"name"` + Role string `json:"role"` + ConfiguredScope string `json:"configuredScope"` + ConfiguredScopeAt string `json:"configuredScopeAt"` + DeclaredScopes []string `json:"declaredScopes"` + AllowsUnscoped bool `json:"allowsUnscoped"` + DeclarationAgeSeconds *int64 `json:"declarationAgeSeconds"` + StaleDeclaration bool `json:"staleDeclaration"` + ObservedScopes []ScopeAuditCount `json:"observedScopes"` + UnscopedObserved int `json:"unscopedObserved"` + UnknownScopeObserved int `json:"unknownScopeObserved"` + AmbiguousHops int `json:"ambiguousHops"` + Forwarded int `json:"forwarded"` + NotObserved []string `json:"notObserved"` + UndeclaredObserved []string `json:"undeclaredObserved"` + WildcardContradiction bool `json:"wildcardContradiction"` + IncompleteEvidence bool `json:"incompleteEvidence"` + Status string `json:"status"` +} +type ScopeAuditSummary struct { + Total int `json:"total"` + NotObserved int `json:"notObserved"` + UndeclaredObserved int `json:"undeclaredObserved"` + WildcardContradiction int `json:"wildcardContradiction"` + Incomplete int `json:"incomplete"` + StaleDeclarations int `json:"staleDeclarations"` +} +type ScopeAuditResponse struct { + Window string `json:"window"` + GeneratedAt string `json:"generatedAt"` + Since string `json:"since"` + Summary ScopeAuditSummary `json:"summary"` + AmbiguousHops int `json:"ambiguousHops"` + Rows []ScopeAuditRow `json:"rows"` +} +type scopeAuditCacheEntry struct { + response *ScopeAuditResponse + at time.Time +} +type scopeAuditCache struct { + mu sync.Mutex + entries map[string]scopeAuditCacheEntry + sf singleflight.Group +} + +func (c *scopeAuditCache) get(window string) *ScopeAuditResponse { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[window] + if ok && time.Since(e.at) < scopeAuditTTL { + return e.response + } + return nil +} + +// load rechecks after entering singleflight so a concurrent cache fill is shared. +func (c *scopeAuditCache) load(window string, build func() (*ScopeAuditResponse, error)) (*ScopeAuditResponse, error) { + if cached := c.get(window); cached != nil { + return cached, nil + } + value, err, _ := c.sf.Do(window, func() (interface{}, error) { + if cached := c.get(window); cached != nil { + return cached, nil + } + response, err := build() + if err != nil { + return nil, err + } + c.mu.Lock() + if c.entries == nil { + c.entries = make(map[string]scopeAuditCacheEntry, 3) + } + c.entries[window] = scopeAuditCacheEntry{response: response, at: time.Now()} + c.mu.Unlock() + return response, nil + }) + if err != nil { + return nil, err + } + return value.(*ScopeAuditResponse), nil +} + +func scopeAuditDuration(window string) time.Duration { + switch window { + case "1h": + return time.Hour + case "7d": + return 7 * 24 * time.Hour + default: + return 24 * time.Hour + } +} +func normalizeAuditScope(name string) string { return strings.TrimPrefix(strings.TrimSpace(name), "#") } +func declaredAuditScopes(raw string) ([]string, bool) { + set := make(map[string]struct{}) + wild := false + for _, part := range strings.Split(raw, ",") { + name := normalizeAuditScope(part) + if name == "*" { + wild = true + } else if name != "" { + set[name] = struct{}{} + } + } + scopes := make([]string, 0, len(set)) + for name := range set { + scopes = append(scopes, name) + } + sort.Strings(scopes) + return scopes, wild +} + +type scopeAuditAccumulator struct { + row ScopeAuditRow + counts map[string]int +} + +func (s *Server) buildScopeAudit(window string, now time.Time) (*ScopeAuditResponse, error) { + ctx, cancel := context.WithTimeout(context.Background(), scopeAuditQueryTimeout) + defer cancel() + return s.buildScopeAuditContext(ctx, window, now) +} + +// One node snapshot and one streaming observation query, never a scan per node. +// Complexity is O(nodes + observed path hops + distinct ambiguous-prefix +// candidates), with token resolution memoized. Dedup sets hold one transmission +// at a time because SQL orders by t.id. The response/cache has at most 3 windows. +func (s *Server) buildScopeAuditContext(ctx context.Context, window string, now time.Time) (*ScopeAuditResponse, error) { + if !s.db.hasConfiguredScope() || !s.db.hasScopeName() { + return nil, fmt.Errorf("scope audit requires configured_scope and scope_name; run ingestor migrations") + } + resp := &ScopeAuditResponse{Window: window, GeneratedAt: now.UTC().Format(time.RFC3339), Since: now.Add(-scopeAuditDuration(window)).UTC().Format(time.RFC3339), Rows: []ScopeAuditRow{}} + rows, err := s.db.conn.QueryContext(ctx, `SELECT public_key,COALESCE(name,''),COALESCE(role,''),configured_scope,COALESCE(configured_scope_at,'') FROM nodes ORDER BY public_key`) + if err != nil { + return nil, err + } + nodes := []nodeInfo{} + targets := make(map[string]*scopeAuditAccumulator) + for rows.Next() { + var pk, name, role, at string + var scope sql.NullString + if err = rows.Scan(&pk, &name, &role, &scope, &at); err != nil { + rows.Close() + return nil, err + } + pk = strings.ToLower(pk) + // Include undeclared and hidden nodes in the collision universe. Visibility is + // applied only when serving the finished result, including cache hits. + nodes = append(nodes, nodeInfo{PublicKey: pk, Name: name, Role: role}) + if !scope.Valid { + continue + } + declared, wild := declaredAuditScopes(scope.String) + row := ScopeAuditRow{PublicKey: pk, Name: name, Role: role, ConfiguredScope: scope.String, ConfiguredScopeAt: at, DeclaredScopes: declared, AllowsUnscoped: wild, ObservedScopes: []ScopeAuditCount{}, NotObserved: []string{}, UndeclaredObserved: []string{}, StaleDeclaration: true} + if stamp, e := parseAnyRFC3339(at); e == nil && !stamp.After(now) { + age := int64(now.Sub(stamp).Seconds()) + row.DeclarationAgeSeconds = &age + row.StaleDeclaration = now.Sub(stamp) > scopeAuditDeclarationMaxAge + } + targets[pk] = &scopeAuditAccumulator{row: row, counts: make(map[string]int)} + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, err + } + // With no confirmed declaration there is nothing to compare. + if len(targets) == 0 { + return resp, nil + } + pm := buildPrefixMap(nodes) + resolve := newRelayTokenResolver(pm) + // Respect listener-only identities just as the normal relay attribution does. + listeners, err := s.db.GetNonRelayObserverPubkeys() + if err != nil { + return nil, err + } + pm.markNonRelay(listeners) + rawCol := "''" + if s.db.hasObsRawHex() { + rawCol = "COALESCE(o.raw_hex,'')" + } + // Observation timestamps, rather than first_seen, include retransmissions + // heard inside the selected window even when their payload was first seen earlier. + // substr retains only header/transport bytes, avoiding loading full raw frames. + query := `SELECT t.id,t.route_type,t.scope_name,substr(t.raw_hex,1,10),COALESCE(o.path_json,''),substr(` + rawCol + `,1,10) + FROM transmissions t JOIN observations o ON o.transmission_id=t.id + WHERE o.timestamp >= ? AND o.timestamp <= ? ORDER BY t.id` + var sinceArg, untilArg interface{} = now.Add(-scopeAuditDuration(window)).Unix(), now.Unix() + if !s.db.isV3() { + sinceArg, untilArg = resp.Since, resp.GeneratedAt + } + obs, err := s.db.conn.QueryContext(ctx, query, sinceArg, untilArg) + if err != nil { + return nil, err + } + defer obs.Close() + lastID := -1 + seen := make(map[string]bool) + ambSeen := make(map[string]bool) + ambCounts := make(map[string]int) + for obs.Next() { + var id int + var route sql.NullInt64 + var scope sql.NullString + var canonical, path, raw string + if err = obs.Scan(&id, &route, &scope, &canonical, &path, &raw); err != nil { + return nil, err + } + if id != lastID { + clear(seen) + clear(ambSeen) + lastID = id + } + effectiveRoute := int(route.Int64) + if !route.Valid { + effectiveRoute = -1 + } + if len(raw) >= 2 { + header, e := strconv.ParseUint(raw[:2], 16, 8) + if e != nil { + continue + } + effectiveRoute = int(header & 3) + } + if effectiveRoute != 0 && effectiveRoute != 1 { + continue + } // Packet::isRouteFlood, firmware/src/Packet.h. + kind := normalizeAuditScope(scope.String) + if effectiveRoute == 1 { + kind = "*" + } else if !scope.Valid || kind == "" || len(raw) > 0 && (len(raw) < 6 || len(canonical) < 6 || !strings.EqualFold(raw[2:6], canonical[2:6])) { + kind = "" + } + visitPathJSONHops(path, func(token string) bool { + if len(resolve.cache) >= 4096 { + clear(resolve.cache) + } + key := resolve.key(token) + if key == "" { + token = strings.ToLower(token) + if len(pm.m[token]) > 1 && !ambSeen[token] { + ambSeen[token] = true + ambCounts[token]++ + resp.AmbiguousHops++ + } + return true + } + target := targets[key] + if target == nil { + return true + } + // Legacy observations lack their own header; retain the evidence caveat. + if raw == "" { + target.row.IncompleteEvidence = true + } + // Same target/content transmission is counted once, even when observed by + // several listeners, or repeated in a looped path. Different scope evidence + // remains visible because a payload may cross scoped and unscoped boundaries. + if !seen[key] { + seen[key] = true + target.row.Forwarded++ + } + evidence := key + "|" + kind + if seen[evidence] { + return true + } + seen[evidence] = true + switch kind { + case "*": + target.row.UnscopedObserved++ + case "": + target.row.UnknownScopeObserved++ + default: + target.counts[kind]++ + } + return true + }) + } + if err = obs.Err(); err != nil { + return nil, err + } + // Expand collided tokens only once, after aggregation. A collision on an + // unrelated prefix cannot mark every declared node's evidence incomplete. + for token, count := range ambCounts { + for _, n := range pm.m[token] { + if target := targets[strings.ToLower(n.PublicKey)]; target != nil { + target.row.AmbiguousHops += count + } + } + } + for _, target := range targets { + finishScopeAuditRow(target) + resp.Rows = append(resp.Rows, target.row) + } + sort.Slice(resp.Rows, func(i, j int) bool { + a, b := resp.Rows[i], resp.Rows[j] + if auditPriority(a) != auditPriority(b) { + return auditPriority(a) > auditPriority(b) + } + return a.PublicKey < b.PublicKey + }) + resp.Summary = summarizeScopeAudit(resp.Rows) + return resp, nil +} + +// Findings outrank quiet declarations; incompleteness remains explicit in status. +func auditPriority(r ScopeAuditRow) int { + if r.WildcardContradiction || len(r.UndeclaredObserved) > 0 { + return 4 + } + if r.UnknownScopeObserved > 0 || r.AmbiguousHops > 0 { + return 3 + } + if r.Forwarded > 0 && len(r.NotObserved) > 0 { + return 2 + } + if r.Forwarded > 0 { + return 1 + } + return 0 +} +func finishScopeAuditRow(a *scopeAuditAccumulator) { + r := &a.row + declared := make(map[string]bool, len(r.DeclaredScopes)) + for _, name := range r.DeclaredScopes { + declared[name] = true + if a.counts[name] == 0 { + r.NotObserved = append(r.NotObserved, name) + } + } + for name, count := range a.counts { + r.ObservedScopes = append(r.ObservedScopes, ScopeAuditCount{Name: name, Count: count}) + if !declared[name] { + r.UndeclaredObserved = append(r.UndeclaredObserved, name) + } + } + sort.Slice(r.ObservedScopes, func(i, j int) bool { return r.ObservedScopes[i].Name < r.ObservedScopes[j].Name }) + sort.Strings(r.UndeclaredObserved) + r.WildcardContradiction = !r.AllowsUnscoped && r.UnscopedObserved > 0 + r.IncompleteEvidence = r.IncompleteEvidence || r.Forwarded == 0 || r.UnknownScopeObserved > 0 || r.AmbiguousHops > 0 || r.StaleDeclaration + switch { + case r.Forwarded == 0: + r.Status = "no-evidence" + case r.IncompleteEvidence: + r.Status = "incomplete" + case len(r.NotObserved) > 0 || len(r.UndeclaredObserved) > 0 || r.WildcardContradiction: + r.Status = "findings" + default: + r.Status = "consistent" + } +} +func summarizeScopeAudit(rows []ScopeAuditRow) ScopeAuditSummary { + summary := ScopeAuditSummary{Total: len(rows)} + for _, r := range rows { + if len(r.NotObserved) > 0 { + summary.NotObserved++ + } + if len(r.UndeclaredObserved) > 0 { + summary.UndeclaredObserved++ + } + if r.WildcardContradiction { + summary.WildcardContradiction++ + } + if r.IncompleteEvidence { + summary.Incomplete++ + } + if r.StaleDeclaration { + summary.StaleDeclarations++ + } + } + return summary +} +func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { + window := r.URL.Query().Get("window") + if window == "" { + window = "24h" + } + if window != "1h" && window != "24h" && window != "7d" { + writeError(w, 400, "window must be 1h, 24h, or 7d") + return + } + response, err := s.scopeAudit.load(window, func() (*ScopeAuditResponse, error) { + ctx, cancel := context.WithTimeout(r.Context(), scopeAuditQueryTimeout) + defer cancel() + return s.buildScopeAuditContext(ctx, window, time.Now()) + }) + if err != nil { + writeError(w, 500, err.Error()) + return + } + keys := make([]string, len(response.Rows)) + for i, row := range response.Rows { + keys[i] = row.PublicKey + } + names, err := s.hiddenIdentityNames(r.Context(), keys) + if err != nil { + writeError(w, 500, err.Error()) + return + } + filtered := *response + filtered.Rows = make([]ScopeAuditRow, 0, len(response.Rows)) + for _, row := range response.Rows { + allNames := append([]string{row.Name}, names[row.PublicKey]...) + if !identityHidden(s.cfg, row.PublicKey, allNames...) { + filtered.Rows = append(filtered.Rows, row) + } + } + filtered.Summary = summarizeScopeAudit(filtered.Rows) + writeJSON(w, &filtered) +} diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go new file mode 100644 index 000000000..c9306f477 --- /dev/null +++ b/cmd/server/scope_audit_test.go @@ -0,0 +1,359 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +func auditFixture(t *testing.T) (*Server, func(string, string, string, string), func(int, int, *string, string, string, time.Time)) { + t.Helper() + s, _ := setupTestServer(t) + for _, q := range []string{"ALTER TABLE nodes ADD COLUMN configured_scope TEXT", "ALTER TABLE nodes ADD COLUMN configured_scope_at TEXT", "ALTER TABLE transmissions ADD COLUMN scope_name TEXT", "DELETE FROM observations", "DELETE FROM transmissions", "DELETE FROM nodes"} { + if _, err := s.db.conn.Exec(q); err != nil { + t.Fatal(err) + } + } + s.db.detectSchema() + node := func(pk, name, scope, at string) { + t.Helper() + if _, err := s.db.conn.Exec(`INSERT INTO nodes(public_key,name,role,configured_scope,configured_scope_at) VALUES(?,?,'repeater',?,?)`, pk, name, scope, at); err != nil { + t.Fatal(err) + } + } + packet := func(id, route int, scope *string, path, raw string, at time.Time) { + t.Helper() + if _, err := s.db.conn.Exec(`INSERT OR IGNORE INTO transmissions(id,hash,raw_hex,first_seen,route_type,scope_name) VALUES(?,?,?, ?,?,?)`, id, id, raw, at.UTC().Format(time.RFC3339), route, scope); err != nil { + t.Fatal(err) + } + if _, err := s.db.conn.Exec(`INSERT INTO observations(transmission_id,path_json,raw_hex,timestamp) VALUES(?,?,?,?)`, id, path, raw, at.Unix()); err != nil { + t.Fatal(err) + } + } + return s, node, packet +} +func TestScopeAuditFindings(t *testing.T) { + s, node, packet := auditFixture(t) + now := time.Now().UTC() + at := now.Format(time.RFC3339) + node(scopeContractMultiPK, "multi", "#dk, eu, #dk", at) + node(scopeContractEmptyPK, "empty", "", at) + node(scopeContractWildPK, "wild", "*", now.Add(-8*24*time.Hour).Format(time.RFC3339)) + dk := "dk" + foreign := "#other" + unknown := "" + packet(1, 0, &dk, `["e1","e1","e3"]`, "0001020304", now) + packet(1, 0, &dk, `["e1","e3"]`, "0001020304", now) + packet(2, 0, &foreign, `["e1"]`, "0001020304", now) + packet(3, 1, nil, `["e1"]`, "01", now) + packet(4, 0, &unknown, `["e1"]`, "0001020304", now) + packet(5, 2, &foreign, `["e1"]`, "02", now) + resp, err := s.buildScopeAudit("24h", now) + if err != nil { + t.Fatal(err) + } + if len(resp.Rows) != 3 { + t.Fatalf("rows=%+v", resp.Rows) + } + byKey := make(map[string]ScopeAuditRow) + for _, row := range resp.Rows { + byKey[row.PublicKey] = row + } + multi := byKey[scopeContractMultiPK] + if multi.PublicKey != scopeContractMultiPK || multi.Forwarded != 4 || multi.UnscopedObserved != 1 || multi.UnknownScopeObserved != 1 || !multi.WildcardContradiction || len(multi.NotObserved) != 1 || multi.NotObserved[0] != "eu" || len(multi.UndeclaredObserved) != 1 || multi.Status != "incomplete" { + t.Fatalf("multi=%+v", multi) + } + if byKey[scopeContractWildPK].Status != "no-evidence" || !byKey[scopeContractWildPK].StaleDeclaration { + t.Fatalf("wild=%+v", byKey[scopeContractWildPK]) + } + if byKey[scopeContractEmptyPK].Status != "findings" || byKey[scopeContractEmptyPK].Forwarded != 1 { + t.Fatalf("empty=%+v", byKey[scopeContractEmptyPK]) + } +} +func TestScopeAuditCollisionAndWindow(t *testing.T) { + s, node, packet := auditFixture(t) + now := time.Now().UTC() + node(scopeContractMultiPK, "declared", "dk", now.Format(time.RFC3339)) + if _, err := s.db.conn.Exec(`INSERT INTO nodes(public_key,name,role) VALUES(?,'other','repeater')`, "e1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); err != nil { + t.Fatal(err) + } + dk := "dk" + packet(1, 0, &dk, `["e1"]`, "0001020304", now) + packet(2, 0, &dk, `["`+scopeContractMultiPK+`"]`, "0001020304", now.Add(-2*time.Hour)) + r, err := s.buildScopeAudit("1h", now) + if err != nil { + t.Fatal(err) + } + if r.AmbiguousHops != 1 || r.Rows[0].Forwarded != 0 || !r.Rows[0].IncompleteEvidence { + t.Fatalf("1h=%+v", r) + } + r, err = s.buildScopeAudit("24h", now) + if err != nil { + t.Fatal(err) + } + if r.Rows[0].Forwarded != 1 { + t.Fatalf("24h=%+v", r.Rows[0]) + } +} +func TestScopeAuditHTTPAndCache(t *testing.T) { + s, node, _ := auditFixture(t) + node(scopeContractMultiPK, "visible", "dk", time.Now().UTC().Format(time.RFC3339)) + get := func(window string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + s.handleScopeAudit(w, httptest.NewRequest("GET", "/api/scope-audit?window="+window, nil)) + return w + } + if w := get("bad"); w.Code != 400 { + t.Fatalf("status=%d", w.Code) + } + w := get("24h") + if w.Code != 200 { + t.Fatal(w.Body.String()) + } + var r ScopeAuditResponse + if err := json.Unmarshal(w.Body.Bytes(), &r); err != nil { + t.Fatal(err) + } + if len(r.Rows) != 1 { + t.Fatal(r) + } + s.db.conn.Exec(`UPDATE nodes SET configured_scope='eu' WHERE public_key=?`, scopeContractMultiPK) + w = get("24h") + json.Unmarshal(w.Body.Bytes(), &r) + if r.Rows[0].ConfiguredScope != "dk" { + t.Fatal("cache missed") + } + s.cfg.SetNodeBlacklist([]string{scopeContractMultiPK}) + w = get("24h") + json.Unmarshal(w.Body.Bytes(), &r) + if len(r.Rows) != 0 || r.Summary.Total != 0 { + t.Fatalf("cached identity leaked: %s", w.Body.String()) + } +} + +func TestScopeAuditObservationRouteAndScope(t *testing.T) { + s, node, packet := auditFixture(t) + now := time.Now().UTC() + node(scopeContractMultiPK, "relay", "#dk,*", now.Format(time.RFC3339)) + dk := "dk" + packet(1, 0, &dk, `["e1"]`, "0001020304", now) + // Same payload crosses a scope boundary; canonical scope is no longer proof. + packet(1, 0, &dk, `["e1"]`, "0005060304", now) + packet(1, 0, &dk, `["e1"]`, "01", now) + // Canonical flood transmission has a direct observation to a second target. + node(scopeContractEmptyPK, "direct-only", "", now.Format(time.RFC3339)) + packet(1, 0, &dk, `["e3"]`, "02", now) + // Random unknown hops are not collisions with known nodes. + packet(2, 0, &dk, `["ff", "malformed"]`, "0001020304", now) + r, err := s.buildScopeAudit("24h", now) + if err != nil { + t.Fatal(err) + } + for _, row := range r.Rows { + if row.PublicKey == scopeContractMultiPK { + if row.Forwarded != 1 || row.UnknownScopeObserved != 1 || row.UnscopedObserved != 1 || len(row.ObservedScopes) != 1 || row.WildcardContradiction { + t.Fatalf("boundary evidence=%+v", row) + } + } else if row.Forwarded != 0 { + t.Fatalf("direct path counted=%+v", row) + } + } + if r.AmbiguousHops != 0 { + t.Fatalf("unknown hops reported as collisions: %d", r.AmbiguousHops) + } +} + +func TestScopeAuditMissingDeclarationAndSevenDayWindow(t *testing.T) { + s, node, packet := auditFixture(t) + now := time.Now().UTC() + node(scopeContractMultiPK, "relay", "dk", now.Format(time.RFC3339)) + if _, err := s.db.conn.Exec(`INSERT INTO nodes(public_key,name,role) VALUES(?,'not-declared','repeater')`, scopeContractNonePK); err != nil { + t.Fatal(err) + } + dk := "dk" + packet(1, 0, &dk, `["e1"]`, "0001020304", now.Add(-48*time.Hour)) + packet(2, 0, &dk, `["e1"]`, "0001020304", now.Add(-8*24*time.Hour)) + r, err := s.buildScopeAudit("7d", now) + if err != nil { + t.Fatal(err) + } + if len(r.Rows) != 1 || r.Rows[0].Forwarded != 1 || r.Rows[0].Status != "consistent" { + t.Fatalf("7d=%+v", r) + } +} + +func BenchmarkScopeAudit30KPackets2KNodes(b *testing.B) { + db := setupTestDB(b) + for _, q := range []string{"ALTER TABLE nodes ADD COLUMN configured_scope TEXT", "ALTER TABLE nodes ADD COLUMN configured_scope_at TEXT", "ALTER TABLE transmissions ADD COLUMN scope_name TEXT", "CREATE INDEX audit_obs_tx ON observations(transmission_id)", "CREATE INDEX audit_obs_ts ON observations(timestamp)"} { + if _, err := db.conn.Exec(q); err != nil { + b.Fatal(err) + } + } + now := time.Now().UTC() + stamp := now.Format(time.RFC3339) + tx, err := db.conn.Begin() + if err != nil { + b.Fatal(err) + } + insertNode, err := tx.Prepare(`INSERT INTO nodes(public_key,name,role,configured_scope,configured_scope_at) VALUES(?,'relay','repeater','dk',?)`) + if err != nil { + b.Fatal(err) + } + keys := make([]string, 2000) + for i := range keys { + keys[i] = fmt.Sprintf("%06x%058x", i+1, 0) + if _, err := insertNode.Exec(keys[i], stamp); err != nil { + b.Fatal(err) + } + } + insertNode.Close() + insertTx, err := tx.Prepare(`INSERT INTO transmissions(id,hash,raw_hex,first_seen,route_type,scope_name) VALUES(?,?,'0001020304',?,0,'dk')`) + if err != nil { + b.Fatal(err) + } + insertObs, err := tx.Prepare(`INSERT INTO observations(transmission_id,path_json,raw_hex,timestamp) VALUES(?,?,'0001020304',?)`) + if err != nil { + b.Fatal(err) + } + for i := 1; i <= 30000; i++ { + if _, err := insertTx.Exec(i, fmt.Sprint(i), stamp); err != nil { + b.Fatal(err) + } + path := fmt.Sprintf(`["%s","%s","%s","%s"]`, keys[i%2000][:6], keys[(i+1)%2000][:6], keys[(i+2)%2000][:6], keys[(i+3)%2000][:6]) + for j := 0; j < 3; j++ { + if _, err := insertObs.Exec(i, path, now.Unix()); err != nil { + b.Fatal(err) + } + } + } + insertTx.Close() + insertObs.Close() + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + db.detectSchema() + s := &Server{db: db, cfg: &Config{}} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + r, err := s.buildScopeAudit("24h", now) + if err != nil { + b.Fatal(err) + } + if len(r.Rows) != 2000 || r.Rows[0].Forwarded != 60 { + b.Fatalf("unexpected audit: rows=%d forwarded=%d", len(r.Rows), r.Rows[0].Forwarded) + } + } +} + +func TestScopeAuditCacheSingleflightExpiryAndErrors(t *testing.T) { + var cache scopeAuditCache + var fills atomic.Int32 + start := make(chan struct{}) + release := make(chan struct{}) + build := func() (*ScopeAuditResponse, error) { + if fills.Add(1) == 1 { + close(start) + } + <-release + return &ScopeAuditResponse{Window: "24h"}, nil + } + var wg sync.WaitGroup + results := make(chan *ScopeAuditResponse, 16) + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + r, err := cache.load("24h", build) + if err != nil { + t.Error(err) + } + results <- r + }() + } + <-start + close(release) + wg.Wait() + close(results) + var first *ScopeAuditResponse + for r := range results { + if first == nil { + first = r + } + if r != first { + t.Fatal("concurrent callers received distinct builds") + } + } + if fills.Load() != 1 { + t.Fatalf("fills=%d want 1", fills.Load()) + } + cache.mu.Lock() + entry := cache.entries["24h"] + entry.at = time.Now().Add(-scopeAuditTTL - time.Second) + cache.entries["24h"] = entry + cache.mu.Unlock() + if _, err := cache.load("24h", func() (*ScopeAuditResponse, error) { fills.Add(1); return &ScopeAuditResponse{}, nil }); err != nil { + t.Fatal(err) + } + if fills.Load() != 2 { + t.Fatal("expired entry not refreshed") + } + for i := 0; i < 2; i++ { + if _, err := cache.load("7d", func() (*ScopeAuditResponse, error) { fills.Add(1); return nil, fmt.Errorf("failed") }); err == nil { + t.Fatal("expected error") + } + } + if fills.Load() != 4 { + t.Fatal("failed build was cached") + } +} + +func TestScopeAuditCachedHiddenNameAndUnsupportedSchema(t *testing.T) { + s, node, _ := auditFixture(t) + node(scopeContractMultiPK, "visible", "", time.Now().UTC().Format(time.RFC3339)) + s.cfg.SetHiddenNamePrefixes([]string{"private-"}) + get := func() *httptest.ResponseRecorder { + w := httptest.NewRecorder() + s.handleScopeAudit(w, httptest.NewRequest("GET", "/api/scope-audit", nil)) + return w + } + if w := get(); w.Code != 200 { + t.Fatal(w.Body.String()) + } + if _, err := s.db.conn.Exec(`UPDATE nodes SET name='private-fixture' WHERE public_key=?`, scopeContractMultiPK); err != nil { + t.Fatal(err) + } + w := get() + var r ScopeAuditResponse + if err := json.Unmarshal(w.Body.Bytes(), &r); err != nil { + t.Fatal(err) + } + if len(r.Rows) != 0 { + t.Fatalf("hidden rename leaked: %s", w.Body.String()) + } + plain, _ := setupTestServer(t) + w = httptest.NewRecorder() + plain.handleScopeAudit(w, httptest.NewRequest("GET", "/api/scope-audit", nil)) + if w.Code != 500 { + t.Fatalf("unsupported schema status=%d", w.Code) + } +} + +func TestScopeAuditNoDeclarationsSkipsObservationScan(t *testing.T) { + s, _, _ := auditFixture(t) + // If the cold path scans observations despite no declarations, this fails. + if _, err := s.db.conn.Exec(`DROP TABLE observations`); err != nil { + t.Fatal(err) + } + r, err := s.buildScopeAudit("24h", time.Now().UTC()) + if err != nil { + t.Fatal(err) + } + if len(r.Rows) != 0 || r.Summary.Total != 0 || r.AmbiguousHops != 0 { + t.Fatalf("empty audit=%+v", r) + } +} diff --git a/docs/user-guide/analytics.md b/docs/user-guide/analytics.md index 020962f1b..a4db85030 100644 --- a/docs/user-guide/analytics.md +++ b/docs/user-guide/analytics.md @@ -75,6 +75,24 @@ Interactive visualization of which nodes can directly hear each other. Shows the Per-observer signal health over time. Identifies observers with degrading reception. +## Scopes: Scope Audit + +Open **Scopes → Scope Audit** to compare each node's confirmed observer declaration with flood forwarding seen in the last **1h, 24h or 7d**. The audit covers the whole network; its own window controls the observations, independently of the global analytics filters. + +The table shows declared regions, observed regions and counts, unscoped traffic, undeclared regions and unscoped forwarding without a declared `*`. Search by node name, public key or declared region. Search, window and pagination are shareable, for example `#/analytics?tab=scopes&sub=audit&swin=7d&saq=dk`. + +**Not observed** means no attributable forwarding was recorded in this window. It does not prove that the node is configured incorrectly: quiet regions, limited observer coverage, hop limits and configuration changes can explain the gap. Declarations older than seven days, unknown declaration timestamps, ambiguous hop prefixes, unknown scoped traffic or legacy observations without their own raw header make the evidence incomplete. A node with no attributable forwarding is shown as **No forwarding evidence**, never as consistent. + +Only nodes with a confirmed declaration are included. An unanswered node is different from an answered empty list. Observer firmware exports flood-allowed regions; an empty list therefore means no flood-allowed regions, rather than proving that the node has no region definitions. Observer reports can omit names when their response buffer fills, without reporting truncation. Treat undeclared forwarding as a finding to investigate. + +Counts deduplicate a node/transmission pair across observers and repeated path hops. A transmission observed under several scope states contributes to each state's count, so those counts can sum above the unique forwarded total. Direct routes are excluded. Only an exact identity or uniquely matching supported hop prefix is attributed; heuristic path resolution is not evidence. + +This first version uses observer declarations already collected by CoreScope. It does not collect configuration itself, ingest CoreDrive declarations or verify unknown region codes against declared names. + +### Initial limits and customizer follow-up + +The initial defaults are 50 visible rows, a 150ms search debounce, the existing 60s refresh interval, a seven-day declaration-age threshold, a 30s backend cache and a 20s query deadline. Expose operator-adjustable display and freshness limits in a later customizer milestone. The resolver memo is capped at 4096 tokens and the cache at the three supported windows; those implementation bounds stay internal. + ## Prefix Tool Test hash prefix lengths to see how many collisions different sizes would produce. Useful for deciding on hash_size settings. diff --git a/public/analytics.js b/public/analytics.js index 39d90f2bf..d1b282640 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -70,6 +70,8 @@ if (_rolesRefreshTimer) { clearInterval(_rolesRefreshTimer); _rolesRefreshTimer = null; } } var _scopesRefreshTimer = null; + var _scopeAuditView = null; + function _stopScopeAudit() { if (_scopeAuditView) { _scopeAuditView.destroy(); _scopeAuditView = null; } } function _stopScopesRefresh() { if (_scopesRefreshTimer) { clearInterval(_scopesRefreshTimer); _scopesRefreshTimer = null; } } @@ -283,7 +285,7 @@ setAreaFilterVisibility(_currentTab); // #1085 — Roles tab owns its own 60s auto-refresh; stop it on switch. if (_currentTab !== 'roles') _stopRolesRefresh(); - if (_currentTab !== 'scopes') _stopScopesRefresh(); + if (_currentTab !== 'scopes') { _stopScopesRefresh(); _stopScopeAudit(); } if (_currentTab !== 'foreign-traffic') _stopForeignTrafficRefresh(); if (_currentTab !== 'wardriving') _stopWardrivingRefresh(); if (_currentTab !== 'areas') _stopAreasRefresh(); @@ -387,7 +389,7 @@ // fallback for a plain visit of the tab. ?window= is the global time // picker above the tab bar (other values, and it drives the shared // loads), so each tab window gets a key of its own. - var SCOPES_SUBTAB = { param: 'sub', storageKey: 'scopes_subtab', allowed: ['overview', 'hopdepth', 'regions', 'hygiene'], dflt: 'overview' }; + var SCOPES_SUBTAB = { param: 'sub', storageKey: 'scopes_subtab', allowed: ['overview', 'hopdepth', 'regions', 'hygiene', 'audit'], dflt: 'overview' }; var SCOPES_WINDOW = { param: 'swin', storageKey: 'scopes_window', allowed: ['1h', '24h', '7d'], dflt: '24h' }; var WARDRIVING_WINDOW = { param: 'wdwin', storageKey: 'wardriving_window', allowed: ['1h', '24h', '7d'], dflt: '24h' }; // #208 — Hash Stats' multi-byte adopters filter. URL only (no storageKey): @@ -407,7 +409,7 @@ 'rf-health': ['range', 'observer', 'from', 'to'], collisions: ['section'], hashsizes: [HASHSTATS_MB_FILTER.param, HASHSTATS_MB_SORT.param, HASHSTATS_MB_DIR.param], - scopes: [SCOPES_SUBTAB.param, SCOPES_WINDOW.param], + scopes: [SCOPES_SUBTAB.param, SCOPES_WINDOW.param, 'saq', 'sap'], wardriving: [WARDRIVING_WINDOW.param], }; @@ -3346,11 +3348,12 @@ } } -function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTrafficRefresh(); _stopWardrivingRefresh(); _stopAreasRefresh(); _leaveDistanceTab(); _cancelLoadRetry(); _analyticsData = {}; _channelData = null; if (_ngState && _ngState.animId) { cancelAnimationFrame(_ngState.animId); } _ngState = null; if (_themeRefreshHandler) { window.removeEventListener('theme-refresh', _themeRefreshHandler); _themeRefreshHandler = null; } } +function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopScopeAudit(); _stopForeignTrafficRefresh(); _stopWardrivingRefresh(); _stopAreasRefresh(); _leaveDistanceTab(); _cancelLoadRetry(); _analyticsData = {}; _channelData = null; if (_ngState && _ngState.animId) { cancelAnimationFrame(_ngState.animId); } _ngState = null; if (_themeRefreshHandler) { window.removeEventListener('theme-refresh', _themeRefreshHandler); _themeRefreshHandler = null; } } // Expose for testing if (typeof window !== 'undefined') { window._analyticsAssignTableIds = assignAnalyticsTableIds; + window._analyticsRenderScopesTab = renderScopesTab; window._analyticsWithQuery = withQuery; window._analyticsResolveViewParam = resolveViewParam; window._analyticsDecorateChannels = decorateAnalyticsChannels; @@ -5153,8 +5156,10 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf async function renderScopesTab(el) { // Both views are deep-linked: ?sub= and ?swin= (#205). var scopesView = restoreViewParams([SCOPES_SUBTAB, SCOPES_WINDOW]); + if (el._refreshScopes && el.querySelector('#scopes-cards')) { el._refreshScopes(scopesView); return; } var selectedSubtab = scopesView[0]; var selectedWindow = scopesView[1]; + var statsLoadedWindow = null; // #1852: the tab grew to stacked sections (windowed adoption stats, // all-time region breakdowns, all-time node/repeater hygiene lists) — @@ -5202,6 +5207,7 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf { key: 'hopdepth', label: 'Hop Depth' }, { key: 'regions', label: 'Regions' }, { key: 'hygiene', label: 'Hygiene' }, + { key: 'audit', label: 'Scope Audit' }, ].map(function(t) { return ''; }).join('') + @@ -5242,12 +5248,33 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf '
' + '
' + '' + + '
' + '
' + '
' + '
' + '
'; - // Sub-tab click listener (once) — pure visibility toggle, no re-fetch. + // Audit is lazy: it fetches only its own bulk endpoint while active. + _stopScopeAudit(); + var auditPanel = el.querySelector('#scopes-panel-audit'); + var auditView = window.ScopeAudit ? window.ScopeAudit.mount(auditPanel, { + api: api, hash: function() { return location.hash; }, + onWindow: function(w) { + selectedWindow = w; setViewParam(SCOPES_WINDOW, w); + el.querySelectorAll('[data-win]').forEach(function(b) { b.classList.toggle('active', b.dataset.win === w); }); + load(w); + }, + onState: function(query, page) { + var parts = location.hash.split('?'); var params = new URLSearchParams(parts[1] || ''); + if (query) params.set('saq', query); else params.delete('saq'); + if (page) params.set('sap', String(page)); else params.delete('sap'); + history.replaceState(history.state, '', parts[0] + (params.toString() ? '?' + params.toString() : '')); + } + }) : null; + _scopeAuditView = auditView; + if (auditView) auditView.setActive(selectedSubtab === 'audit'); + + // Sub-tab click listener (once). var subtabsEl = el.querySelector('#scopesSubtabs'); if (subtabsEl) { subtabsEl.addEventListener('click', function(e) { @@ -5260,11 +5287,14 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf var panel = document.getElementById('scopes-panel-' + key); if (panel) panel.style.display = key === selectedSubtab ? '' : 'none'; }); + if (auditView) auditView.setActive(selectedSubtab === 'audit'); + if (selectedSubtab === 'audit' || statsLoadedWindow !== selectedWindow) load(selectedWindow); }); } // Attach window-button click listeners (once) el.querySelectorAll('[data-win]').forEach(function(btn) { + if (btn.dataset.auditWindow) return; btn.addEventListener('click', function() { selectedWindow = btn.dataset.win; setViewParam(SCOPES_WINDOW, selectedWindow); @@ -5482,6 +5512,12 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf } async function load(w) { + if (selectedSubtab === 'audit') { + if (auditView) return auditView.load(w); + el.querySelector('#scopes-panel-audit').textContent = 'Scope Audit could not initialize. Reload to retry.'; + return; + } + var loadingEl = document.getElementById('scopes-loading'); if (loadingEl) loadingEl.style.display = ''; try { @@ -5494,6 +5530,7 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf return; } await updateData(data, w); + statsLoadedWindow = w; } catch (err) { if (loadingEl) loadingEl.style.display = 'none'; var cardsEl3 = document.getElementById('scopes-cards'); @@ -6040,6 +6077,17 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf } + el._refreshScopes = function(view) { + selectedSubtab = view[0]; selectedWindow = view[1]; + el.querySelectorAll('[data-subtab]').forEach(function(b) { b.classList.toggle('active', b.dataset.subtab === selectedSubtab); }); + el.querySelectorAll('[data-win]').forEach(function(b) { b.classList.toggle('active', b.dataset.win === selectedWindow); }); + SCOPES_SUBTAB.allowed.forEach(function(key) { + var panel = el.querySelector('#scopes-panel-' + key); + if (panel) panel.style.display = key === selectedSubtab ? '' : 'none'; + }); + if (auditView) { auditView.setActive(selectedSubtab === 'audit'); auditView.syncState(location.hash); } + load(selectedWindow); + }; load(selectedWindow); // Fix 6: auto-refresh every 60s diff --git a/public/index.html b/public/index.html index 0d2d3f980..df840d705 100644 --- a/public/index.html +++ b/public/index.html @@ -219,6 +219,7 @@ + diff --git a/public/scope-audit.js b/public/scope-audit.js new file mode 100644 index 000000000..5f1299d19 --- /dev/null +++ b/public/scope-audit.js @@ -0,0 +1,151 @@ +/* Scope Audit: one bulk request, local search, and a bounded keyed row diff. */ +(function(root) { + 'use strict'; + var PAGE_SIZE = 50; + function windowValue(value) { return ['1h', '24h', '7d'].indexOf(value) >= 0 ? value : '24h'; } + function stateFromHash(hash) { + var p = new URLSearchParams(String(hash || '').split('?')[1] || ''); + return { window: windowValue(p.get('swin')), query: p.get('saq') || '', page: Math.max(0, Math.floor(Number(p.get('sap')) || 0)) }; + } + function searchRows(rows, query) { + var q = String(query || '').trim().toLowerCase(); + return rows.filter(function(r) { + return !q || [r.name, r.publicKey, r.configuredScope].concat(r.declaredScopes || []).join(' ').toLowerCase().indexOf(q) >= 0; + }); + } + function declarationAge(seconds) { + if (seconds == null || !Number.isFinite(Number(seconds))) return 'Unknown'; + var n = Math.max(0, Number(seconds)); + return n < 3600 ? Math.floor(n / 60) + 'm' : n < 86400 ? Math.floor(n / 3600) + 'h' : Math.floor(n / 86400) + 'd'; + } + function count(value) { return Math.max(0, Number(value) || 0); } + function assessment(row) { + if (row.status === 'no-evidence' || !count(row.forwarded)) return 'No forwarding evidence'; + var findings = (row.undeclaredObserved || []).length || row.wildcardContradiction; + if (row.incompleteEvidence || row.staleDeclaration || count(row.unknownScopeObserved)) return findings ? 'Findings · incomplete evidence' : 'Incomplete evidence'; + return findings || (row.notObserved || []).length ? 'Findings to investigate' : 'Consistent with observations'; + } + function mount(container, options) { + var doc = container.ownerDocument; + var rows = [], rowNodes = new Map(), generation = 0, active = true, searchTimer = null; + var state = stateFromHash(options.hash()); + function element(tag, text, cls) { + var el = doc.createElement(tag); + if (text != null) el.textContent = String(text); + if (cls) el.className = cls; + return el; + } + function add(parent, tag, text, cls) { var el = element(tag, text, cls); parent.appendChild(el); return el; } + add(container, 'h3', 'Scope Audit'); + add(container, 'p', 'Whole-network audit. The audit window is independent of the global time and region filters.', 'text-muted'); + add(container, 'p', 'Compares confirmed observer declarations with forwarding observed in the selected window. A declared scope without observed traffic does not prove a configuration fault. Observer coverage, quiet regions and configuration changes can explain gaps.', 'text-muted'); + add(container, 'p', 'Short windows may contain little traffic; 7d may span configuration changes. Declarations older than 7d or with an unknown timestamp, unknown scopes and ambiguous hop attribution make the evidence incomplete.', 'text-muted'); + var toolbar = add(container, 'div', null, 'scope-audit-controls'); + ['1h', '24h', '7d'].forEach(function(w) { + var btn = add(toolbar, 'button', w, 'tab-btn'); btn.dataset.win = w; btn.dataset.auditWindow = 'true'; + btn.addEventListener('click', function() { options.onWindow(w); }); + }); + var label = add(toolbar, 'label', 'Search repeaters / scopes '); + var input = add(label, 'input'); input.type = 'search'; input.id = 'scope-audit-search'; input.value = state.query; + input.placeholder = 'Name, public key or scope'; + var refresh = add(toolbar, 'button', 'Refresh', 'tab-btn'); + refresh.addEventListener('click', function() { load(state.window, true); }); + var message = add(container, 'p', '', 'text-muted'); message.id = 'scope-audit-message'; message.setAttribute('role', 'status'); + var summary = add(container, 'p'); summary.id = 'scope-audit-summary'; + var wrap = add(container, 'div', null, 'scope-audit-table-wrap'); + var table = add(wrap, 'table', null, 'data-table scope-audit-table'); table.id = 'scope-audit-table'; + var header = add(add(table, 'thead'), 'tr'); + ['Repeater', 'Declaration / age', 'Declared scopes', 'Observed forwarding', 'Assessment'].forEach(function(t) { add(header, 'th', t); }); + var tbody = add(table, 'tbody'); tbody.id = 'scope-audit-rows'; + var pager = add(container, 'div', null, 'scope-audit-controls'); + var previous = add(pager, 'button', 'Previous', 'tab-btn'); + var pageLabel = add(pager, 'span'); + var next = add(pager, 'button', 'Next', 'tab-btn'); + function save() { options.onState(state.query, state.page); } + previous.addEventListener('click', function() { state.page = Math.max(0, state.page - 1); save(); render(); }); + next.addEventListener('click', function() { state.page++; save(); render(); }); + input.addEventListener('input', function() { + if (!active || !container.isConnected) return; + clearTimeout(searchTimer); + state.query = input.value; state.page = 0; save(); + searchTimer = setTimeout(function() { if (active && container.isConnected) render(); }, 150); + }); + function rowElement(r) { + var tr = element('tr'); tr.dataset.publicKey = String(r.publicKey || ''); + var name = add(tr, 'td'); + var link = add(name, 'a', r.name || String(r.publicKey || '').slice(0, 16)); + link.href = '#/nodes/' + encodeURIComponent(String(r.publicKey || '')); + add(name, 'div', r.role || 'Repeater', 'text-muted'); + var declaration = add(tr, 'td'); + add(declaration, 'div', r.configuredScope === '' ? 'Empty declaration · no flood scopes' : r.allowsUnscoped ? 'Allows unscoped (*)' : 'Scoped only'); + add(declaration, 'div', declarationAge(r.declarationAgeSeconds) + (r.staleDeclaration ? ' · stale / unknown timestamp' : ''), 'text-muted'); + if (r.configuredScopeAt) declaration.title = String(r.configuredScopeAt); + var scopes = add(tr, 'td'); + var observed = new Set((r.observedScopes || []).map(function(s) { return String(s.name); })); + (r.declaredScopes || []).forEach(function(s) { + add(scopes, 'span', String(s) + (observed.has(String(s)) ? ' · observed' : ' · not observed'), 'scope-audit-chip' + (observed.has(String(s)) ? '' : ' scope-audit-caution')); + }); + if (!(r.declaredScopes || []).length) add(scopes, 'span', 'No named scopes', 'text-muted'); + var traffic = add(tr, 'td'); + (r.observedScopes || []).forEach(function(s) { add(traffic, 'div', String(s.name) + ': ' + count(s.count)); }); + add(traffic, 'div', 'Unscoped: ' + count(r.unscopedObserved)); + if (count(r.unknownScopeObserved)) add(traffic, 'div', 'Unknown scopes: ' + count(r.unknownScopeObserved), 'scope-audit-caution'); + if (count(r.ambiguousHops)) add(traffic, 'div', 'Ambiguous hops: ' + count(r.ambiguousHops), 'scope-audit-caution'); + var status = add(tr, 'td'); + add(status, 'strong', assessment(r)); + if ((r.notObserved || []).length) add(status, 'div', count(r.notObserved.length) + ' declared scope(s) not observed', 'text-muted'); + if ((r.undeclaredObserved || []).length) add(status, 'div', 'Observed outside declaration: ' + r.undeclaredObserved.join(', '), 'scope-audit-caution'); + if (r.wildcardContradiction) add(status, 'div', 'Unscoped forwarding without declared *', 'scope-audit-caution'); + return tr; + } + function render() { + var filtered = searchRows(rows, state.query); + var pages = Math.max(1, Math.ceil(filtered.length / PAGE_SIZE)); + if (state.page >= pages) { state.page = pages - 1; save(); } + var visible = filtered.slice(state.page * PAGE_SIZE, (state.page + 1) * PAGE_SIZE); + var keep = new Set(); + visible.forEach(function(r) { + var key = String(r.publicKey || ''); keep.add(key); + var signature = JSON.stringify(r), cached = rowNodes.get(key); + if (!cached || cached.signature !== signature) { + var tr = rowElement(r); + if (cached) tbody.replaceChild(tr, cached.el); + cached = { el: tr, signature: signature }; rowNodes.set(key, cached); + } + }); + rowNodes.forEach(function(cached, key) { if (!keep.has(key)) { cached.el.remove(); rowNodes.delete(key); } }); + visible.forEach(function(r, index) { + var node = rowNodes.get(String(r.publicKey || '')).el; + if (tbody.children[index] !== node) tbody.insertBefore(node, tbody.children[index] || null); + }); + pageLabel.textContent = filtered.length + ' of ' + rows.length + ' repeaters · Page ' + (state.page + 1) + ' of ' + pages; + previous.disabled = state.page === 0; next.disabled = state.page + 1 >= pages; + if (!filtered.length) message.textContent = rows.length ? 'No repeaters match this search.' : 'No confirmed observer declarations available. An unknown declaration is not the same as an empty declaration.'; + else if (message.dataset.loaded) message.textContent = 'Observed forwarding in ' + state.window + '. Results are limited to available observer coverage.'; + } + async function load(w, bust) { + state.window = windowValue(w); + toolbar.querySelectorAll('[data-win]').forEach(function(b) { b.classList.toggle('active', b.dataset.win === state.window); }); + var token = ++generation; + if (!active) return; + message.textContent = 'Loading scope audit…'; delete message.dataset.loaded; + try { + var data = await options.api('/scope-audit?window=' + encodeURIComponent(state.window), { ttl: 30000, bust: !!bust, retry503: false }); + if (token !== generation || !active || !container.isConnected) return; + if (!data || data.error || !Array.isArray(data.rows)) throw new Error(data && data.error || 'Invalid audit response'); + rows = data.rows; + var s = data.summary || {}; + summary.textContent = count(s.total) + ' confirmed declarations · ' + count(s.notObserved) + ' with unobserved scopes · ' + count(s.undeclaredObserved) + ' with undeclared scopes · ' + count(s.wildcardContradiction) + ' unscoped mismatches · ' + count(s.incomplete) + ' incomplete · ' + count(data.ambiguousHops) + ' ambiguous hop observations'; + message.dataset.loaded = 'true'; render(); + } catch (err) { + if (token !== generation || !active || !container.isConnected) return; + rows = []; render(); summary.textContent = ''; + message.textContent = 'Failed to load scope audit: ' + String(err.message || err) + '. Use Refresh to retry.'; + } + } + return { load: load, syncState: function(hash) { clearTimeout(searchTimer); state = stateFromHash(hash); input.value = state.query; render(); }, setActive: function(value) { active = value; generation++; }, destroy: function() { active = false; generation++; clearTimeout(searchTimer); } }; + } + var exported = { mount: mount, windowValue: windowValue, stateFromHash: stateFromHash, searchRows: searchRows, declarationAge: declarationAge, assessment: assessment, pageSize: PAGE_SIZE }; + if (typeof module !== 'undefined' && module.exports) module.exports = exported; + else root.ScopeAudit = exported; +})(typeof window !== 'undefined' ? window : globalThis); diff --git a/public/style.css b/public/style.css index 9fb2ca15d..e098130af 100644 --- a/public/style.css +++ b/public/style.css @@ -5817,3 +5817,18 @@ body.embed #app.app-fixed { .privacy-page ul { padding-left: 1.3em; } .privacy-page li { margin: 6px 0; } .privacy-page a { color: var(--link-color); } + +/* Scope audit keeps search/pagination stable and at most 50 rows mounted. */ +.scope-audit-controls { display:flex; flex-wrap:wrap; align-items:center; gap:8px; margin:12px 0; } +.scope-audit-controls input { max-width:100%; padding:6px 8px; background:var(--bg-secondary); color:var(--text); border:1px solid var(--border); border-radius:4px; } +.scope-audit-table-wrap { overflow-x:auto; } +.scope-audit-chip { display:inline-block; margin:2px 4px 2px 0; padding:2px 5px; border:1px solid var(--border); border-radius:4px; } +.scope-audit-caution { color:var(--warning); } +#scopes-panel-audit .scope-audit-table { width:100%; min-width:900px; table-layout:fixed; } +#scopes-panel-audit .scope-audit-table th:nth-child(1) { width:20%; } +#scopes-panel-audit .scope-audit-table th:nth-child(2) { width:15%; } +#scopes-panel-audit .scope-audit-table th:nth-child(3) { width:25%; } +#scopes-panel-audit .scope-audit-table th:nth-child(4) { width:15%; } +#scopes-panel-audit .scope-audit-table th:nth-child(5) { width:25%; } +#scopes-panel-audit .scope-audit-table th, +#scopes-panel-audit .scope-audit-table td { white-space:normal; overflow:visible; text-overflow:clip; overflow-wrap:anywhere; vertical-align:top; } diff --git a/test-all.sh b/test-all.sh index 3cfdc43a7..c0e89bd14 100755 --- a/test-all.sh +++ b/test-all.sh @@ -78,6 +78,7 @@ run test-channel-issue-1087.js run test-issue-1409-no-encrypted-flood.js run test-analytics-channels-integration.js run test-analytics-foreign-traffic-tab.js +run test-scope-audit.js run test-analytics-nodes-without-scope.js run test-analytics-table-ids-unique.js run test-analytics-relay-airtime-dumbbell.js diff --git a/test-analytics-subtab-deeplinks-205.js b/test-analytics-subtab-deeplinks-205.js index 0727735a7..0b3406d94 100644 --- a/test-analytics-subtab-deeplinks-205.js +++ b/test-analytics-subtab-deeplinks-205.js @@ -210,6 +210,8 @@ function pageEnv(opts) { await flush(); }, initError: () => initError, + async refreshScopes(hash) { ctx.location.hash = hash; await ctx.window._analyticsRenderScopesTab(content()); await flush(); }, + panel: key => el('scopes-panel-' + key), // History entries (#208): the current one as { hash, state }; a new // entry (a link, location.hash = …) has no state; Back/Forward brings // an entry back with the state it had when it was left. @@ -513,6 +515,34 @@ function pageEnv(opts) { assert.strictEqual(env.params().bytes, '2', 'bytes= lost: ' + env.hash()); }); + await test('retained Scopes DOM restores current hash window and panel', async () => { + const env = pageEnv(); + await env.mount('#/analytics?tab=scopes&sub=overview&swin=24h'); + await env.refreshScopes('#/analytics?tab=scopes&sub=hopdepth&swin=7d'); + assert.strictEqual(env.panel('hopdepth').style.display, ''); + assert.strictEqual(env.panel('overview').style.display, 'none'); + assert.ok(env.scopeStatsUrls().some(u => u.includes('window=7d'))); + }); + + await test('Audit window change reloads Overview at the selected window', async () => { + const env = pageEnv(); + await env.mount('#/analytics?tab=scopes&swin=24h'); + await env.refreshScopes('#/analytics?tab=scopes&sub=audit&swin=7d'); + await env.clickSubtab('overview'); + assert.ok(env.scopeStatsUrls().some(u => u.includes('window=7d')), 'Overview did not fetch 7d data'); + }); + + await test('failed Scope Statistics load is retried after a sub-tab toggle', async () => { + const env = pageEnv(); + await env.mount('#/analytics?tab=scopes&sub=audit'); + let requests = 0; + env.ctx.api = async () => { requests++; throw new Error('test unavailable'); }; + await env.clickSubtab('overview'); + const initial = requests; + await env.clickSubtab('hopdepth'); + assert.ok(requests > initial, 'A failed load was treated as already loaded'); + }); + console.log('\n=== #205: Wardriving window (wdwin=) ==='); for (const w of ['1h', '24h', '7d']) { diff --git a/test-e2e-playwright.js b/test-e2e-playwright.js index 74182fce1..b3647a16d 100644 --- a/test-e2e-playwright.js +++ b/test-e2e-playwright.js @@ -3630,6 +3630,8 @@ async function run() { `.vcr-scope-btn.active border should track --accent-strong, got ${result.border}`); }); + await require('./tools/scope-audit-e2e')({page, BASE, assert, test}); + await browser.close(); // Summary diff --git a/test-scope-audit.js b/test-scope-audit.js new file mode 100644 index 000000000..9e5c38eb4 --- /dev/null +++ b/test-scope-audit.js @@ -0,0 +1,77 @@ +'use strict'; +const assert = require('assert'); +const audit = require('./public/scope-audit.js'); +assert.strictEqual(audit.windowValue('bad'), '24h'); +assert.strictEqual(audit.windowValue('7d'), '7d'); +assert.strictEqual(audit.searchRows([{name: 'ALPHA', publicKey: 'abc'}, {name: 'Beta', publicKey: 'def'}], 'alpha').length, 1); +assert.strictEqual(audit.searchRows([{name: null, publicKey: 123}], '123').length, 1); +assert.strictEqual(audit.declarationAge('86400'), '1d'); +assert.strictEqual(audit.declarationAge(null), 'Unknown'); +assert.strictEqual(audit.stateFromHash('#/analytics?tab=scopes&sub=audit&swin=7d&saq=a%26b').query, 'a&b'); +assert.strictEqual(audit.stateFromHash('#/analytics?tab=scopes&sub=audit&swin=no').window, '24h'); +console.log('Scope Audit helpers passed'); +assert.strictEqual(audit.assessment({forwarded: '4', status: 'consistent', unknownScopeObserved: '2'}), 'Incomplete evidence'); +assert.strictEqual(audit.assessment({forwarded: '4', staleDeclaration: true, wildcardContradiction: true}), 'Findings · incomplete evidence'); +assert.strictEqual(audit.assessment({forwarded: '0', status: 'consistent'}), 'No forwarding evidence'); +// Minimal DOM tracks node identity and treats innerHTML writes as a test failure. +class Element { + constructor(tag, doc) { this.tagName = tag; this.ownerDocument = doc; this.children = []; this.dataset = {}; this.handlers = {}; this._text = ''; this.isConnected = true; this.classList = { toggle() {} }; } + set textContent(v) { this._text = String(v); this.children = []; } + get textContent() { return this._text + this.children.map(c => c.textContent).join(''); } + set innerHTML(v) { throw new Error('Unsafe HTML sink'); } + appendChild(el) { return this.insertBefore(el, null); } + insertBefore(el, before) { if (el.parentNode) el.remove(); let i = before ? this.children.indexOf(before) : this.children.length; this.children.splice(i, 0, el); el.parentNode = this; return el; } + replaceChild(el, old) { let i = this.children.indexOf(old); this.children[i] = el; el.parentNode = this; old.parentNode = null; } + remove() { if (this.parentNode) { let p = this.parentNode; p.children.splice(p.children.indexOf(this), 1); this.parentNode = null; } } + setAttribute(k, v) { this[k] = v; } + addEventListener(k, f) { this.handlers[k] = f; } + querySelectorAll() { let result = []; this.children.forEach(c => { if (c.dataset.win) result.push(c); result.push(...c.querySelectorAll()); }); return result; } +} +function descendants(el) { return [el, ...el.children.flatMap(descendants)]; } +function fixture(apiFn, hash = '#/analytics?tab=scopes&sub=audit') { + const doc = {createElement: tag => new Element(tag, doc)}; + const container = new Element('div', doc); let writes = []; + const controller = audit.mount(container, {api: apiFn, hash: () => hash, onWindow() {}, onState: (...v) => writes.push(v)}); + return {container, controller, writes, byId: id => descendants(container).find(e => e.id === id)}; +} +const row = {publicKey: 'a'.repeat(64), name: '', configuredScope: '#alpha', declaredScopes: ['#alpha'], declarationAgeSeconds: '7200', observedScopes: [{name: '#alpha', count: '3'}], unscopedObserved: '1', forwarded: '4', status: 'consistent'}; +(async () => { + let calls = []; + const f = fixture(async (path) => { calls.push(path); return {rows: [row], summary: {total: '1'}}; }); + await f.controller.load('24h'); + assert.strictEqual(calls.length, 1); + assert(f.container.textContent.includes(row.name), 'node-controlled names are literal text'); + assert(!descendants(f.container).some(e => e.tagName === 'img'), 'XSS cannot create an element'); + f.controller.syncState('#/analytics?tab=scopes&sub=audit&swin=7d&saq=alpha&sap=0'); + assert.strictEqual(f.byId('scope-audit-search').value, 'alpha', 'Current hash search restored on retained DOM'); + assert.strictEqual(calls.length, 1, 'State sync does not fetch'); + const original = f.byId('scope-audit-rows').children[0]; + await f.controller.load('24h'); + assert.strictEqual(f.byId('scope-audit-rows').children[0], original, 'unchanged row survives refresh'); + const input = f.byId('scope-audit-search'); input.value = 'missing'; input.handlers.input(); + await new Promise(r => setTimeout(r, 170)); + assert.strictEqual(calls.length, 2, 'search does not fetch'); + assert.strictEqual(f.byId('scope-audit-rows').children.length, 0); + assert.strictEqual(f.writes.at(-1)[0], 'missing'); + input.value = 'hidden'; input.handlers.input(); const writeCount = f.writes.length; f.controller.setActive(false); + await new Promise(r => setTimeout(r, 170)); + assert.strictEqual(f.writes.length, writeCount, 'inactive timer cannot rewrite URL'); + input.value = 'inactive'; input.handlers.input(); + assert.strictEqual(f.writes.length, writeCount, 'inactive events cannot rewrite URL'); + let resolves = []; + const race = fixture(() => new Promise(r => resolves.push(r))); + const first = race.controller.load('1h'), last = race.controller.load('7d'); + resolves[1]({rows: [{...row, name: 'Latest'}], summary: {total: 1}}); await last; + resolves[0]({rows: [{...row, name: 'Obsolete'}], summary: {total: 1}}); await first; + assert(race.container.textContent.includes('Latest') && !race.container.textContent.includes('Obsolete')); + const bounded = fixture(async () => ({rows: Array.from({length: 2000}, (_, i) => ({...row, publicKey: String(i)})), summary: {total: 2000}})); + await bounded.controller.load('24h'); assert.strictEqual(bounded.byId('scope-audit-rows').children.length, audit.pageSize); + const empty = fixture(async () => ({rows: [], summary: {total: 0}})); await empty.controller.load('1h'); + assert(empty.container.textContent.includes('unknown declaration is not the same as an empty')); + const error = fixture(async () => { throw new Error(''); }); await error.controller.load('24h'); + assert(error.container.textContent.includes('Failed to load scope audit: ')); + const late = fixture(() => new Promise(r => resolves.push(r))); + const pending = late.controller.load('1h'); late.controller.destroy(); resolves[2]({rows: [row]}); await pending; + assert.strictEqual(late.byId('scope-audit-rows').children.length, 0); + console.log('Scope Audit DOM, bounded rendering, search, XSS and lifecycle tests passed'); +})().catch(err => { console.error(err); process.exitCode = 1; }); diff --git a/tools/scope-audit-e2e.js b/tools/scope-audit-e2e.js new file mode 100644 index 000000000..84411fe2e --- /dev/null +++ b/tools/scope-audit-e2e.js @@ -0,0 +1,82 @@ +/* Invoked by the existing Playwright suite against its local server. */ +'use strict'; +module.exports = async function scopeAuditE2E({page, BASE, assert, test}) { + const pattern = '**/api/scope-audit*'; + const row = {publicKey: 'a'.repeat(64), name: '', role: 'repeater', configuredScope: '#alpha', configuredScopeAt: '2026-01-01T00:00:00Z', declaredScopes: ['alpha', 'quiet'], allowsUnscoped: false, declarationAgeSeconds: '7200', staleDeclaration: false, observedScopes: [{name: 'alpha', count: '3'}], unscopedObserved: '1', unknownScopeObserved: '2', forwarded: '6', notObserved: ['quiet'], undeclaredObserved: [], wildcardContradiction: true, incompleteEvidence: true, status: 'incomplete'}; + let mode = 'rows', calls = [], statsWindows = []; + const statsListener = request => { + const url = new URL(request.url()); + if (url.pathname === '/api/scope-stats') statsWindows.push(url.searchParams.get('window')); + }; + page.on('request', statsListener); + await page.route(pattern, async route => { + calls.push(new URL(route.request().url()).searchParams.get('window')); + if (mode === 'error') return route.fulfill({status: 503, contentType: 'application/json', body: JSON.stringify({error: 'Audit unavailable'})}); + return route.fulfill({status: 200, contentType: 'application/json', body: JSON.stringify({window: calls.at(-1), rows: mode === 'empty' ? [] : [row], summary: {total: mode === 'empty' ? 0 : 1, incomplete: 1}, ambiguousHops: 2})}); + }); + try { + await test('Scope Audit: lazy tab, window, local search, stable refresh and deep link', async () => { + await page.goto(`${BASE}/#/analytics?tab=scopes&sub=overview&swin=24h`, {waitUntil: 'domcontentloaded'}); + await page.reload({waitUntil: 'domcontentloaded'}); + await page.waitForSelector('[data-subtab="audit"]'); + assert(calls.length === 0, 'Audit should not fetch while Overview is selected'); + await page.click('[data-subtab="audit"]'); + await page.waitForSelector('#scope-audit-rows tr'); + assert(calls.join(',') === '24h', 'Selecting Audit should issue one bulk request'); + assert(await page.locator('#scope-audit-rows').innerText().then(t => t.includes('Findings · incomplete evidence')), 'Unknown evidence must not be consistent'); + assert(await page.locator('#scope-audit-rows img').count() === 0, 'Node names must remain text'); + assert(await page.evaluate(() => !window.auditXss), 'XSS payload must not execute'); + const layout = await page.evaluate(() => { + const table = document.querySelector('#scope-audit-table'); + const cells = Array.from(table.querySelectorAll('tbody td')); + return {wrapped: cells.every(cell => getComputedStyle(cell).whiteSpace === 'normal'), + contained: cells.every(cell => cell.scrollWidth <= cell.clientWidth + 1), + resized: table.querySelectorAll('.col-resize-handle').length}; + }); + assert(layout.wrapped && layout.contained, 'Audit long labels must wrap within their columns'); + assert(layout.resized === 0, 'Audit table must avoid empty-table generic resizing'); + await page.evaluate(() => { window.auditStableRow = document.querySelector('#scope-audit-rows tr'); }); + await page.locator('#scopes-panel-audit').getByRole('button', {name: 'Refresh', exact: true}).click(); + await page.waitForFunction(() => !document.querySelector('#scope-audit-message').textContent.includes('Loading')); + assert(await page.evaluate(() => window.auditStableRow === document.querySelector('#scope-audit-rows tr')), 'Refresh must retain unchanged DOM rows'); + await page.locator('#scopes-panel-audit [data-win="7d"]').click(); + await page.waitForFunction(() => document.querySelector('#scope-audit-message').textContent.includes('7d')); + assert(calls.at(-1) === '7d', 'Window should reach endpoint'); + assert(page.url().includes('swin=7d'), 'Window must be deep linked'); + await page.click('[data-subtab="overview"]'); + await page.waitForFunction(() => !document.querySelector('#scopes-loading') || document.querySelector('#scopes-loading').style.display === 'none'); + assert(statsWindows.includes('7d'), 'Returning to Overview must load the Audit-selected window'); + await page.click('[data-subtab="audit"]'); + await page.waitForSelector('#scope-audit-rows tr'); + if (process.env.SCOPE_AUDIT_SCREENSHOT) await page.screenshot({path: process.env.SCOPE_AUDIT_SCREENSHOT, fullPage: true}); + const count = calls.length; + await page.fill('#scope-audit-search', 'quiet'); + await page.waitForFunction(() => location.hash.includes('saq=quiet')); + assert(calls.length === count, 'Search must be local'); + await page.reload({waitUntil: 'domcontentloaded'}); + await page.waitForSelector('#scope-audit-rows tr'); + assert(await page.inputValue('#scope-audit-search') === 'quiet', 'Search deep link must restore'); + await page.evaluate(async () => { + history.replaceState(history.state, '', '#/analytics?tab=scopes&sub=audit&swin=1h&saq=alpha'); + await window._analyticsRenderScopesTab(document.querySelector('#analyticsContent')); + }); + await page.waitForFunction(() => document.querySelector('#scope-audit-message').textContent.includes('1h')); + assert(await page.inputValue('#scope-audit-search') === 'alpha', 'Retained DOM must restore the current hash search'); + assert(calls.at(-1) === '1h', 'Retained DOM must restore the current hash window'); + await page.fill('#scope-audit-search', 'does-not-match'); + await page.waitForFunction(() => document.querySelector('#scope-audit-message').textContent.includes('No repeaters match')); + }); + await test('Scope Audit: empty and failed responses remain actionable', async () => { + mode = 'empty'; + await page.goto(`${BASE}/#/analytics?tab=scopes&sub=audit&swin=1h`, {waitUntil: 'domcontentloaded'}); + await page.reload({waitUntil: 'domcontentloaded'}); + await page.waitForFunction(() => document.querySelector('#scope-audit-message')?.textContent.includes('No confirmed observer declarations')); + mode = 'error'; + await page.locator('#scopes-panel-audit').getByRole('button', {name: 'Refresh', exact: true}).click(); + await page.waitForFunction(() => document.querySelector('#scope-audit-message').textContent.includes('Failed to load scope audit')); + mode = 'rows'; + await page.locator('#scopes-panel-audit').getByRole('button', {name: 'Refresh', exact: true}).click(); + await page.waitForSelector('#scope-audit-rows tr'); + }); + } finally { await page.unroute(pattern); page.off('request', statsListener); } +}; From 25569b160c8a73b8df944b2b8054b4e6a6d3b101 Mon Sep 17 00:00:00 2001 From: dborup Date: Wed, 14 Oct 2026 12:37:34 +0200 Subject: [PATCH 2/2] reviewfix(scope-audit): detach shared build from caller cancellation, close mutant gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review — CS-MacBook PR#378: handleScopeAudit derived the shared singleflight build's context from the first caller's r.Context(), so that caller disconnecting canceled the build for every coalesced waiter. Detach with context.WithoutCancel, keeping only the absolute deadline. Also: move the scopeAuditCache field out of the scope-stats struct comment group (cosmetic), and add tests for the four surviving mutants (singleflight recheck, per-row AmbiguousHops expansion, listener-only exclusion, declarationAge hours formatting). --- cmd/server/routes.go | 4 +- cmd/server/scope_audit.go | 13 +++- cmd/server/scope_audit_test.go | 108 ++++++++++++++++++++++++++++++++- test-scope-audit.js | 1 + 4 files changed, 123 insertions(+), 3 deletions(-) diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 8998161b0..04d97a8a7 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -83,11 +83,13 @@ type Server struct { advertRoutes nodeAdvertRouteCache // Cached /api/scope-stats response — per-window, recomputed at most once every 30s - scopeAudit scopeAuditCache scopeStatsMu sync.Mutex scopeStatsCache map[string]*ScopeStatsResponse scopeStatsCachedAt map[string]time.Time + // Cached /api/scope-audit response — per-window, recomputed at most once every 30s + scopeAudit scopeAuditCache + // Cached /api/analytics/wardriving response — per-window, recomputed at most once every 30s wardrivingStatsMu sync.Mutex wardrivingStatsCache map[string]*WardrivingStatsResponse diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go index c2b2e3578..dd44d0795 100644 --- a/cmd/server/scope_audit.go +++ b/cmd/server/scope_audit.go @@ -70,6 +70,11 @@ type scopeAuditCache struct { mu sync.Mutex entries map[string]scopeAuditCacheEntry sf singleflight.Group + // recheckHook, if non-nil, runs once after the outer cache miss and + // before entering singleflight. Tests use it to land a concurrent fill + // in that gap and prove the recheck inside Do reuses it instead of + // rebuilding; nil (zero overhead) in production. + recheckHook func() } func (c *scopeAuditCache) get(window string) *ScopeAuditResponse { @@ -87,6 +92,9 @@ func (c *scopeAuditCache) load(window string, build func() (*ScopeAuditResponse, if cached := c.get(window); cached != nil { return cached, nil } + if c.recheckHook != nil { + c.recheckHook() + } value, err, _ := c.sf.Do(window, func() (interface{}, error) { if cached := c.get(window); cached != nil { return cached, nil @@ -410,7 +418,10 @@ func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { return } response, err := s.scopeAudit.load(window, func() (*ScopeAuditResponse, error) { - ctx, cancel := context.WithTimeout(r.Context(), scopeAuditQueryTimeout) + // The build is shared across every coalesced singleflight waiter, so it + // must not die with the first caller's connection: detach from its + // cancellation and keep only the absolute deadline. + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), scopeAuditQueryTimeout) defer cancel() return s.buildScopeAuditContext(ctx, window, time.Now()) }) diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index c9306f477..8ea6e93a9 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "net/http/httptest" @@ -88,7 +89,7 @@ func TestScopeAuditCollisionAndWindow(t *testing.T) { if err != nil { t.Fatal(err) } - if r.AmbiguousHops != 1 || r.Rows[0].Forwarded != 0 || !r.Rows[0].IncompleteEvidence { + if r.AmbiguousHops != 1 || r.Rows[0].Forwarded != 0 || !r.Rows[0].IncompleteEvidence || r.Rows[0].AmbiguousHops != 1 { t.Fatalf("1h=%+v", r) } r, err = s.buildScopeAudit("24h", now) @@ -343,6 +344,111 @@ func TestScopeAuditCachedHiddenNameAndUnsupportedSchema(t *testing.T) { } } +// The shared singleflight build must not die with the first caller's +// connection: a coalesced caller whose own context stays live must still +// get a response when another caller's context is canceled mid-build. +func TestScopeAuditHandlerSingleflightShareSurvivesFirstCallerDisconnect(t *testing.T) { + s, node, packet := auditFixture(t) + now := time.Now().UTC() + node(scopeContractMultiPK, "relay", "dk", now.Format(time.RFC3339)) + dk := "dk" + // Enough observations that the build takes long enough for both callers + // to join the same singleflight call before the first's cancel lands. + for i := 1; i <= 6000; i++ { + packet(i, 0, &dk, `["e1","aa","bb","cc"]`, "0001020304", now) + } + + cancelCtx, cancel := context.WithCancel(context.Background()) + var wg sync.WaitGroup + codes := make(chan int, 2) + wg.Add(1) + go func() { + defer wg.Done() + req := httptest.NewRequest("GET", "/api/scope-audit?window=24h", nil).WithContext(cancelCtx) + w := httptest.NewRecorder() + s.handleScopeAudit(w, req) + codes <- w.Code + }() + // Give caller A a head start so it reliably becomes the singleflight + // leader (whose context drives the build) before caller B joins it. + time.Sleep(2 * time.Millisecond) + wg.Add(1) + go func() { + defer wg.Done() + req := httptest.NewRequest("GET", "/api/scope-audit?window=24h", nil) + w := httptest.NewRecorder() + s.handleScopeAudit(w, req) + codes <- w.Code + }() + time.Sleep(2 * time.Millisecond) + cancel() // simulate caller A's connection dropping mid-build + wg.Wait() + close(codes) + sawLiveCallerSucceed := false + for code := range codes { + if code == 200 { + sawLiveCallerSucceed = true + } + } + if !sawLiveCallerSucceed { + t.Fatal("a disconnecting caller's context cancellation took down the shared build for a still-connected caller") + } +} + +// A declared repeater that the observers table marks listener-only +// (can_relay=0, #1290) must not have its own hop token confirmed as a +// relay — the same exclusion resolveWithContext applies elsewhere. +func TestScopeAuditListenerOnlyNodeExcludedFromForwardedCount(t *testing.T) { + s, node, packet := auditFixture(t) + now := time.Now().UTC() + at := now.Format(time.RFC3339) + node(scopeContractMultiPK, "listener", "dk", at) + if _, err := s.db.conn.Exec(`ALTER TABLE observers ADD COLUMN can_relay INTEGER DEFAULT 1`); err != nil { + t.Fatal(err) + } + if _, err := s.db.conn.Exec( + `INSERT INTO observers(id,name,last_seen,first_seen,packet_count,can_relay) VALUES(?,'obs',?,?,1,0)`, + scopeContractMultiPK, at, at); err != nil { + t.Fatal(err) + } + dk := "dk" + packet(1, 0, &dk, `["e1"]`, "0001020304", now) + r, err := s.buildScopeAudit("24h", now) + if err != nil { + t.Fatal(err) + } + if len(r.Rows) != 1 || r.Rows[0].Forwarded != 0 || r.Rows[0].Status != "no-evidence" { + t.Fatalf("listener-only hop counted as a confirmed relay: %+v", r.Rows[0]) + } +} + +// load rechecks the cache after entering singleflight (comment on scopeAuditCache.load): +// a fill that lands in the gap between the outer miss and entering singleflight +// must be reused instead of triggering a redundant rebuild. +func TestScopeAuditCacheRecheckInsideSingleflightReusesConcurrentFill(t *testing.T) { + var cache scopeAuditCache + want := &ScopeAuditResponse{Window: "fresh-from-concurrent-fill"} + cache.recheckHook = func() { + cache.mu.Lock() + cache.entries = map[string]scopeAuditCacheEntry{"24h": {response: want, at: time.Now()}} + cache.mu.Unlock() + } + var built bool + got, err := cache.load("24h", func() (*ScopeAuditResponse, error) { + built = true + return &ScopeAuditResponse{Window: "should-not-be-used"}, nil + }) + if err != nil { + t.Fatal(err) + } + if built { + t.Fatal("build ran despite a concurrent fill landing before the singleflight entry") + } + if got != want { + t.Fatalf("got=%+v want=%+v", got, want) + } +} + func TestScopeAuditNoDeclarationsSkipsObservationScan(t *testing.T) { s, _, _ := auditFixture(t) // If the cold path scans observations despite no declarations, this fails. diff --git a/test-scope-audit.js b/test-scope-audit.js index 9e5c38eb4..86ddb8e32 100644 --- a/test-scope-audit.js +++ b/test-scope-audit.js @@ -6,6 +6,7 @@ assert.strictEqual(audit.windowValue('7d'), '7d'); assert.strictEqual(audit.searchRows([{name: 'ALPHA', publicKey: 'abc'}, {name: 'Beta', publicKey: 'def'}], 'alpha').length, 1); assert.strictEqual(audit.searchRows([{name: null, publicKey: 123}], '123').length, 1); assert.strictEqual(audit.declarationAge('86400'), '1d'); +assert.strictEqual(audit.declarationAge('7200'), '2h'); assert.strictEqual(audit.declarationAge(null), 'Unknown'); assert.strictEqual(audit.stateFromHash('#/analytics?tab=scopes&sub=audit&swin=7d&saq=a%26b').query, 'a&b'); assert.strictEqual(audit.stateFromHash('#/analytics?tab=scopes&sub=audit&swin=no').window, '24h');