diff --git a/.eslintrc.json b/.eslintrc.json index 6c4252dce..ce7dfa49e 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -35,6 +35,7 @@ "MAX_HOP_DIST": "readonly", "MeshAudio": "readonly", "MeshConfigReady": "readonly", + "NodeAdverts": "readonly", "PAYLOAD_COLORS": "readonly", "PAYLOAD_TYPES": "readonly", "PERF_SLOW_MS": "readonly", diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e9555c3ce..833573c2b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -189,6 +189,7 @@ jobs: node test-coverage-gate.js node test-node-reach-coverage.js node test-reach-rank.js + node test-node-adverts.js node test-issue-1107-live-layout.js node test-issue-1833-legend-toggle-vcr-offset.js node test-issue-1532-live-fullscreen.js @@ -430,6 +431,11 @@ jobs: # required shape before the server boots. run: ./corescope-migrate -db test-fixtures/e2e-fixture.db + - name: Seed route-classified adverts for #2073 E2E + # Runs after the migration because it sets transmissions.route_mask + # (#89). See the file header for the rows and expected classes. + run: sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-2073-route-adverts.sql + - name: Start Go server with fixture DB run: | fuser -k 13581/tcp 2>/dev/null || true @@ -500,6 +506,7 @@ jobs: BASE_URL=http://localhost:13581 node test-issue-1146-path-link-contrast-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 node test-issue-1705-subpath-contrast-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1147-section-order-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node test-issue-2073-recent-adverts-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1151-orphan-separators-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1486-collapse-reopens-detail-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-rebrand-e2e.js 2>&1 | tee -a e2e-output.txt diff --git a/cmd/server/advert_stats.go b/cmd/server/advert_stats.go index a67d790f6..4d82f57f0 100644 --- a/cmd/server/advert_stats.go +++ b/cmd/server/advert_stats.go @@ -16,28 +16,62 @@ type floodAdvertEntry struct { hash string } -// countFloodAdverts counts distinct flood adverts (route_type == -// advertRouteTypeFlood) whose first-seen lies within the past windowHours. Entries -// with unparseable timestamps are skipped, matching relay-liveness behaviour; +// advertWindow is the exact first-seen window shared by the windowed advert +// counters (flood_advert_count_7d and the #2073 advertCounts). Entries with +// unparseable timestamps are skipped, matching relay-liveness behaviour; // entries without a hash fall back to their timestamp as the dedup key. +type advertWindow struct { + cutoff time.Time + seen map[string]struct{} +} + +func newAdvertWindow(now time.Time, windowHours float64) advertWindow { + return advertWindow{ + cutoff: now.Add(-time.Duration(windowHours * float64(time.Hour))), + seen: map[string]struct{}{}, + } +} + +// admit reports whether an advert first seen at ts lies inside the window +// and has not been counted yet (by hash), and records it. +func (w advertWindow) admit(ts, hash string) bool { + t, ok := parseRelayTS(ts) + if !ok || !t.After(w.cutoff) { + return false + } + key := hash + if key == "" { + key = ts + } + if _, dup := w.seen[key]; dup { + return false + } + w.seen[key] = struct{}{} + return true +} + +// advertDateFloor is the SQL pre-filter for an advert window: a DATE-ONLY +// string one day before the window start. A date prefix compares lexically +// the same whatever follows it (separator, precision, zone suffix), so the +// floor never drops an in-window row; the exact check is advertWindow.admit +// in Go, which skips what parseRelayTS cannot parse (e.g. a legacy +// 'YYYY-MM-DD HH:MM:SS' value). +func advertDateFloor(now time.Time, windowHours float64) string { + return now.UTC().Add(-time.Duration(windowHours*float64(time.Hour))).AddDate(0, 0, -1).Format("2006-01-02") +} + +// countFloodAdverts counts distinct flood adverts (route_type == +// advertRouteTypeFlood) whose first-seen lies within the past windowHours +// (see advertWindow). func countFloodAdverts(entries []floodAdvertEntry, now time.Time, windowHours float64) int { - cutoff := now.Add(-time.Duration(windowHours * float64(time.Hour))) - seen := map[string]struct{}{} + w := newAdvertWindow(now, windowHours) + n := 0 for _, e := range entries { - if e.rt != advertRouteTypeFlood { - continue - } - t, ok := parseRelayTS(e.ts) - if !ok || !t.After(cutoff) { - continue + if e.rt == advertRouteTypeFlood && w.admit(e.ts, e.hash) { + n++ } - key := e.hash - if key == "" { - key = e.ts - } - seen[key] = struct{}{} } - return len(seen) + return n } // CountFloodAdvertsForNode returns how many distinct FLOOD adverts pubkey @@ -47,9 +81,17 @@ func countFloodAdverts(entries []floodAdvertEntry, now time.Time, windowHours fl // // route_type is filtered in SQL so an advert-spamming node cannot truncate // the flood count (review feedback on the earlier LIMIT approach). The time -// floor is a DATE-ONLY string with one day of slack: a date prefix compares -// lexically the same across every first_seen format parseRelayTS accepts -// ('T' and ' ' separators alike); the exact window check stays in Go. +// floor is advertDateFloor; the exact window check stays in Go. +// +// This is an external contract (the ArcScope advisor reads it) and is kept +// as is, counted fresh on every request and never cached: route_type 1 +// only, i.e. the first-inserted route. It therefore differs from +// advertCounts["7d"].flood (node_advert_routes.go), which is route_mask +// based: that one also counts transport flood (route 0) and never counts a +// mixed advert (flood + zero-hop), while this one counts a mixed advert +// whenever its first-inserted route was 1. When a node-detail request scans +// the node for the #2073 breakdown itself, that scan yields this same number +// (GetNodeAdvertRoutes, pinned by a parity test) and this query is skipped. // // The row cap is a pure safety valve on per-request allocation: it applies to // flood adverts inside the floor window only, and 50000 in ~8 days is ~4 per @@ -61,7 +103,14 @@ func countFloodAdverts(entries []floodAdvertEntry, now time.Time, windowHours fl const floodAdvertRowCap = 50000 func (db *DB) CountFloodAdvertsForNode(pubkey string, windowHours float64, rowCap int) (int, error) { - floor := time.Now().UTC().Add(-time.Duration(windowHours*float64(time.Hour))).AddDate(0, 0, -1).Format("2006-01-02") + return db.countFloodAdvertsForNodeAt(pubkey, windowHours, rowCap, time.Now()) +} + +// countFloodAdvertsForNodeAt is CountFloodAdvertsForNode at a given instant, +// for the parity test of the #2073 scan that can stand in for it +// (GetNodeAdvertRoutes). +func (db *DB) countFloodAdvertsForNodeAt(pubkey string, windowHours float64, rowCap int, now time.Time) (int, error) { + floor := advertDateFloor(now, windowHours) rows, err := db.conn.Query( "SELECT COALESCE(first_seen, ''), COALESCE(route_type, -1), COALESCE(hash, '') FROM transmissions WHERE from_pubkey = ? AND payload_type = ? AND route_type = ? AND first_seen >= ? ORDER BY id DESC LIMIT ?", pubkey, payloadTypeAdvert, advertRouteTypeFlood, floor, rowCap) @@ -77,5 +126,5 @@ func (db *DB) CountFloodAdvertsForNode(pubkey string, windowHours float64, rowCa } entries = append(entries, e) } - return countFloodAdverts(entries, time.Now(), windowHours), nil + return countFloodAdverts(entries, now, windowHours), nil } diff --git a/cmd/server/db.go b/cmd/server/db.go index c2079fd81..427a8458a 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -1248,19 +1248,29 @@ func (db *DB) GetNodeByPubkey(pubkey string) (map[string]interface{}, error) { // same-name false positives and an adversarial spoof path where any node // could attribute its transmissions to a victim by naming itself with the // victim's pubkey. Pubkey is unique by design — that's the whole point. -func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int) ([]map[string]interface{}, error) { +// +// withRouteClass (node detail with include=advertRoutes, #2073) adds +// route_class to the ADVERT rows; without it the rows are unchanged. +func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int, withRouteClass bool) (NodeAdvertRows, error) { if limit <= 0 { limit = 20 } - - selectCols, observerJoin := db.transmissionBaseSQL() - // #1345: order by ingest id, not first_seen (=rxTime). Buffered observer // uploads with old rxTime would otherwise displace fresh activity from // the "recent transmissions for node" list. - querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE t.from_pubkey = ? ORDER BY t.id DESC LIMIT ?", - selectCols, observerJoin) - args := []interface{}{pubkey, limit} + return db.queryNodeAdvertRows(true, withRouteClass, "t.from_pubkey = ? ORDER BY t.id DESC LIMIT ?", pubkey, limit) +} + +// queryNodeAdvertRows runs the transmission-centric query with the given +// WHERE tail (including ORDER BY/LIMIT). With observations it attaches every +// observation (otherwise the rows keep the transmission shape and the best +// observation's fields only: the lean #2073 per-class rows). With +// routeClass it adds, for ADVERTs, route_class (#2073: classifyAdvertRoute +// over route_mask with the route_type fallback); a failed mask read only +// leaves route_class out. +func (db *DB) queryNodeAdvertRows(observations, routeClass bool, whereTail string, args ...interface{}) (NodeAdvertRows, error) { + selectCols, observerJoin := db.transmissionBaseSQL() + querySQL := fmt.Sprintf("SELECT %s FROM transmissions t %s WHERE %s", selectCols, observerJoin, whereTail) rows, err := db.conn.Query(querySQL, args...) if err != nil { @@ -1268,22 +1278,30 @@ func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int) ([]map[str } defer rows.Close() - packets := make([]map[string]interface{}, 0) + packets := make(NodeAdvertRows, 0) var txIDs []int for rows.Next() { p := db.scanTransmissionRow(rows) if p != nil { - // Placeholder for observations — filled below - p["observations"] = []map[string]interface{}{} + if observations { + // Placeholder for observations — filled below + p["observations"] = []map[string]interface{}{} + } if id, ok := p["id"].(int); ok { txIDs = append(txIDs, id) } packets = append(packets, p) } } + if err := rows.Err(); err != nil { + return nil, err + } + if len(txIDs) == 0 { + return packets, nil + } // Fetch observations for all transmissions - if len(txIDs) > 0 { + if observations { obsMap := db.getObservationsForTransmissions(txIDs) for _, p := range packets { if id, ok := p["id"].(int); ok { @@ -1293,7 +1311,21 @@ func (db *DB) GetRecentTransmissionsForNode(pubkey string, limit int) ([]map[str } } } - + if !routeClass { + return packets, nil + } + masks := make(map[int]sql.NullInt64, len(txIDs)) + if db.hasRouteMask() { + if err := db.readRouteMasks(txIDs, masks); err != nil { + log.Printf("[node-adverts] route_mask read failed, route_class omitted: %v", err) + return packets, nil + } + } + for _, p := range packets { + if id, ok := p["id"].(int); ok { + setAdvertRouteClass(p, masks[id]) + } + } return packets, nil } diff --git a/cmd/server/node_advert_routes.go b/cmd/server/node_advert_routes.go new file mode 100644 index 000000000..c65ed6805 --- /dev/null +++ b/cmd/server/node_advert_routes.go @@ -0,0 +1,266 @@ +package main + +import ( + "database/sql" + "fmt" + "net/http" + "strings" + "time" + + "github.com/meshcore-analyzer/packetpath" +) + +// Node-detail advert route breakdown: a port/extension of upstream +// `Kpa-clawbot/CoreScope#2073`. recentAdvertsByRoute lists a node's newest +// adverts per route class and advertCounts counts them per class over 24h +// and 7d. Both classify exactly like Relay Airtime Share (#89, +// classifyAdvertRoute): route_mask when the ingestor has set it, else the +// first-inserted route_type. The no-usable-route class is "unknown" here and +// "legacy" in Relay Airtime Share's route_class (its historical plain ADVERT +// row); both name the same relayAirtimeAdvertUnknown bucket. +// +// The breakdown is opt-in: GET /api/nodes/{pubkey} computes it only with +// include=advertRoutes, which only the node page sends. + +// Route class names on the wire (NodeAdvertsByRoute keys, route_class). +const ( + advertClassFlood = "flood" + advertClassZeroHop = "zero_hop" + advertClassMixed = "mixed" + advertClassUnknown = "unknown" +) + +// nodeAdvertRouteLimit is the per-class list length, the same 20 as the +// chronological recentAdverts list. +const nodeAdvertRouteLimit = 20 + +func advertRouteClassName(c relayAirtimeAdvertRoute) string { + switch c { + case relayAirtimeAdvertFlood: + return advertClassFlood + case relayAirtimeAdvertZeroHop: + return advertClassZeroHop + case relayAirtimeAdvertMixed: + return advertClassMixed + } + return advertClassUnknown +} + +// nodeDetailIncludeAdvertRoutes is the include value of GET +// /api/nodes/{pubkey} that opts in to the breakdown. +const nodeDetailIncludeAdvertRoutes = "advertRoutes" + +// wantsNodeAdvertRoutes reports whether the request opted in with +// include=advertRoutes (the include parameter may repeat and holds a +// comma-separated list). +func wantsNodeAdvertRoutes(r *http.Request) bool { + for _, v := range r.URL.Query()["include"] { + for _, item := range strings.Split(v, ",") { + if strings.TrimSpace(item) == nodeDetailIncludeAdvertRoutes { + return true + } + } + } + return false +} + +// classifyAdvertRouteRow classifies a transmissions row from its route_mask +// and route_type columns (NULL mask: not backfilled or no column). +func classifyAdvertRouteRow(mask, routeType sql.NullInt64) relayAirtimeAdvertRoute { + var rt *int + if routeType.Valid { + v := int(routeType.Int64) + rt = &v + } + return classifyAdvertRoute(mask.Int64, mask.Valid, rt) +} + +// setAdvertRouteClass adds route_class to an ADVERT row from +// scanTransmissionRow; other payload types have no advert route class. +func setAdvertRouteClass(p NodeAdvertRow, mask sql.NullInt64) { + if pt, ok := p["payload_type"].(int); !ok || pt != payloadTypeAdvert { + return + } + var routeType sql.NullInt64 + if rt, ok := p["route_type"].(int); ok { + routeType = sql.NullInt64{Int64: int64(rt), Valid: true} + } + p["route_class"] = advertRouteClassName(classifyAdvertRouteRow(mask, routeType)) +} + +// advertRouteEntry is one advert inside the count floor: first-seen, hash +// (dedup key) and route class. +type advertRouteEntry struct { + ts, hash, class string +} + +// countAdvertRoutes counts distinct adverts per class whose first-seen lies +// in the past windowHours (advertWindow: exact parseRelayTS window, dedup by +// hash, as flood_advert_count_7d). +func countAdvertRoutes(entries []advertRouteEntry, now time.Time, windowHours float64) AdvertRouteCounts { + w := newAdvertWindow(now, windowHours) + var c AdvertRouteCounts + for _, e := range entries { + if !w.admit(e.ts, e.hash) { + continue + } + switch e.class { + case advertClassFlood: + c.Flood++ + case advertClassZeroHop: + c.ZeroHop++ + case advertClassMixed: + c.Mixed++ + default: + c.Unknown++ + } + } + return c +} + +// nodeAdvertScanSQL reads a node's ADVERT rows newest id first, in the order +// of idx_transmissions_from_pubkey (no sort, no window). Beside the class +// inputs it returns the two predicates of CountFloodAdvertsForNode as SQL +// evaluates them - first_seen >= floor and route_type = 1 - so the flood +// count derived from this scan is that function's bit for bit, whatever the +// stored types. The class inputs are normalized to what the SQL classifier +// (and TestAdvertRouteClassSQL_MatchesGoClassifier) saw: route_type only as +// one of the four route values, route_mask only in its route bits. The unary +// + on payload_type keeps it off idx_transmissions_payload_type, so the +// planner always walks idx_transmissions_from_pubkey: without ANALYZE +// statistics (neither binary runs it) the choice between the two otherwise +// depends on index creation order and SQLite version, and the payload_type +// index would visit every ADVERT in the database. For an INTEGER column +// "+payload_type = ?" matches exactly the rows "payload_type = ?" does. +// Parameters: floor, flood route type, pubkey, payload type. +func nodeAdvertScanSQL(hasRouteMask bool) string { + mask := "NULL" + if hasRouteMask { + mask = fmt.Sprintf("route_mask & %d", packetpath.RouteMaskAll) + } + return fmt.Sprintf(`SELECT id, + CASE WHEN route_type IN (%d, %d, %d, %d) THEN CAST(route_type AS INTEGER) END, + %s, + COALESCE(first_seen, ''), COALESCE(hash, ''), + COALESCE(first_seen >= ?, 0), COALESCE(route_type = ?, 0) + FROM transmissions WHERE from_pubkey = ? AND +payload_type = ? + ORDER BY id DESC`, + RouteTransportFlood, RouteFlood, RouteDirect, RouteTransportDirect, mask) +} + +// nodeAdvertScan is what one pass over a node's ADVERT rows yields: the ids +// to list (newest first) with their class, the per-class counts and +// flood_advert_count_7d. +type nodeAdvertScan struct { + ids []int + classes []string // classes[i] is the class ids[i] is listed under + counts NodeAdvertCounts + flood7d int +} + +// scanNodeAdvertRoutes is the pass behind GetNodeAdvertRoutes (see there). +func (db *DB) scanNodeAdvertRoutes(pubkey string, perClass int, now time.Time, rowCap int) (nodeAdvertScan, error) { + var sc nodeAdvertScan + const floodWindowHours = 7 * 24 + floor := advertDateFloor(now, floodWindowHours) + rows, err := db.conn.Query(nodeAdvertScanSQL(db.hasRouteMask()), floor, advertRouteTypeFlood, pubkey, payloadTypeAdvert) + if err != nil { + return sc, err + } + defer rows.Close() + perClassSeen := map[string]int{} + var entries []advertRouteEntry + var floods []floodAdvertEntry + for rows.Next() { + var id, inFloor, isFlood int + var routeType, mask sql.NullInt64 + var ts, hash string + if err := rows.Scan(&id, &routeType, &mask, &ts, &hash, &inFloor, &isFlood); err != nil { + return sc, err + } + cls := advertRouteClassName(classifyAdvertRouteRow(mask, routeType)) + if perClassSeen[cls] < perClass { + perClassSeen[cls]++ + sc.ids = append(sc.ids, id) + sc.classes = append(sc.classes, cls) + } + if inFloor == 0 { + continue + } + if len(entries) < rowCap { + entries = append(entries, advertRouteEntry{ts: ts, hash: hash, class: cls}) + } else { + sc.counts.Truncated = true + } + if isFlood != 0 && len(floods) < rowCap { + floods = append(floods, floodAdvertEntry{ts: ts, rt: advertRouteTypeFlood, hash: hash}) + } + } + if err := rows.Err(); err != nil { + return sc, err + } + sc.counts.H24 = countAdvertRoutes(entries, now, 24) + sc.counts.D7 = countAdvertRoutes(entries, now, floodWindowHours) + sc.flood7d = countFloodAdverts(floods, now, floodWindowHours) + return sc, nil +} + +// GetNodeAdvertRoutes returns pubkey's newest perClass adverts per route +// class, its per-class advert counts for 24h and 7d, and its +// flood_advert_count_7d. +// +// One index-ordered pass over the node's ADVERT rows (nodeAdvertScanSQL) +// serves all three, because visiting the rows - a table lookup each after +// the from_pubkey index - is the cost on an advert-spamming node. Go +// classifies each row (classifyAdvertRoute) and keeps the first perClass rows +// of every class, so the class filter runs before the limit; every row at or +// after the 7d date floor also feeds the counts. The counts window is +// first_seen - when the advert was first heard, the same axis as +// flood_advert_count_7d - not the ingest id that orders the lists (#1345), so +// a late buffered upload counts in the window it was sent in. +// +// rowCap (> 0) bounds the count entries held per request like +// CountFloodAdvertsForNode (newest rows win, Truncated reports it); the +// lists are unaffected. flood7d equals CountFloodAdvertsForNode(pubkey, 7*24, +// rowCap) evaluated at now on the same rows: its own rowCap over the rows +// with route_type = 1 inside the floor, then countFloodAdverts +// (TestNodeAdvertRoutes_FloodCountMatchesCountFloodAdvertsForNode). The full +// rows of the listed ids are read in one batch afterwards. RouteMaskBackfill +// is left for the caller (Server.nodeAdvertRoutes, which also caches the +// lists and counts, never flood7d). +func (db *DB) GetNodeAdvertRoutes(pubkey string, perClass int, now time.Time, rowCap int) (NodeAdvertsByRoute, NodeAdvertCounts, int, error) { + byRoute := NodeAdvertsByRoute{Limit: perClass, Flood: NodeAdvertRows{}, ZeroHop: NodeAdvertRows{}, Mixed: NodeAdvertRows{}} + sc, err := db.scanNodeAdvertRoutes(pubkey, perClass, now, rowCap) + if err != nil || len(sc.ids) == 0 { + return byRoute, sc.counts, sc.flood7d, err + } + // Lean rows (no observation arrays): they are cached and sit next to + // the full recentAdverts rows. route_class is the class the rows were + // selected by, so a row never disagrees with its list even if the + // ingestor widens its route_mask meanwhile. + listed := make(map[int]string, len(sc.ids)) + args := make([]interface{}, len(sc.ids)) + for i, id := range sc.ids { + listed[id] = sc.classes[i] + args[i] = id + } + full, err := db.queryNodeAdvertRows(false, false, "t.id IN ("+strings.TrimSuffix(strings.Repeat("?,", len(args)), ",")+") ORDER BY t.id DESC", args...) + if err != nil { + return byRoute, sc.counts, 0, err + } + for _, p := range full { + id, _ := p["id"].(int) + p["route_class"] = listed[id] + switch listed[id] { + case advertClassFlood: + byRoute.Flood = append(byRoute.Flood, p) + case advertClassZeroHop: + byRoute.ZeroHop = append(byRoute.ZeroHop, p) + case advertClassMixed: + byRoute.Mixed = append(byRoute.Mixed, p) + default: + byRoute.Unknown = append(byRoute.Unknown, p) + } + } + return byRoute, sc.counts, sc.flood7d, nil +} diff --git a/cmd/server/node_advert_routes_bench_test.go b/cmd/server/node_advert_routes_bench_test.go new file mode 100644 index 000000000..8ae870e38 --- /dev/null +++ b/cmd/server/node_advert_routes_bench_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "os" + "testing" + "time" +) + +// Benchmarks for the #2073 node-detail advert route breakdown: the stream +// (which also yields flood_advert_count_7d), the window query it replaced, +// and the queries a node-detail request runs anyway. They need a +// large database, so they run only against NODE_ADVERT_BENCH_DB (a migrated +// CoreScope SQLite file) with NODE_ADVERT_BENCH_PUBKEY as the node to read: +// +// NODE_ADVERT_BENCH_DB=/path/bench.db NODE_ADVERT_BENCH_PUBKEY= \ +// go test -run '^$' -bench NodeAdvertRoutes -benchtime 50x . +func nodeAdvertBenchDB(b *testing.B) (*DB, string) { + b.Helper() + path, pubkey := os.Getenv("NODE_ADVERT_BENCH_DB"), os.Getenv("NODE_ADVERT_BENCH_PUBKEY") + if path == "" || pubkey == "" { + b.Skip("set NODE_ADVERT_BENCH_DB and NODE_ADVERT_BENCH_PUBKEY") + } + db, err := OpenDB(path) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { db.conn.Close() }) + return db, pubkey +} + +func BenchmarkNodeAdvertRoutes(b *testing.B) { + db, pubkey := nodeAdvertBenchDB(b) + for i := 0; i < b.N; i++ { + if _, _, _, err := db.GetNodeAdvertRoutes(pubkey, nodeAdvertRouteLimit, time.Now(), floodAdvertRowCap); err != nil { + b.Fatal(err) + } + } +} + +// The ROW_NUMBER() window query GetNodeAdvertRoutes replaced (PR #97 review +// P2-1), kept as the test-only oracle. +func BenchmarkNodeAdvertRoutes_WindowOracle(b *testing.B) { + db, pubkey := nodeAdvertBenchDB(b) + for i := 0; i < b.N; i++ { + if _, _, err := oracleNodeAdvertRoutesWindow(db, pubkey, nodeAdvertRouteLimit, time.Now(), floodAdvertRowCap); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkNodeAdvertRoutes_FloodAdvertCount7d(b *testing.B) { + db, pubkey := nodeAdvertBenchDB(b) + for i := 0; i < b.N; i++ { + if _, err := db.CountFloodAdvertsForNode(pubkey, 7*24, floodAdvertRowCap); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkNodeAdvertRoutes_RecentAdverts(b *testing.B) { + db, pubkey := nodeAdvertBenchDB(b) + for i := 0; i < b.N; i++ { + if _, err := db.GetRecentTransmissionsForNode(pubkey, 20, true); err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/server/node_advert_routes_cache.go b/cmd/server/node_advert_routes_cache.go new file mode 100644 index 000000000..1f5bd9fcf --- /dev/null +++ b/cmd/server/node_advert_routes_cache.go @@ -0,0 +1,170 @@ +package main + +import ( + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +// The #2073 advert route breakdown (asked for with include=advertRoutes) +// scans all of a node's ADVERT rows: a table lookup per row, ~110 ms for a +// node with 50,000 adverts on a 1,000,000-transmission database on an Apple +// M5 and about three times that on a 4-vCPU server +// (node_advert_routes_bench_test.go). The result is cached per pubkey so +// repeated views do not rescan: +// - nodeAdvertRouteTTL bounds staleness from changes that do not add a +// row - a route_mask promoted to mixed, the backfill, the windows +// sliding; +// - a newer transmission from the node invalidates its entry alone (read +// from the covering from_pubkey index), debounced so a node that adverts +// every few seconds is rescanned at most once per nodeAdvertRouteDebounce; +// - at most nodeAdvertRouteCacheMax entries, expired ones dropped on every +// put and the oldest evicted first. An entry holds up to 4 x 20 lean +// rows (no observation arrays, but raw_hex and decoded_json), measured +// at ~2.5 KB of heap per row: ~125 KB for a typical entry (~52 rows), +// ~200 KB at the 80-row maximum, so ~16 MB for 128 typical entries and +// at most ~25 MB; +// - concurrent misses for one pubkey share a single scan (singleflight). +// +// These are hardcoded for now; like the other cache TTLs they are candidates +// for config. +const ( + nodeAdvertRouteTTL = 30 * time.Second + nodeAdvertRouteDebounce = 5 * time.Second + nodeAdvertRouteCacheMax = 128 +) + +type nodeAdvertRouteEntry struct { + byRoute NodeAdvertsByRoute + counts NodeAdvertCounts + latestID int64 // the node's newest transmission id when computed + at time.Time +} + +// nodeAdvertRouteCache is usable as a zero value. Cached rows are shared +// between responses and never mutated after put. +type nodeAdvertRouteCache struct { + mu sync.Mutex + entries map[string]nodeAdvertRouteEntry + flight singleflight.Group + onLookup func() // test hook: called once per nodeAdvertRoutes call + onScan func() // test hook: called once per scan +} + +func (c *nodeAdvertRouteCache) get(pubkey string, latestID int64, now time.Time) (nodeAdvertRouteEntry, bool) { + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[pubkey] + if !ok { + return e, false + } + age := now.Sub(e.at) + if age >= nodeAdvertRouteTTL || (e.latestID != latestID && age >= nodeAdvertRouteDebounce) { + delete(c.entries, pubkey) + return e, false + } + return e, true +} + +// put stores e, dropping expired entries first and then, if still full, the +// oldest one. O(nodeAdvertRouteCacheMax) under the lock, no I/O. +func (c *nodeAdvertRouteCache) put(pubkey string, e nodeAdvertRouteEntry, now time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + if c.entries == nil { + c.entries = make(map[string]nodeAdvertRouteEntry) + } + for k, v := range c.entries { + if now.Sub(v.at) >= nodeAdvertRouteTTL { + delete(c.entries, k) + } + } + if _, ok := c.entries[pubkey]; !ok && len(c.entries) >= nodeAdvertRouteCacheMax { + oldest := "" + for k, v := range c.entries { + if oldest == "" || v.at.Before(c.entries[oldest].at) { + oldest = k + } + } + delete(c.entries, oldest) + } + c.entries[pubkey] = e +} + +func (c *nodeAdvertRouteCache) len() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.entries) +} + +// latestTransmissionIDForNode is the node's newest transmission id (0 when +// none), an index-only read of idx_transmissions_from_pubkey. +func (db *DB) latestTransmissionIDForNode(pubkey string) (int64, error) { + var id int64 + err := db.conn.QueryRow(`SELECT COALESCE(MAX(id), 0) FROM transmissions WHERE from_pubkey = ?`, pubkey).Scan(&id) + return id, err +} + +// nodeAdvertRouteResult is the breakdown for one request. floodAdvertCount7d +// is set only when this request ran the scan itself (not a cache hit, not a +// scan shared with a concurrent request): then it is the node's fresh +// flood_advert_count_7d and CountFloodAdvertsForNode can be skipped. It is +// never cached. +type nodeAdvertRouteResult struct { + byRoute NodeAdvertsByRoute + counts NodeAdvertCounts + floodAdvertCount7d *int +} + +// nodeAdvertRouteScan is what one scan hands to singleflight. +type nodeAdvertRouteScan struct { + entry nodeAdvertRouteEntry + flood7d int +} + +// nodeAdvertRoutes is GetNodeAdvertRoutes through the per-pubkey cache; the +// route_mask backfill status is read fresh (it has its own short cache). +// Callers apply the privacy gates first: this is only reached for a visible +// identity that asked for the breakdown. +func (s *Server) nodeAdvertRoutes(pubkey string, now time.Time) (nodeAdvertRouteResult, error) { + c := &s.advertRoutes + if c.onLookup != nil { + c.onLookup() + } + latestID, err := s.db.latestTransmissionIDForNode(pubkey) + if err != nil { + return nodeAdvertRouteResult{}, err + } + var res nodeAdvertRouteResult + e, ok := c.get(pubkey, latestID, now) + if !ok { + // singleflight runs fn in the calling goroutine of the one request + // that scans; the others wait and see scanned == false. + scanned := false + v, err, _ := c.flight.Do(pubkey, func() (interface{}, error) { + scanned = true + if c.onScan != nil { + c.onScan() + } + byRoute, counts, flood7d, err := s.db.GetNodeAdvertRoutes(pubkey, nodeAdvertRouteLimit, now, floodAdvertRowCap) + if err != nil { + return nil, err + } + fresh := nodeAdvertRouteEntry{byRoute: byRoute, counts: counts, latestID: latestID, at: now} + c.put(pubkey, fresh, now) + return nodeAdvertRouteScan{entry: fresh, flood7d: flood7d}, nil + }) + if err != nil { + return nodeAdvertRouteResult{}, err + } + scan := v.(nodeAdvertRouteScan) + e = scan.entry + if scanned { + res.floodAdvertCount7d = &scan.flood7d + } + } + res.byRoute, res.counts = e.byRoute, e.counts + res.counts.RouteMaskBackfill = s.db.routeMaskBackfillStatus() + return res, nil +} diff --git a/cmd/server/node_advert_routes_cache_test.go b/cmd/server/node_advert_routes_cache_test.go new file mode 100644 index 000000000..a4d15148c --- /dev/null +++ b/cmd/server/node_advert_routes_cache_test.go @@ -0,0 +1,141 @@ +package main + +import ( + "fmt" + "sync" + "sync/atomic" + "testing" + "time" +) + +// The #2073 advert route breakdown is cached per pubkey: bounded in size, +// expired by TTL, and invalidated for that pubkey alone when the node has a +// newer transmission (debounced, so a spamming node recomputes at most once +// per nodeAdvertRouteDebounce). + +func narCacheServer(t *testing.T) *Server { + t.Helper() + db := narDB(t) + return NewServer(db, &Config{}, NewHub()) +} + +func TestNodeAdvertRouteCache_HitAndTTL(t *testing.T) { + s := narCacheServer(t) + id := narInsert(t, s.db, narNode, "c-a", payloadTypeAdvert, 1, 0b0010, narAgo(time.Hour), true) + t0 := time.Now() + res, err := s.nodeAdvertRoutes(narNode, t0) + br := res.byRoute + if err != nil || len(br.Flood) != 1 { + t.Fatalf("first read: flood=%d err=%v", len(br.Flood), err) + } + // A route_mask update does not change the node's latest id: served from + // the cache until the TTL expires. + if _, err := s.db.conn.Exec(`UPDATE transmissions SET route_mask = 6 WHERE id = ?`, id); err != nil { + t.Fatal(err) + } + res, _ = s.nodeAdvertRoutes(narNode, t0.Add(nodeAdvertRouteTTL-time.Second)) + br = res.byRoute + if len(br.Flood) != 1 || len(br.Mixed) != 0 { + t.Fatalf("within TTL want cached flood=1 mixed=0, got flood=%d mixed=%d", len(br.Flood), len(br.Mixed)) + } + res, _ = s.nodeAdvertRoutes(narNode, t0.Add(nodeAdvertRouteTTL+time.Second)) + br, counts := res.byRoute, res.counts + if len(br.Flood) != 0 || len(br.Mixed) != 1 { + t.Fatalf("after TTL want recomputed flood=0 mixed=1, got flood=%d mixed=%d", len(br.Flood), len(br.Mixed)) + } + if counts.RouteMaskBackfill.Status == "" { + t.Fatal("route_mask_backfill must be set on every read") + } +} + +func TestNodeAdvertRouteCache_TargetedInvalidation(t *testing.T) { + s := narCacheServer(t) + const other = "2073bb0011223344556677889900aabbccddeeff00112233445566778899aabb" + narInsert(t, s.db, narNode, "t-a", payloadTypeAdvert, 1, 0b0010, narAgo(time.Hour), true) + narInsert(t, s.db, other, "t-b", payloadTypeAdvert, 1, 0b0010, narAgo(time.Hour), true) + t0 := time.Now() + s.nodeAdvertRoutes(narNode, t0) + s.nodeAdvertRoutes(other, t0) + + narInsert(t, s.db, narNode, "t-a2", payloadTypeAdvert, 2, 0b0100, narAgo(time.Minute), true) + // Inside the debounce window a newer advert does not force a rescan yet. + res, _ := s.nodeAdvertRoutes(narNode, t0.Add(nodeAdvertRouteDebounce/2)) + br := res.byRoute + if len(br.ZeroHop) != 0 { + t.Fatalf("within debounce want cached result, got zero_hop=%d", len(br.ZeroHop)) + } + // Past it, the node's newer transmission invalidates its entry only. + if _, err := s.db.conn.Exec(`UPDATE transmissions SET route_mask = 6 WHERE hash = 't-b'`); err != nil { + t.Fatal(err) + } + later := t0.Add(nodeAdvertRouteDebounce + time.Second) + res, _ = s.nodeAdvertRoutes(narNode, later) + br, counts := res.byRoute, res.counts + if len(br.ZeroHop) != 1 || counts.H24.ZeroHop != 1 { + t.Fatalf("after invalidation want zero_hop list=1 count=1, got %d/%d", len(br.ZeroHop), counts.H24.ZeroHop) + } + res, _ = s.nodeAdvertRoutes(other, later) + br = res.byRoute + if len(br.Flood) != 1 || len(br.Mixed) != 0 { + t.Fatalf("other node must stay cached (flood=1 mixed=0), got flood=%d mixed=%d", len(br.Flood), len(br.Mixed)) + } +} + +func TestNodeAdvertRouteCache_SizeBound(t *testing.T) { + var c nodeAdvertRouteCache + t0 := time.Now() + for i := 0; i < nodeAdvertRouteCacheMax+10; i++ { + at := t0.Add(time.Duration(i) * time.Millisecond) + c.put(fmt.Sprintf("pk%03d", i), nodeAdvertRouteEntry{latestID: 1, at: at}, at) + } + if n := c.len(); n != nodeAdvertRouteCacheMax { + t.Fatalf("cache holds %d entries, want the bound %d", n, nodeAdvertRouteCacheMax) + } + if _, ok := c.get("pk000", 1, t0); ok { + t.Fatal("oldest entry should have been evicted") + } + if _, ok := c.get(fmt.Sprintf("pk%03d", nodeAdvertRouteCacheMax+9), 1, t0.Add(time.Second)); !ok { + t.Fatal("newest entry should be cached") + } +} + +// put drops expired entries, so a cache that filled up once does not keep +// holding stale breakdowns until the same pubkeys are read again. +func TestNodeAdvertRouteCache_PutSweepsExpired(t *testing.T) { + var c nodeAdvertRouteCache + t0 := time.Now() + for i := 0; i < 10; i++ { + c.put(fmt.Sprintf("old%02d", i), nodeAdvertRouteEntry{latestID: 1, at: t0}, t0) + } + later := t0.Add(nodeAdvertRouteTTL + time.Second) + c.put("fresh", nodeAdvertRouteEntry{latestID: 1, at: later}, later) + if n := c.len(); n != 1 { + t.Fatalf("cache holds %d entries after the others expired, want 1", n) + } +} + +// Concurrent misses for one node share a single scan. +func TestNodeAdvertRouteCache_ConcurrentMissesShareOneScan(t *testing.T) { + s := narCacheServer(t) + narInsert(t, s.db, narNode, "sf-a", payloadTypeAdvert, 1, 0b0010, narAgo(time.Hour), true) + var scans atomic.Int32 + release := make(chan struct{}) + s.advertRoutes.onScan = func() { scans.Add(1); <-release } + var wg sync.WaitGroup + now := time.Now() + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if res, err := s.nodeAdvertRoutes(narNode, now); err != nil || len(res.byRoute.Flood) != 1 { + t.Errorf("flood=%d err=%v", len(res.byRoute.Flood), err) + } + }() + } + time.Sleep(100 * time.Millisecond) // let the goroutines join the flight + close(release) + wg.Wait() + if n := scans.Load(); n != 1 { + t.Fatalf("%d scans for 8 concurrent misses, want 1", n) + } +} diff --git a/cmd/server/node_advert_routes_optin_test.go b/cmd/server/node_advert_routes_optin_test.go new file mode 100644 index 000000000..dfb8e2c83 --- /dev/null +++ b/cmd/server/node_advert_routes_optin_test.go @@ -0,0 +1,227 @@ +package main + +import ( + "context" + "encoding/json" + "net/http/httptest" + "reflect" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// The #2073 advert route breakdown is opt-in (PR #97 review P2-2): only the +// node page asks for it with include=advertRoutes. Every other caller of +// GET /api/nodes/{pubkey} (packets, live, channels, route view, claimed +// nodes) gets master's response and costs master's queries. + +const narIncludeQuery = "?include=advertRoutes" + +// narCountRouteWork counts cache lookups and scans of the breakdown. +func narCountRouteWork(s *Server) (lookups, scans *atomic.Int32) { + lookups, scans = new(atomic.Int32), new(atomic.Int32) + s.advertRoutes.onLookup = func() { lookups.Add(1) } + s.advertRoutes.onScan = func() { scans.Add(1) } + return lookups, scans +} + +func narTopLevelKeys(t *testing.T, raw string) []string { + t.Helper() + var top map[string]json.RawMessage + if err := json.Unmarshal([]byte(raw), &top); err != nil { + t.Fatalf("bad JSON: %v", err) + } + keys := make([]string, 0, len(top)) + for k := range top { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// Without the opt-in the response is master's: the same two top-level +// fields, no route_class on recentAdverts, and neither the breakdown's cache +// nor its scan is touched. Near misses of the parameter do not opt in. +func TestNodeDetail_WithoutIncludeIsMasterShaped(t *testing.T) { + srv, router := narServer(t) + lookups, scans := narCountRouteWork(srv) + for _, q := range []string{"", "?include=", "?include=other", "?include=advertroutes", "?include=advertRoutesX", "?advertRoutes=1", "?include_advertRoutes=1"} { + body, raw := narGetNode(t, router, narNode+q, 200) + if got := narTopLevelKeys(t, raw); !reflect.DeepEqual(got, []string{"node", "recentAdverts"}) { + t.Fatalf("%q: top-level keys = %v, want master's [node recentAdverts]", q, got) + } + if strings.Contains(raw, "route_class") || strings.Contains(raw, "advertCounts") || strings.Contains(raw, "recentAdvertsByRoute") { + t.Fatalf("%q: breakdown fields leaked into the default response: %s", q, raw) + } + if len(body.RecentAdverts) != 5 { + t.Fatalf("%q: recentAdverts = %d rows, want 5", q, len(body.RecentAdverts)) + } + if _, ok := body.RecentAdverts[0]["observations"]; !ok { + t.Fatalf("%q: recentAdverts rows keep their observations", q) + } + if got := body.Node["flood_advert_count_7d"]; got != float64(2) { + t.Fatalf("%q: flood_advert_count_7d = %v, want 2", q, got) + } + } + if l, s := lookups.Load(), scans.Load(); l != 0 || s != 0 { + t.Fatalf("default requests touched the breakdown: %d cache lookups, %d scans", l, s) + } + if n := srv.advertRoutes.len(); n != 0 { + t.Fatalf("default requests filled the breakdown cache: %d entries", n) + } +} + +// include takes a comma-separated list and may repeat; advertRoutes in any +// position opts in. +func TestNodeDetail_IncludeAdvertRoutesListForms(t *testing.T) { + srv, router := narServer(t) + lookups, _ := narCountRouteWork(srv) + for _, q := range []string{narIncludeQuery, "?include=other,advertRoutes", "?include=other&include=advertRoutes", "?include=%20advertRoutes"} { + body, raw := narGetNode(t, router, narNode+q, 200) + if body.ByRoute == nil || body.Counts == nil || !strings.Contains(raw, "route_class") { + t.Fatalf("%q did not opt in: %s", q, raw) + } + } + if lookups.Load() != 4 { + t.Fatalf("%d cache lookups for 4 opted-in requests", lookups.Load()) + } +} + +// The privacy gates run before the cache: a hidden identity never reaches a +// lookup or a scan, even when it asks for the breakdown. +func TestNodeDetail_IncludeAdvertRoutesPrivacyBeforeCache(t *testing.T) { + srv, router := narServer(t) + lookups, scans := narCountRouteWork(srv) + srv.cfg.ObserverBlacklist = []string{strings.ToUpper(narNode)} + _, raw := narGetNode(t, router, narNode+narIncludeQuery, 200) + if strings.Contains(raw, "route_class") || strings.Contains(raw, "advertCounts") { + t.Fatalf("hidden identity leaked: %s", raw) + } + if l, s := lookups.Load(), scans.Load(); l != 0 || s != 0 { + t.Fatalf("hidden identity reached the breakdown: %d lookups, %d scans", l, s) + } +} + +// N1 (re-review of 658d8a08): a failed identity-hidden lookup must fail +// closed, not merely agree with a hidden-by-config identity by coincidence. +// The mutant `advertRoutes = !hidden` (dropping the err check) survives the +// PrivacyBeforeCache test above because isIdentityHidden short-circuits to +// (true, nil) for a blacklisted/observer-blacklisted identity without ever +// running the name lookup that can fail — so `!hidden` and `err == nil && +// !hidden` agree there. They only diverge when the lookup itself errors: +// isIdentityHidden then returns (false, err), so the mutant's bare !hidden +// is true (fail OPEN) while the real gate is false. Forcing that error +// needs a non-empty hidden-name-prefix list (otherwise hiddenIdentityNames +// short-circuits to (nil, nil) without touching the DB) plus a request +// whose context is already canceled once the lookup's QueryContext / +// QueryRowContext calls run. The configured prefix must not match this +// node's own name, so the request reaches this code instead of 404-ing on +// the earlier, unrelated IsNameHidden(node name) gate. +func TestNodeDetail_IncludeAdvertRoutesFailsClosedOnLookupError(t *testing.T) { + srv, router := narServer(t) + lookups, scans := narCountRouteWork(srv) + srv.cfg.SetHiddenNamePrefixes([]string{"ZZ-does-not-match-any-node-or-observer"}) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + req := httptest.NewRequest("GET", "/api/nodes/"+narNode+narIncludeQuery, nil).WithContext(ctx) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != 200 { + t.Fatalf("GET with a canceled context = %d, want 200: %s", w.Code, w.Body.String()) + } + raw := w.Body.String() + if strings.Contains(raw, "route_class") || strings.Contains(raw, "advertCounts") || strings.Contains(raw, "recentAdvertsByRoute") { + t.Fatalf("a failed privacy lookup must fail closed, leaked breakdown fields: %s", raw) + } + var body narResponse + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("bad JSON: %v", err) + } + if body.ByRoute != nil || body.Counts != nil { + t.Fatalf("a failed lookup must not populate the breakdown: byRoute=%v counts=%v", body.ByRoute, body.Counts) + } + if len(body.RecentAdverts) != 5 { + t.Fatalf("recentAdverts = %d rows, want master's 5", len(body.RecentAdverts)) + } + if _, ok := body.RecentAdverts[0]["route_class"]; ok { + t.Fatalf("recentAdverts must not carry route_class when the lookup failed: %v", body.RecentAdverts[0]) + } + if l, s := lookups.Load(), scans.Load(); l != 0 || s != 0 { + t.Fatalf("a failed lookup must never reach the breakdown cache or scan: %d lookups, %d scans", l, s) + } +} + +// flood_advert_count_7d is an external contract and never cached: with the +// breakdown served from the cache it is still counted fresh, and when this +// request's own scan produced it, it equals CountFloodAdvertsForNode. +func TestNodeDetail_FloodAdvertCount7dStaysFresh(t *testing.T) { + srv, router := narServer(t) + _, scans := narCountRouteWork(srv) + body, _ := narGetNode(t, router, narNode+narIncludeQuery, 200) + if got := body.Node["flood_advert_count_7d"]; got != float64(2) { + t.Fatalf("cold flood_advert_count_7d = %v, want 2", got) + } + // A new route_type 1 advert inside the debounce: the breakdown stays + // cached, the flood count must not. + narInsert(t, srv.db, narNode, "h-new-flood", payloadTypeAdvert, 1, 0b0010, narAgo(time.Minute), true) + body, _ = narGetNode(t, router, narNode+narIncludeQuery, 200) + if scans.Load() != 1 { + t.Fatalf("%d scans, want the second request served from the cache", scans.Load()) + } + if len(body.ByRoute.Flood) != 3 { + t.Fatalf("cached flood list = %d rows, want the cached 3", len(body.ByRoute.Flood)) + } + if got := body.Node["flood_advert_count_7d"]; got != float64(3) { + t.Fatalf("flood_advert_count_7d = %v next to a cached breakdown, want the fresh 3", got) + } + want, err := srv.db.CountFloodAdvertsForNode(narNode, 7*24, floodAdvertRowCap) + if err != nil || want != 3 { + t.Fatalf("CountFloodAdvertsForNode = %d, %v", want, err) + } +} + +// The scan's flood count is handed only to the request that ran the scan: +// a request that shared another's scan (singleflight) or hit the cache counts +// its own, so the value is never older than the request. +func TestNodeAdvertRoutes_FloodCountOnlyFromOwnScan(t *testing.T) { + s := narCacheServer(t) + narInsert(t, s.db, narNode, "own-a", payloadTypeAdvert, 1, 0b0010, narAgo(time.Hour), true) + release := make(chan struct{}) + var scans atomic.Int32 + s.advertRoutes.onScan = func() { scans.Add(1); <-release } + var withFlood atomic.Int32 + var wg sync.WaitGroup + now := time.Now() + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + res, err := s.nodeAdvertRoutes(narNode, now) + if err != nil { + t.Error(err) + return + } + if res.floodAdvertCount7d != nil { + withFlood.Add(1) + if *res.floodAdvertCount7d != 1 { + t.Errorf("scan flood count = %d, want 1", *res.floodAdvertCount7d) + } + } + }() + } + time.Sleep(100 * time.Millisecond) // let the goroutines join the flight + close(release) + wg.Wait() + if scans.Load() != 1 || withFlood.Load() != 1 { + t.Fatalf("scans=%d, results with a flood count=%d; want 1 and 1 (the scanning request only)", scans.Load(), withFlood.Load()) + } + res, err := s.nodeAdvertRoutes(narNode, now) + if err != nil || res.floodAdvertCount7d != nil { + t.Fatalf("cache hit must not carry a flood count: %v, %v", res.floodAdvertCount7d, err) + } +} diff --git a/cmd/server/node_advert_routes_stream_test.go b/cmd/server/node_advert_routes_stream_test.go new file mode 100644 index 000000000..07ab1bb21 --- /dev/null +++ b/cmd/server/node_advert_routes_stream_test.go @@ -0,0 +1,465 @@ +package main + +import ( + "fmt" + "math/rand" + "reflect" + "strings" + "testing" + "time" + + "github.com/meshcore-analyzer/packetpath" +) + +// PR #97 review P2-1: GetNodeAdvertRoutes is one index-ordered pass over the +// node's ADVERT rows, classified and limited per class in Go. The window +// query it replaced (ROW_NUMBER() per class in SQL) stays here as a +// test-only oracle, and so does its SQL classifier; the stream must agree +// with both on randomized data, and its flood_advert_count_7d must agree bit +// for bit with CountFloodAdvertsForNode. + +// advertRouteClassSQL is classifyAdvertRoute as a SQL expression over the +// unaliased transmissions columns, yielding the route class name. Without +// the route_mask column every row takes the route_type fallback. +// TestAdvertRouteClassSQL_MatchesGoClassifier pins it to the Go rule. +func advertRouteClassSQL(hasRouteMask bool) string { + fallback := fmt.Sprintf(`WHEN route_type IN (%d, %d) THEN '%s' WHEN route_type IN (%d, %d) THEN '%s' ELSE '%s'`, + RouteTransportFlood, RouteFlood, advertClassFlood, + RouteDirect, RouteTransportDirect, advertClassZeroHop, advertClassUnknown) + if !hasRouteMask { + return "(CASE " + fallback + " END)" + } + flood, direct := packetpath.RouteMaskFlood, packetpath.RouteMaskDirect + return fmt.Sprintf(`(CASE WHEN route_mask & %d <> 0 THEN (CASE WHEN route_mask & %d <> 0 AND route_mask & %d <> 0 THEN '%s' WHEN route_mask & %d <> 0 THEN '%s' ELSE '%s' END) %s END)`, + packetpath.RouteMaskAll, flood, direct, advertClassMixed, flood, advertClassFlood, advertClassZeroHop, fallback) +} + +// oracleNodeAdvertRoutesWindow is the replaced implementation: the class +// computed in SQL and ROW_NUMBER() per class, then the same lean-row fetch. +// oracleScanWindow is its query and count loop, verbatim but for returning +// the listed ids instead of fetching them. +func oracleNodeAdvertRoutesWindow(db *DB, pubkey string, perClass int, now time.Time, rowCap int) (NodeAdvertsByRoute, NodeAdvertCounts, error) { + byRoute := NodeAdvertsByRoute{Limit: perClass, Flood: NodeAdvertRows{}, ZeroHop: NodeAdvertRows{}, Mixed: NodeAdvertRows{}} + sc, err := oracleScanWindow(db, pubkey, perClass, now, rowCap) + if err != nil || len(sc.ids) == 0 { + return byRoute, sc.counts, err + } + listed := map[int]string{} + var ids []interface{} + for i, id := range sc.ids { + listed[id] = sc.classes[i] + ids = append(ids, id) + } + full, err := db.queryNodeAdvertRows(false, false, "t.id IN ("+strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")+") ORDER BY t.id DESC", ids...) + if err != nil { + return byRoute, sc.counts, err + } + for _, p := range full { + id, _ := p["id"].(int) + p["route_class"] = listed[id] + switch listed[id] { + case advertClassFlood: + byRoute.Flood = append(byRoute.Flood, p) + case advertClassZeroHop: + byRoute.ZeroHop = append(byRoute.ZeroHop, p) + case advertClassMixed: + byRoute.Mixed = append(byRoute.Mixed, p) + default: + byRoute.Unknown = append(byRoute.Unknown, p) + } + } + return byRoute, sc.counts, nil +} + +func oracleScanWindow(db *DB, pubkey string, perClass int, now time.Time, rowCap int) (nodeAdvertScan, error) { + var sc nodeAdvertScan + floor := advertDateFloor(now, 7*24) + class := advertRouteClassSQL(db.hasRouteMask()) + rows, err := db.conn.Query(`SELECT id, cls, rn, first_seen, hash FROM ( + SELECT id, cls, COALESCE(first_seen, '') AS first_seen, COALESCE(hash, '') AS hash, + ROW_NUMBER() OVER (PARTITION BY cls ORDER BY id DESC) AS rn + FROM (SELECT id, first_seen, hash, `+class+` AS cls FROM transmissions WHERE from_pubkey = ? AND payload_type = ?) + ) WHERE rn <= ? OR first_seen >= ? + ORDER BY id DESC`, pubkey, payloadTypeAdvert, perClass, floor) + if err != nil { + return sc, err + } + defer rows.Close() + var entries []advertRouteEntry + for rows.Next() { + var id, rn int + var cls, ts, hash string + if err := rows.Scan(&id, &cls, &rn, &ts, &hash); err != nil { + return sc, err + } + if rn <= perClass { + sc.ids = append(sc.ids, id) + sc.classes = append(sc.classes, cls) + } + if ts >= floor { + if len(entries) < rowCap { + entries = append(entries, advertRouteEntry{ts: ts, hash: hash, class: cls}) + } else { + sc.counts.Truncated = true + } + } + } + if err := rows.Err(); err != nil { + return sc, err + } + sc.counts.H24 = countAdvertRoutes(entries, now, 24) + sc.counts.D7 = countAdvertRoutes(entries, now, 7*24) + return sc, nil +} + +// narRelaxTransmissions rebuilds transmissions without NOT NULL / UNIQUE, so +// the randomized data can hold what the parity must also survive: duplicate +// and NULL hashes (the dedup paths) and NULL first_seen. It leaves out +// idx_transmissions_payload_type, which would put the verbatim oracle query +// on a scan of every ADVERT (TestNodeAdvertRoutes_StreamQueryPlan covers the +// stream with that index present). +func narRelaxTransmissions(t *testing.T, db *DB) { + t.Helper() + rows, err := db.conn.Query(`SELECT name, type FROM pragma_table_info('transmissions') ORDER BY cid`) + if err != nil { + t.Fatal(err) + } + var cols []string + for rows.Next() { + var name, typ string + if err := rows.Scan(&name, &typ); err != nil { + t.Fatal(err) + } + if name == "id" { + cols = append(cols, "id INTEGER PRIMARY KEY AUTOINCREMENT") + } else { + cols = append(cols, name+" "+typ) + } + } + rows.Close() + for _, q := range []string{ + `DROP TABLE transmissions`, + `CREATE TABLE transmissions (` + strings.Join(cols, ", ") + `)`, + `CREATE INDEX idx_transmissions_from_pubkey ON transmissions(from_pubkey)`, + `CREATE INDEX idx_transmissions_first_seen ON transmissions(first_seen)`, + } { + if _, err := db.conn.Exec(q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } +} + +type narParityNode struct { + pubkey string + perClass int + rowCap int +} + +type narParityVariant struct { + name string + withMask bool // the route_mask column exists + relaxed bool // duplicate/NULL hashes and NULL first_seen allowed + nodes int +} + +// 1,000 random nodes over three schemas: migrated with and without the +// relaxed constraints, and a legacy database without route_mask. +var narParityVariants = []narParityVariant{ + {"relaxed, route_mask", true, true, 500}, + {"relaxed, no route_mask column", false, true, 250}, + {"production schema, route_mask", true, false, 250}, +} + +// narParityTS returns a first_seen value around now in one of the formats +// seen in the wild (plus garbage, empty, date-only and window-edge values). +func narParityTS(rng *rand.Rand, now time.Time) interface{} { + age := time.Duration(rng.Int63n(int64(10*24*time.Hour))) - time.Hour + switch rng.Intn(12) { + case 0: + return now.Add(-24*time.Hour + time.Duration(rng.Intn(3)-1)*time.Second).Format(time.RFC3339) + case 1: + return now.Add(-7*24*time.Hour + time.Duration(rng.Intn(3)-1)*time.Second).Format(time.RFC3339) + case 2: + return advertDateFloor(now, 7*24) + []string{"", "T00:00:00Z", " 00:00:00", "T23:59:59.999Z"}[rng.Intn(4)] + case 3: + return now.Add(-age).Format(time.RFC3339Nano) + case 4: + return now.Add(-age).Format("2006-01-02T15:04:05.000Z") + case 5: + return now.Add(-age).In(time.FixedZone("CEST", 2*3600)).Format(time.RFC3339) + case 6: + return now.Add(-age).Format("2006-01-02 15:04:05") + case 7: + return []interface{}{"garbage", "", "zzzz", nil}[rng.Intn(4)] + default: + return now.Add(-age).Format(time.RFC3339) + } +} + +// narSeedParity seeds v.nodes random nodes (ids interleaved across nodes) +// and returns them with a per-class limit and row cap each. +func narSeedParity(t *testing.T, db *DB, rng *rand.Rand, v narParityVariant, now time.Time) []narParityNode { + t.Helper() + routeTypes := []interface{}{nil, 0, 1, 1, 1, 2, 2, 3, 4, 99, -1, 1.5, "x", "1"} + masks := []interface{}{nil, nil, 0, -1, 16, "6", "abc"} + for m := 1; m < 32; m++ { + masks = append(masks, m) + } + type row struct { + pubkey, hash interface{} + firstSeen interface{} + routeType interface{} + payloadType interface{} + mask interface{} + } + var all []row + nodes := make([]narParityNode, v.nodes) + serial := 0 + for n := range nodes { + pk := fmt.Sprintf("%064x", rng.Uint64()) + fmt.Sprintf("-%s-%d", v.name, n) + nodes[n] = narParityNode{pubkey: pk, perClass: nodeAdvertRouteLimit, rowCap: floodAdvertRowCap} + if rng.Intn(4) == 0 { + nodes[n].perClass = 1 + rng.Intn(5) + } + if rng.Intn(2) == 0 { + nodes[n].rowCap = []int{1, 2, 3, 7, 15, 40}[rng.Intn(6)] + } + count := rng.Intn(70) + switch rng.Intn(10) { + case 0: + count = rng.Intn(4) + case 1: + count = 150 + rng.Intn(50) + } + dupPool := 1 + rng.Intn(6) + for i := 0; i < count; i++ { + serial++ + r := row{pubkey: pk, firstSeen: narParityTS(rng, now), routeType: routeTypes[rng.Intn(len(routeTypes))], payloadType: payloadTypeAdvert} + if v.withMask { + r.mask = masks[rng.Intn(len(masks))] + } + if rng.Intn(8) == 0 { + r.payloadType = []interface{}{5, 2, nil}[rng.Intn(3)] + } + r.hash = fmt.Sprintf("u%d", serial) + if v.relaxed { + switch rng.Intn(6) { + case 0: + r.hash = fmt.Sprintf("%s-dup%d", pk, rng.Intn(dupPool)) + case 1: + r.hash = []interface{}{"", nil}[rng.Intn(2)] + } + } else if r.firstSeen == nil { + r.firstSeen = "garbage" + } + all = append(all, r) + } + } + rng.Shuffle(len(all), func(i, j int) { all[i], all[j] = all[j], all[i] }) + tx, err := db.conn.Begin() + if err != nil { + t.Fatal(err) + } + for _, r := range all { + if v.withMask { + _, err = tx.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, from_pubkey, route_mask) + VALUES ('1100', ?, ?, ?, ?, '{"type":"ADVERT"}', ?, ?)`, r.hash, r.firstSeen, r.routeType, r.payloadType, r.pubkey, r.mask) + } else { + _, err = tx.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, from_pubkey) + VALUES ('1100', ?, ?, ?, ?, '{"type":"ADVERT"}', ?)`, r.hash, r.firstSeen, r.routeType, r.payloadType, r.pubkey) + } + if err != nil { + tx.Rollback() + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + return nodes +} + +func narParityDB(t *testing.T, v narParityVariant) *DB { + t.Helper() + db := setupTestDB(t) + if v.relaxed { + narRelaxTransmissions(t, db) + } + if v.withMask { + narAddRouteMask(t, db) + } + return db +} + +// The stream returns exactly what the window query returned - the same +// listed ids, order and classes and the same counts and truncation - for +// 1,000 random nodes with mixed, NULL, out-of-range and mistyped masks and +// route types, every first_seen format, duplicate and missing hashes, +// non-advert rows, small per-class limits and row caps. The scans are +// compared for every node; the full responses (same lean-row fetch) for +// every 25th. +func TestNodeAdvertRoutes_StreamMatchesWindowOracle(t *testing.T) { + now := time.Now().UTC() + var total, full, truncated, limited, withUnknown, withMixed int + for vi, v := range narParityVariants { + db := narParityDB(t, v) + rng := rand.New(rand.NewSource(int64(2073 + vi))) + for i, n := range narSeedParity(t, db, rng, v, now) { + want, err := oracleScanWindow(db, n.pubkey, n.perClass, now, n.rowCap) + if err != nil { + t.Fatalf("%s: oracle: %v", v.name, err) + } + got, err := db.scanNodeAdvertRoutes(n.pubkey, n.perClass, now, n.rowCap) + if err != nil { + t.Fatalf("%s: stream: %v", v.name, err) + } + if !reflect.DeepEqual(got.ids, want.ids) || !reflect.DeepEqual(got.classes, want.classes) { + t.Fatalf("%s node %s (perClass=%d): lists differ\nstream %v %v\noracle %v %v", v.name, n.pubkey, n.perClass, got.ids, got.classes, want.ids, want.classes) + } + if got.counts != want.counts { + t.Fatalf("%s node %s (rowCap=%d): counts differ\nstream %+v\noracle %+v", v.name, n.pubkey, n.rowCap, got.counts, want.counts) + } + if i%25 == 0 { + wantBR, wantCounts, err := oracleNodeAdvertRoutesWindow(db, n.pubkey, n.perClass, now, n.rowCap) + if err != nil { + t.Fatal(err) + } + gotBR, gotCounts, _, err := db.GetNodeAdvertRoutes(n.pubkey, n.perClass, now, n.rowCap) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(gotBR, wantBR) || gotCounts != wantCounts { + t.Fatalf("%s node %s: responses differ\nstream flood=%v zero_hop=%v mixed=%v unknown=%v\noracle flood=%v zero_hop=%v mixed=%v unknown=%v", + v.name, n.pubkey, narHashes(gotBR.Flood), narHashes(gotBR.ZeroHop), narHashes(gotBR.Mixed), narHashes(gotBR.Unknown), + narHashes(wantBR.Flood), narHashes(wantBR.ZeroHop), narHashes(wantBR.Mixed), narHashes(wantBR.Unknown)) + } + full++ + } + total++ + perClass := map[string]int{} + for _, c := range want.classes { + perClass[c]++ + } + if want.counts.Truncated { + truncated++ + } + if perClass[advertClassZeroHop] == n.perClass || perClass[advertClassFlood] == n.perClass { + limited++ + } + if perClass[advertClassUnknown] > 0 { + withUnknown++ + } + if perClass[advertClassMixed] > 0 { + withMixed++ + } + } + } + t.Logf("compared %d nodes (%d full responses): %d truncated, %d at a per-class limit, %d with unknown, %d with mixed", total, full, truncated, limited, withUnknown, withMixed) + if total < 1000 || full < 40 || truncated == 0 || limited == 0 || withUnknown == 0 || withMixed == 0 { + t.Fatalf("the random data must exercise every path (nodes=%d full=%d truncated=%d limited=%d unknown=%d mixed=%d)", total, full, truncated, limited, withUnknown, withMixed) + } +} + +// The stream's flood count is CountFloodAdvertsForNode(pubkey, 7d, rowCap) +// bit for bit at the same instant: route_type = 1 and the first_seen floor +// evaluated by SQL as there, the newest rowCap flood rows inside the floor, +// the same dedup and parseRelayTS window. +func TestNodeAdvertRoutes_FloodCountMatchesCountFloodAdvertsForNode(t *testing.T) { + now := time.Now().UTC() + floor := advertDateFloor(now, 7*24) + var total, nonZero, capped, skipped, duplicated int + for vi, v := range narParityVariants { + db := narParityDB(t, v) + rng := rand.New(rand.NewSource(int64(4073 + vi))) + nodes := narSeedParity(t, db, rng, v, now) + // Per node: route_type 1 rows inside the floor, and hash groups + // (NULL and empty together) holding more than one of them. + type stat struct{ rows, dupGroups int } + stats := map[string]stat{} + rows, err := db.conn.Query(`SELECT from_pubkey, SUM(n), SUM(n > 1) FROM ( + SELECT from_pubkey, COUNT(*) AS n FROM transmissions + WHERE payload_type = ? AND route_type = 1 AND first_seen >= ? + GROUP BY from_pubkey, COALESCE(hash, '')) GROUP BY from_pubkey`, payloadTypeAdvert, floor) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var pk string + var st stat + if err := rows.Scan(&pk, &st.rows, &st.dupGroups); err != nil { + t.Fatal(err) + } + stats[pk] = st + } + rows.Close() + for _, n := range nodes { + want, err := db.countFloodAdvertsForNodeAt(n.pubkey, 7*24, n.rowCap, now) + if err != nil { + t.Fatalf("%s: CountFloodAdvertsForNode: %v", v.name, err) + } + got, err := db.scanNodeAdvertRoutes(n.pubkey, n.perClass, now, n.rowCap) + if err != nil { + t.Fatalf("%s: stream: %v", v.name, err) + } + if got.flood7d != want { + t.Fatalf("%s node %s (rowCap=%d): flood_advert_count_7d stream=%d CountFloodAdvertsForNode=%d", v.name, n.pubkey, n.rowCap, got.flood7d, want) + } + total++ + st := stats[n.pubkey] + if want > 0 { + nonZero++ + } + if st.rows > n.rowCap { + capped++ // the row cap binds + } else if st.rows > want { + skipped++ // rows dropped by the exact window, parseRelayTS or dedup + } + if st.dupGroups > 0 { + duplicated++ // same (or missing) hash on several flood rows: the dedup path + } + } + } + t.Logf("compared %d nodes: %d with flood adverts, %d where the row cap binds, %d with window/parse/dedup skips, %d with duplicate hashes", total, nonZero, capped, skipped, duplicated) + if total < 1000 || nonZero == 0 || capped == 0 || skipped == 0 || duplicated == 0 { + t.Fatalf("the random data must exercise every path (nodes=%d nonzero=%d capped=%d skipped=%d duplicated=%d)", total, nonZero, capped, skipped, duplicated) + } +} + +// The stream reads the node's rows in index order: from_pubkey index, newest +// id first, no temp B-tree sort and no window - also without ANALYZE +// statistics and with idx_transmissions_payload_type created after the +// from_pubkey index, where SQLite would otherwise walk every ADVERT via the +// payload_type index (the unary + in nodeAdvertScanSQL). +func TestNodeAdvertRoutes_StreamQueryPlan(t *testing.T) { + for _, withMask := range []bool{true, false} { + db := setupTestDB(t) + if withMask { + narAddRouteMask(t, db) + } + if _, err := db.conn.Exec(`CREATE INDEX idx_transmissions_payload_type ON transmissions(payload_type)`); err != nil { + t.Fatal(err) + } + floor := advertDateFloor(time.Now(), 7*24) + rows, err := db.conn.Query(`EXPLAIN QUERY PLAN `+nodeAdvertScanSQL(withMask), floor, advertRouteTypeFlood, narNode, payloadTypeAdvert) + if err != nil { + t.Fatal(err) + } + var plan []string + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + t.Fatal(err) + } + plan = append(plan, detail) + } + rows.Close() + joined := strings.Join(plan, " | ") + if !strings.Contains(joined, "idx_transmissions_from_pubkey") || strings.Contains(joined, "TEMP B-TREE") { + t.Fatalf("withMask=%v: plan = %s, want an index-ordered from_pubkey scan", withMask, joined) + } + if strings.Contains(strings.ToUpper(nodeAdvertScanSQL(withMask)), "OVER") { + t.Fatal("the stream must not use a window function") + } + } +} diff --git a/cmd/server/node_advert_routes_test.go b/cmd/server/node_advert_routes_test.go new file mode 100644 index 000000000..095d63d33 --- /dev/null +++ b/cmd/server/node_advert_routes_test.go @@ -0,0 +1,587 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" +) + +// Node-detail advert route breakdown (port/extension of upstream +// `Kpa-clawbot/CoreScope#2073`): recentAdvertsByRoute lists the newest +// adverts per route class (class filtered in SQL) and advertCounts counts +// distinct adverts per class over 24h and 7d, both classified exactly like +// Relay Airtime Share (#89). + +const narNode = "2073aa0011223344556677889900aabbccddeeff00112233445566778899aabb" + +// narDB is a test DB with the route_mask column, like a migrated database. +func narDB(t *testing.T) *DB { + t.Helper() + db := setupTestDB(t) + narAddRouteMask(t, db) + return db +} + +func narAddRouteMask(t *testing.T, db *DB) { + t.Helper() + if _, err := db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN route_mask INTEGER`); err != nil { + t.Fatal(err) + } + db.hasRouteMaskFlag.forceTrue() +} + +// narInsert seeds one transmission from pubkey. mask and routeType may be +// nil (NULL); withMask=false omits the route_mask column (legacy schema). +func narInsert(t *testing.T, db *DB, pubkey, hash string, payloadType int, routeType, mask interface{}, firstSeen string, withMask bool) int64 { + t.Helper() + var res sql.Result + var err error + if withMask { + res, err = db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, from_pubkey, route_mask) + VALUES ('1100', ?, ?, ?, ?, '{"type":"ADVERT"}', ?, ?)`, hash, firstSeen, routeType, payloadType, pubkey, mask) + } else { + res, err = db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, from_pubkey) + VALUES ('1100', ?, ?, ?, ?, '{"type":"ADVERT"}', ?)`, hash, firstSeen, routeType, payloadType, pubkey) + } + if err != nil { + t.Fatalf("insert %s: %v", hash, err) + } + id, _ := res.LastInsertId() + return id +} + +func narAgo(d time.Duration) string { + return time.Now().UTC().Add(-d).Format(time.RFC3339) +} + +func narHashes(rows NodeAdvertRows) []string { + out := make([]string, 0, len(rows)) + for _, r := range rows { + h, _ := r["hash"].(string) + out = append(out, h) + } + return out +} + +func narRouteTypes() []*int { + out := []*int{nil} + for _, v := range []int{-1, 0, 1, 2, 3, 4, 99} { + v := v + out = append(out, &v) + } + return out +} + +// classifyAdvertRoute is the rule behind advertRouteClass: Relay Airtime +// Share must classify every (mask, known, route_type) exactly as before. +func TestClassifyAdvertRoute_ParityWithRelayAirtimeShare(t *testing.T) { + for mask := 0; mask < 32; mask++ { + for _, known := range []bool{false, true} { + for _, rt := range narRouteTypes() { + tx := &StoreTx{RouteType: rt, routeMask: uint8(mask), routeMaskKnown: known} + if got, want := classifyAdvertRoute(int64(mask), known, rt), advertRouteClass(tx); got != want { + t.Fatalf("mask=%d known=%v rt=%v: classifyAdvertRoute=%v, advertRouteClass=%v", mask, known, rt, got, want) + } + } + } + } +} + +// The SQL CASE that filters the per-class lists must agree with the Go +// classifier for every mask (NULL, 0..31 including bits above route 3) and +// every route_type (NULL, invalid, 0..3), with and without the column. +func TestAdvertRouteClassSQL_MatchesGoClassifier(t *testing.T) { + db := narDB(t) + masks := []interface{}{nil} + for m := 0; m < 32; m++ { + masks = append(masks, m) + } + masks = append(masks, -1) + type want struct { + mask sql.NullInt64 + rt sql.NullInt64 + } + seeded := map[string]want{} + i := 0 + for _, m := range masks { + for _, rt := range narRouteTypes() { + var rtv interface{} + w := want{} + if rt != nil { + rtv = *rt + w.rt = sql.NullInt64{Int64: int64(*rt), Valid: true} + } + if m != nil { + w.mask = sql.NullInt64{Int64: int64(m.(int)), Valid: true} + } + hash := fmt.Sprintf("parity%04d", i) + i++ + narInsert(t, db, narNode, hash, payloadTypeAdvert, rtv, m, narAgo(time.Hour), true) + seeded[hash] = w + } + } + for _, withMask := range []bool{true, false} { + rows, err := db.conn.Query(`SELECT hash, ` + advertRouteClassSQL(withMask) + ` FROM transmissions WHERE hash LIKE 'parity%'`) + if err != nil { + t.Fatal(err) + } + n := 0 + for rows.Next() { + var hash, class string + if err := rows.Scan(&hash, &class); err != nil { + t.Fatal(err) + } + w := seeded[hash] + mask := w.mask + if !withMask { + mask = sql.NullInt64{} // column absent: always the legacy fallback + } + if goClass := advertRouteClassName(classifyAdvertRouteRow(mask, w.rt)); class != goClass { + t.Fatalf("withMask=%v %s (mask=%v rt=%v): SQL=%q Go=%q", withMask, hash, w.mask, w.rt, class, goClass) + } + n++ + } + rows.Close() + if n != len(seeded) { + t.Fatalf("withMask=%v: compared %d rows, seeded %d", withMask, n, len(seeded)) + } + } +} + +// Frequent zero-hop adverts must not push the rare flood adverts out: the +// class filter runs in SQL before the per-class limit. The chronological +// 20 hold no flood advert at all, so a client-side split would show none. +func TestNodeAdvertRoutes_PerClassLimitInSQL(t *testing.T) { + db := narDB(t) + narInsert(t, db, narNode, "flood-a", payloadTypeAdvert, 0, 0b0001, narAgo(6*time.Hour), true) + narInsert(t, db, narNode, "flood-b", payloadTypeAdvert, 1, 0b0010, narAgo(5*time.Hour), true) + for i := 0; i < 100; i++ { + narInsert(t, db, narNode, fmt.Sprintf("zh%03d", i), payloadTypeAdvert, 2, 0b0100, narAgo(time.Duration(100-i)*time.Minute), true) + } + byRoute, counts, _, err := db.GetNodeAdvertRoutes(narNode, nodeAdvertRouteLimit, time.Now(), floodAdvertRowCap) + if err != nil { + t.Fatal(err) + } + if got := narHashes(byRoute.Flood); !reflect.DeepEqual(got, []string{"flood-b", "flood-a"}) { + t.Fatalf("flood list = %v, want both flood adverts, newest ingest first", got) + } + if len(byRoute.ZeroHop) != nodeAdvertRouteLimit || narHashes(byRoute.ZeroHop)[0] != "zh099" { + t.Fatalf("zero_hop list = %v, want the newest %d", narHashes(byRoute.ZeroHop), nodeAdvertRouteLimit) + } + for _, r := range byRoute.Flood { + if r["route_class"] != advertClassFlood { + t.Fatalf("flood row route_class = %v", r["route_class"]) + } + // The per-class rows are lean: no observation arrays (cached, and + // repeated next to recentAdverts), but the best observation's fields. + if _, ok := r["observations"]; ok { + t.Fatal("per-class rows must not carry observations") + } + if _, ok := r["observation_count"]; !ok { + t.Fatal("per-class rows keep observation_count") + } + } + recent, err := db.GetRecentTransmissionsForNode(narNode, 20, true) + if err != nil { + t.Fatal(err) + } + for _, r := range recent { + if r["route_class"] == advertClassFlood { + t.Fatalf("chronological list unexpectedly holds a flood advert: %v", r["hash"]) + } + } + if counts.H24 != (AdvertRouteCounts{Flood: 2, ZeroHop: 100}) { + t.Fatalf("24h counts = %+v", counts.H24) + } +} + +// A hash seen on flood and zero-hop routes is one mixed advert: it is listed +// and counted only as mixed, never as flood or zero-hop. +func TestNodeAdvertRoutes_MixedOnlyInMixed(t *testing.T) { + db := narDB(t) + narInsert(t, db, narNode, "mixed1", payloadTypeAdvert, 1, 0b0110, narAgo(time.Hour), true) + narInsert(t, db, narNode, "flood1", payloadTypeAdvert, 1, 0b0010, narAgo(2*time.Hour), true) + narInsert(t, db, narNode, "zh1", payloadTypeAdvert, 2, 0b0100, narAgo(3*time.Hour), true) + byRoute, counts, _, err := db.GetNodeAdvertRoutes(narNode, nodeAdvertRouteLimit, time.Now(), floodAdvertRowCap) + if err != nil { + t.Fatal(err) + } + if got := narHashes(byRoute.Mixed); !reflect.DeepEqual(got, []string{"mixed1"}) { + t.Fatalf("mixed list = %v", got) + } + if got := narHashes(byRoute.Flood); !reflect.DeepEqual(got, []string{"flood1"}) { + t.Fatalf("flood list = %v (mixed advert must not appear)", got) + } + if got := narHashes(byRoute.ZeroHop); !reflect.DeepEqual(got, []string{"zh1"}) { + t.Fatalf("zero_hop list = %v (mixed advert must not appear)", got) + } + want := AdvertRouteCounts{Flood: 1, ZeroHop: 1, Mixed: 1} + if counts.H24 != want || counts.D7 != want { + t.Fatalf("counts 24h=%+v 7d=%+v, want %+v in both", counts.H24, counts.D7, want) + } +} + +// Rows the backfill has not reached (NULL mask) and databases without the +// column fall back to the first-inserted route_type, as #89 does: routes 0 +// and 1 are flood, 2 and 3 zero-hop, NULL route unknown. Unknown is only +// listed when there is data. +func TestNodeAdvertRoutes_LegacyFallback(t *testing.T) { + for _, withMask := range []bool{true, false} { + t.Run(fmt.Sprintf("column=%v", withMask), func(t *testing.T) { + db := setupTestDB(t) + if withMask { + narAddRouteMask(t, db) + } + narInsert(t, db, narNode, "legacy-r0", payloadTypeAdvert, 0, nil, narAgo(time.Hour), withMask) + narInsert(t, db, narNode, "legacy-r1", payloadTypeAdvert, 1, nil, narAgo(2*time.Hour), withMask) + narInsert(t, db, narNode, "legacy-r2", payloadTypeAdvert, 2, nil, narAgo(3*time.Hour), withMask) + byRoute, counts, _, err := db.GetNodeAdvertRoutes(narNode, nodeAdvertRouteLimit, time.Now(), floodAdvertRowCap) + if err != nil { + t.Fatal(err) + } + if got := narHashes(byRoute.Flood); !reflect.DeepEqual(got, []string{"legacy-r1", "legacy-r0"}) { + t.Fatalf("flood list = %v", got) + } + if got := narHashes(byRoute.ZeroHop); !reflect.DeepEqual(got, []string{"legacy-r2"}) { + t.Fatalf("zero_hop list = %v", got) + } + if byRoute.Unknown != nil { + t.Fatalf("unknown list must be absent without data, got %v", narHashes(byRoute.Unknown)) + } + if counts.H24 != (AdvertRouteCounts{Flood: 2, ZeroHop: 1}) { + t.Fatalf("24h counts = %+v", counts.H24) + } + narInsert(t, db, narNode, "legacy-null", payloadTypeAdvert, nil, nil, narAgo(4*time.Hour), withMask) + byRoute, counts, _, err = db.GetNodeAdvertRoutes(narNode, nodeAdvertRouteLimit, time.Now(), floodAdvertRowCap) + if err != nil { + t.Fatal(err) + } + if got := narHashes(byRoute.Unknown); !reflect.DeepEqual(got, []string{"legacy-null"}) { + t.Fatalf("unknown list = %v", got) + } + if counts.H24.Unknown != 1 { + t.Fatalf("24h unknown = %d, want 1", counts.H24.Unknown) + } + }) + } +} + +// Only ADVERTs (payload_type 4) are listed and counted. +func TestNodeAdvertRoutes_AdvertsOnly(t *testing.T) { + db := narDB(t) + narInsert(t, db, narNode, "grp1", 5, 1, 0b0010, narAgo(time.Hour), true) + narInsert(t, db, narNode, "adv1", payloadTypeAdvert, 1, 0b0010, narAgo(2*time.Hour), true) + byRoute, counts, _, err := db.GetNodeAdvertRoutes(narNode, nodeAdvertRouteLimit, time.Now(), floodAdvertRowCap) + if err != nil { + t.Fatal(err) + } + if got := narHashes(byRoute.Flood); !reflect.DeepEqual(got, []string{"adv1"}) { + t.Fatalf("flood list = %v", got) + } + if counts.H24.Flood != 1 { + t.Fatalf("24h flood = %d, want 1", counts.H24.Flood) + } +} + +// Window boundaries on first_seen, across the first_seen formats the +// ingestor writes: the 24h window holds only the last 24 hours; 7d excludes +// an advert that sits inside the SQL date floor (window + 1 day slack) but +// outside 7 days. A legacy space-separated value is skipped, exactly as +// flood_advert_count_7d skips it (shared parseRelayTS window). +func TestNodeAdvertRoutes_WindowsAndTimestampFormats(t *testing.T) { + db := narDB(t) + now := time.Now().UTC() + at := func(d time.Duration, layout string) string { return now.Add(-d).Format(layout) } + narInsert(t, db, narNode, "w-1h-rfc", payloadTypeAdvert, 1, 0b0010, at(time.Hour, time.RFC3339), true) + narInsert(t, db, narNode, "w-2h-nano", payloadTypeAdvert, 1, 0b0010, at(2*time.Hour, time.RFC3339Nano), true) + narInsert(t, db, narNode, "w-2h-space", payloadTypeAdvert, 1, 0b0010, at(2*time.Hour, "2006-01-02 15:04:05"), true) + narInsert(t, db, narNode, "w-3h-millis", payloadTypeAdvert, 2, 0b0100, at(3*time.Hour, "2006-01-02T15:04:05.000Z"), true) + narInsert(t, db, narNode, "w-23h", payloadTypeAdvert, 2, 0b1000, at(23*time.Hour, time.RFC3339), true) + narInsert(t, db, narNode, "w-25h", payloadTypeAdvert, 1, 0b0001, at(25*time.Hour, time.RFC3339), true) + narInsert(t, db, narNode, "w-6d-millis", payloadTypeAdvert, 2, 0b0100, at(6*24*time.Hour, "2006-01-02T15:04:05.000Z"), true) + narInsert(t, db, narNode, "w-7.5d", payloadTypeAdvert, 1, 0b0010, at(180*time.Hour, time.RFC3339), true) + narInsert(t, db, narNode, "w-9d", payloadTypeAdvert, 1, 0b0010, at(9*24*time.Hour, time.RFC3339), true) + narInsert(t, db, narNode, "w-bad", payloadTypeAdvert, 1, 0b0010, "not-a-time-but-sorts-high", true) + _, counts, _, err := db.GetNodeAdvertRoutes(narNode, nodeAdvertRouteLimit, now, floodAdvertRowCap) + if err != nil { + t.Fatal(err) + } + if want := (AdvertRouteCounts{Flood: 2, ZeroHop: 2}); counts.H24 != want { + t.Fatalf("24h = %+v, want %+v", counts.H24, want) + } + if want := (AdvertRouteCounts{Flood: 3, ZeroHop: 3}); counts.D7 != want { + t.Fatalf("7d = %+v, want %+v", counts.D7, want) + } + if counts.Truncated { + t.Fatal("not truncated below the row cap") + } +} + +// Pure counter: duplicate hashes count once, hash-less rows dedup by +// timestamp, unparseable timestamps are skipped, every class is counted. +func TestCountAdvertRoutes_DedupAndClasses(t *testing.T) { + now := time.Now() + entries := []advertRouteEntry{ + {ts: advertTS(1), hash: "d1", class: advertClassFlood}, + {ts: advertTS(2), hash: "d1", class: advertClassFlood}, // same advert, second row + {ts: advertTS(3), hash: "d2", class: advertClassZeroHop}, + {ts: advertTS(4), hash: "d3", class: advertClassMixed}, + {ts: advertTS(5), hash: "d4", class: advertClassUnknown}, + {ts: advertTS(6), class: advertClassZeroHop}, + {ts: advertTS(7), class: advertClassZeroHop}, + {ts: "garbage", hash: "d5", class: advertClassFlood}, + {ts: advertTS(30), hash: "d6", class: advertClassFlood}, + } + if got, want := countAdvertRoutes(entries, now, 24), (AdvertRouteCounts{Flood: 1, ZeroHop: 3, Mixed: 1, Unknown: 1}); got != want { + t.Fatalf("24h = %+v, want %+v", got, want) + } + if got, want := countAdvertRoutes(entries, now, 7*24), (AdvertRouteCounts{Flood: 2, ZeroHop: 3, Mixed: 1, Unknown: 1}); got != want { + t.Fatalf("7d = %+v, want %+v", got, want) + } +} + +// The row cap bounds per-request work: past it the counts saturate (newest +// rows win) and the response says so; the per-class lists are unaffected. +func TestNodeAdvertRoutes_RowCap(t *testing.T) { + db := narDB(t) + narInsert(t, db, narNode, "cap-flood", payloadTypeAdvert, 1, 0b0010, narAgo(10*time.Hour), true) + for i := 0; i < 3; i++ { + narInsert(t, db, narNode, fmt.Sprintf("cap%d", i), payloadTypeAdvert, 2, 0b0100, narAgo(time.Duration(3-i)*time.Hour), true) + } + byRoute, counts, _, err := db.GetNodeAdvertRoutes(narNode, nodeAdvertRouteLimit, time.Now(), 2) + if err != nil { + t.Fatal(err) + } + if !counts.Truncated { + t.Fatal("want truncated=true past the row cap") + } + if counts.H24 != (AdvertRouteCounts{ZeroHop: 2}) { + t.Fatalf("24h = %+v, want the newest 2 rows only", counts.H24) + } + if got := narHashes(byRoute.Flood); !reflect.DeepEqual(got, []string{"cap-flood"}) { + t.Fatalf("flood list = %v, the row cap must not starve the lists", got) + } +} + +type narResponse struct { + Node map[string]interface{} `json:"node"` + RecentAdverts []map[string]interface{} `json:"recentAdverts"` + ByRoute *struct { + Limit int `json:"limit"` + Flood []map[string]interface{} `json:"flood"` + ZeroHop []map[string]interface{} `json:"zero_hop"` + Mixed []map[string]interface{} `json:"mixed"` + Unknown []map[string]interface{} `json:"unknown"` + } `json:"recentAdvertsByRoute"` + Counts *struct { + H24 map[string]int `json:"24h"` + D7 map[string]int `json:"7d"` + Truncated bool `json:"truncated"` + Backfill RouteMaskBackfillStatus `json:"route_mask_backfill"` + } `json:"advertCounts"` +} + +func narGetNode(t *testing.T, router http.Handler, pubkey string, wantCode int) (narResponse, string) { + t.Helper() + req := httptest.NewRequest("GET", "/api/nodes/"+pubkey, nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != wantCode { + t.Fatalf("GET /api/nodes/%s = %d, want %d: %s", pubkey, w.Code, wantCode, w.Body.String()) + } + var body narResponse + if wantCode == 200 { + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("bad JSON: %v", err) + } + } + return body, w.Body.String() +} + +func narServer(t *testing.T) (*Server, http.Handler) { + t.Helper() + srv, router := setupTestServer(t) + narAddRouteMask(t, srv.db) + if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, last_seen, first_seen) VALUES (?, 'Route Mix', 'repeater', ?, ?)`, + narNode, narAgo(time.Hour), narAgo(48*time.Hour)); err != nil { + t.Fatal(err) + } + narInsert(t, srv.db, narNode, "h-flood-r0", payloadTypeAdvert, 0, 0b0001, narAgo(time.Hour), true) + narInsert(t, srv.db, narNode, "h-flood-r1", payloadTypeAdvert, 1, 0b0010, narAgo(2*time.Hour), true) + narInsert(t, srv.db, narNode, "h-mixed", payloadTypeAdvert, 2, 0b0110, narAgo(3*time.Hour), true) + narInsert(t, srv.db, narNode, "h-zh", payloadTypeAdvert, 2, 0b0100, narAgo(4*time.Hour), true) + narInsert(t, srv.db, narNode, "h-legacy", payloadTypeAdvert, 1, nil, narAgo(30*time.Hour), true) + return srv, router +} + +// The wire contract: recentAdverts keeps its shape (plus route_class), +// recentAdvertsByRoute and advertCounts are added, and flood_advert_count_7d +// keeps its external meaning. +func TestNodeDetail_AdvertRouteFields(t *testing.T) { + _, router := narServer(t) + body, _ := narGetNode(t, router, narNode+narIncludeQuery, 200) + if len(body.RecentAdverts) != 5 { + t.Fatalf("recentAdverts = %d rows, want 5", len(body.RecentAdverts)) + } + for _, key := range []string{"id", "hash", "first_seen", "timestamp", "route_type", "payload_type", "observations", "observation_count", "route_class"} { + if _, ok := body.RecentAdverts[0][key]; !ok { + t.Fatalf("recentAdverts row lacks %q: %v", key, body.RecentAdverts[0]) + } + } + if body.RecentAdverts[0]["hash"] != "h-legacy" || body.RecentAdverts[0]["route_class"] != advertClassFlood { + t.Fatalf("recentAdverts must stay in ingest (id DESC) order: first = %v/%v", body.RecentAdverts[0]["hash"], body.RecentAdverts[0]["route_class"]) + } + if body.ByRoute == nil || body.Counts == nil { + t.Fatal("recentAdvertsByRoute / advertCounts missing") + } + if body.ByRoute.Limit != nodeAdvertRouteLimit || len(body.ByRoute.Flood) != 3 || len(body.ByRoute.Mixed) != 1 || len(body.ByRoute.ZeroHop) != 1 { + t.Fatalf("byRoute limit=%d flood=%d mixed=%d zero_hop=%d", body.ByRoute.Limit, len(body.ByRoute.Flood), len(body.ByRoute.Mixed), len(body.ByRoute.ZeroHop)) + } + if body.ByRoute.Unknown != nil { + t.Fatal("unknown list must be omitted without data") + } + if got := body.Counts.H24; got["flood"] != 2 || got["zero_hop"] != 1 || got["mixed"] != 1 || got["unknown"] != 0 { + t.Fatalf("24h = %v", got) + } + if got := body.Counts.D7; got["flood"] != 3 || got["zero_hop"] != 1 || got["mixed"] != 1 { + t.Fatalf("7d = %v", got) + } + if body.Counts.Backfill.Status == "" { + t.Fatal("route_mask_backfill status missing") + } + // flood_advert_count_7d is an external contract (ArcScope advisor): it + // still counts route_type 1 only, so it leaves out route 0 (transport + // flood) and the mixed advert whose first-inserted route was zero-hop - + // here 2 against advertCounts["7d"].flood = 3 (+1 mixed). + if got := body.Node["flood_advert_count_7d"]; got != float64(2) { + t.Fatalf("flood_advert_count_7d = %v, want 2 (route_type 1 only, unchanged)", got) + } +} + +// Privacy (#68): the new fields follow the node-detail visibility rule and +// identityHidden. Blacklisted and hidden-name nodes stay 404; an identity +// hidden only through the observer blacklist gets no route breakdown. +func TestNodeDetail_AdvertRouteFieldsPrivacy(t *testing.T) { + t.Run("node blacklist", func(t *testing.T) { + srv, router := narServer(t) + srv.cfg.SetNodeBlacklist([]string{narNode}) + _, raw := narGetNode(t, router, narNode+narIncludeQuery, 404) + if strings.Contains(raw, "advertCounts") || strings.Contains(raw, "h-flood") { + t.Fatalf("blacklisted node leaked: %s", raw) + } + }) + t.Run("hidden name prefix", func(t *testing.T) { + srv, router := narServer(t) + srv.cfg.SetHiddenNamePrefixes([]string{"Route"}) + _, raw := narGetNode(t, router, narNode+narIncludeQuery, 404) + if strings.Contains(raw, "advertCounts") || strings.Contains(raw, "h-flood") { + t.Fatalf("hidden node leaked: %s", raw) + } + }) + // route_class is route_mask data too: it must not ride along on the + // recentAdverts rows of a hidden identity. + leaked := func(body narResponse, raw string) bool { + return body.ByRoute != nil || body.Counts != nil || strings.Contains(raw, "advertCounts") || + strings.Contains(raw, "recentAdvertsByRoute") || strings.Contains(raw, "route_class") + } + t.Run("observer blacklist", func(t *testing.T) { + srv, router := narServer(t) + srv.cfg.ObserverBlacklist = []string{strings.ToUpper(narNode)} // read lazily on first use + body, raw := narGetNode(t, router, narNode+narIncludeQuery, 200) + if leaked(body, raw) || len(body.RecentAdverts) != 5 { + t.Fatalf("identity hidden via observer blacklist leaked route fields (or lost recentAdverts): %s", raw) + } + }) + t.Run("hidden observer name", func(t *testing.T) { + srv, router := narServer(t) + if _, err := srv.db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES (?, 'Secret Observer', 'TST')`, narNode); err != nil { + t.Fatal(err) + } + srv.cfg.SetHiddenNamePrefixes([]string{"Secret"}) + body, raw := narGetNode(t, router, narNode+narIncludeQuery, 200) + if leaked(body, raw) { + t.Fatalf("identity hidden via its observer name leaked route fields: %s", raw) + } + }) +} + +// recentAdverts keeps its chronological role for nodeWithHealthActivity and +// other consumers: same 20-row limit and id DESC order as before. route_class +// is added only when asked for (include=advertRoutes); otherwise the rows are +// exactly master's. +func TestGetRecentTransmissionsForNode_RouteClassAdditive(t *testing.T) { + db := narDB(t) + for i := 0; i < 25; i++ { + narInsert(t, db, narNode, fmt.Sprintf("r%02d", i), payloadTypeAdvert, 2, 0b0100, narAgo(time.Duration(25-i)*time.Minute), true) + } + narInsert(t, db, narNode, "r-grp", 5, 1, 0b0010, narAgo(time.Minute), true) + for _, withRouteClass := range []bool{true, false} { + rows, err := db.GetRecentTransmissionsForNode(narNode, 20, withRouteClass) + if err != nil { + t.Fatal(err) + } + if len(rows) != 20 || rows[0]["hash"] != "r-grp" || rows[1]["hash"] != "r24" { + t.Fatalf("withRouteClass=%v: rows=%d first=%v second=%v", withRouteClass, len(rows), rows[0]["hash"], rows[1]["hash"]) + } + if _, ok := rows[0]["route_class"]; ok { + t.Fatal("route_class is an ADVERT classification; non-advert rows must not carry it") + } + if _, ok := rows[1]["observations"]; !ok { + t.Fatalf("withRouteClass=%v: recentAdverts rows keep their observations", withRouteClass) + } + got, ok := rows[1]["route_class"] + if withRouteClass && got != advertClassZeroHop { + t.Fatalf("advert route_class = %v", got) + } + if !withRouteClass && ok { + t.Fatalf("without include=advertRoutes the rows must not carry route_class, got %v", got) + } + } +} + +// OpenAPI documents every JSON field the new response types emit. +func TestOpenAPI_NodeAdvertRouteSchemas(t *testing.T) { + spec := fetchSpec(t) + schemas := asMap(t, asMap(t, spec["components"], "components")["schemas"], "schemas") + props := func(name string) map[string]interface{} { + return asMap(t, asMap(t, schemas[name], name)["properties"], name+".properties") + } + check := func(schema string, v interface{}) { + p := props(schema) + rt := reflect.TypeOf(v) + for i := 0; i < rt.NumField(); i++ { + tag := strings.Split(rt.Field(i).Tag.Get("json"), ",")[0] + if tag == "" || tag == "-" { + continue + } + if _, ok := p[tag]; !ok { + t.Errorf("%s.%s is emitted but not documented", schema, tag) + } + } + } + check("NodeDetailResponse", NodeDetailResponse{}) + check("NodeAdvertsByRoute", NodeAdvertsByRoute{}) + check("NodeAdvertCounts", NodeAdvertCounts{}) + check("AdvertRouteCounts", AdvertRouteCounts{}) + if _, ok := props("NodeAdvert")["route_class"]; !ok { + t.Error("NodeAdvert.route_class is emitted but not documented") + } + get := asMap(t, asMap(t, asMap(t, spec["paths"], "paths")["/api/nodes/{pubkey}"], "/api/nodes/{pubkey}")["get"], "get") + params, _ := get["parameters"].([]interface{}) + documented := false + for _, p := range params { + if m, ok := p.(map[string]interface{}); ok && m["name"] == "include" && m["in"] == "query" && + strings.Contains(fmt.Sprint(m["description"]), nodeDetailIncludeAdvertRoutes) { + documented = true + } + } + if !documented { + t.Errorf("GET /api/nodes/{pubkey} must document the include=%s opt-in: %v", nodeDetailIncludeAdvertRoutes, params) + } +} diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index fcb1e168e..33f46f12e 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -87,7 +87,10 @@ func routeDescriptions() map[string]routeMeta { "GET /api/nodes/search": {Summary: "Search nodes", Description: "Search nodes by name or public key prefix.", Tag: "nodes", QueryParams: []paramMeta{{Name: "q", Description: "Search query", Type: "string", Required: true}}}, "GET /api/nodes/bulk-health": {Summary: "Bulk node health", Description: "Returns health status for all nodes in one call.", Tag: "nodes"}, "GET /api/nodes/network-status": {Summary: "Network status summary", Description: "Returns counts of active, stale, and offline nodes.", Tag: "nodes"}, - "GET /api/nodes/{pubkey}": {Summary: "Get node detail", Description: "Returns full detail for a single node by public key. For repeater/room nodes this includes the issue #672 usefulness axes + composite score/grade (see the Node schema).", Tag: "nodes", Response: schemaRef("NodeDetailResponse")}, + "GET /api/nodes/{pubkey}": {Summary: "Get node detail", Description: "Returns full detail for a single node by public key. For repeater/room nodes this includes the issue #672 usefulness axes + composite score/grade (see the Node schema). recentAdverts is the chronological list; with include=advertRoutes, recentAdvertsByRoute and advertCounts (#2073) split the node's adverts into flood / zero_hop / mixed and the ADVERT rows of recentAdverts carry route_class.", Tag: "nodes", Response: schemaRef("NodeDetailResponse"), + QueryParams: []paramMeta{ + {Name: "include", Description: "Opt-in extras, comma-separated (the parameter may also repeat). advertRoutes (#2073): adds recentAdvertsByRoute, advertCounts and route_class on the recentAdverts ADVERT rows. That costs a scan of all the node's ADVERT rows (cached per node for up to 30 s), so only the node page asks for it; without it the response has neither field and no route_class. Unknown values are ignored. Hidden identities never get the extras.", Type: "string"}, + }}, "GET /api/nodes/{pubkey}/clock-skew": {Summary: "Get node clock skew", Description: "Per-node clock-skew analysis derived from ADVERT advert-timestamps vs observation times, calibrated per observer (see ClockSkewEngine). samples is the full per-advert time series in chronological order (sparkline data) by default. Fase 5.2b's sample_limit trims that array before it's sent to the client — it only reduces JSON serialization/payload/client-decoding cost, not the server-side computation or allocation that already produced the full samples slice.", Tag: "nodes", QueryParams: []paramMeta{ {Name: "sample_limit", Description: "Trims samples to at most the N most-recent entries, chronological order preserved. Omitted, non-numeric, or negative: unchanged, every sample returned (legacy default). 0: samples is omitted from the response entirely. N at or above the current sample count: unchanged, every sample returned. sampleCount and every other field are unaffected regardless of sample_limit.", Type: "integer"}, @@ -122,7 +125,7 @@ func routeDescriptions() map[string]routeMeta { "GET /api/analytics/subpaths-bulk": {Summary: "Bulk subpath analysis", Tag: "analytics"}, "GET /api/analytics/subpath-detail": {Summary: "Subpath detail", Tag: "analytics"}, "GET /api/analytics/neighbor-graph": {Summary: "Neighbor graph", Description: "Full neighbor affinity graph for visualization.", Tag: "analytics"}, - "GET /api/analytics/relay-airtime-share": {Summary: "Relay airtime share per payload type", Description: "Overview tab's Relay Airtime Share. Each distinct packet (deduplicated by content hash) scores LoRa Time-on-Air of its raw bytes × distinct repeaters that relayed it, under the assumed preset returned in preset (config analytics.loraPreset); originator TX is excluded. rows[] = {payload_type, type, route_class, count, count_pct, score, airtime_pct}, sorted by airtime_pct desc. type is the numeric payload type; payload_type is a display label and may change. ADVERT is split into up to four rows that all have type 4, classified from transmissions.route_mask (every raw route type observed for the content hash, independent of ingest order): route_class \"flood\" (only route 0/1 seen, label \"ADVERT (flood)\"), \"zero_hop\" (only route 2/3, label \"ADVERT (zero-hop)\"), \"mixed\" (the same payload was seen on both flood and zero-hop routes, e.g. a contact re-shared as a zero-hop advert; label \"ADVERT (mixed)\"; counted once, with all of its relays) and \"legacy\" (no usable route, label \"ADVERT\"). Rows whose route_mask is not backfilled yet fall back to the legacy first-inserted route_type; route_mask_backfill {status: pending|backfilling|complete, remaining} reports whether that fallback is still in use: complete only when no transmission lacks a mask and this server has read every backfilled mask (the server picks them up while running); remaining counts rows without a mask, or, once none are left, an upper bound on the transmissions this server has not re-read yet (null while it cannot be counted). route_class is null on every other row, so key rows by (type, route_class), never by type alone. Time-on-Air uses the frame of the first inserted observation, so its length (path bytes, transport codes) can still depend on ingest order. count_pct and airtime_pct are shares of total_count and total_score (nanoseconds). Payload Type Mix (/api/analytics/rf payloadTypes) is not split. Cached in the RF analytics cache (60s default).", Tag: "analytics", + "GET /api/analytics/relay-airtime-share": {Summary: "Relay airtime share per payload type", Description: "Overview tab's Relay Airtime Share. Each distinct packet (deduplicated by content hash) scores LoRa Time-on-Air of its raw bytes × distinct repeaters that relayed it, under the assumed preset returned in preset (config analytics.loraPreset); originator TX is excluded. rows[] = {payload_type, type, route_class, count, count_pct, score, airtime_pct}, sorted by airtime_pct desc. type is the numeric payload type; payload_type is a display label and may change. ADVERT is split into up to four rows that all have type 4, classified from transmissions.route_mask (every raw route type observed for the content hash, independent of ingest order): route_class \"flood\" (only route 0/1 seen, label \"ADVERT (flood)\"), \"zero_hop\" (only route 2/3, label \"ADVERT (zero-hop)\"), \"mixed\" (the same payload was seen on both flood and zero-hop routes, e.g. a contact re-shared as a zero-hop advert; label \"ADVERT (mixed)\"; counted once, with all of its relays) and \"legacy\" (no usable route, label \"ADVERT\"; node detail's route_class calls the same bucket \"unknown\", one classifier). Rows whose route_mask is not backfilled yet fall back to the legacy first-inserted route_type; route_mask_backfill {status: pending|backfilling|complete, remaining} reports whether that fallback is still in use: complete only when no transmission lacks a mask and this server has read every backfilled mask (the server picks them up while running); remaining counts rows without a mask, or, once none are left, an upper bound on the transmissions this server has not re-read yet (null while it cannot be counted). route_class is null on every other row, so key rows by (type, route_class), never by type alone. Time-on-Air uses the frame of the first inserted observation, so its length (path bytes, transport codes) can still depend on ingest order. count_pct and airtime_pct are shares of total_count and total_score (nanoseconds). Payload Type Mix (/api/analytics/rf payloadTypes) is not split. Cached in the RF analytics cache (60s default).", Tag: "analytics", QueryParams: []paramMeta{ {Name: "window", Description: "Relative window: 1h, 24h/1d, 3d, 7d/1w or 30d (default: all loaded packets)", Type: "string"}, {Name: "from", Description: "RFC3339 start; if from or to is given, window is ignored (an unparseable value leaves that bound open)", Type: "string"}, @@ -205,6 +208,69 @@ func schemaRef(name string) map[string]interface{} { return map[string]interface{}{"$ref": "#/components/schemas/" + name} } +// openAPISchema is a typed OpenAPI 3.0 schema object. It marshals to the +// same JSON as the map literals in componentSchemas; schemas added since +// #2073 use it instead of new untyped map literals (#1383). +type openAPISchema struct { + Ref string `json:"$ref,omitempty"` + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` + Enum []string `json:"enum,omitempty"` + Nullable bool `json:"nullable,omitempty"` + Minimum *int `json:"minimum,omitempty"` + Items *openAPISchema `json:"items,omitempty"` + Properties map[string]*openAPISchema `json:"properties,omitempty"` +} + +func openAPIRef(name string) *openAPISchema { + return &openAPISchema{Ref: "#/components/schemas/" + name} +} + +// nodeAdvertRouteSchemas documents the #2073 node-detail advert route fields +// (port of upstream Kpa-clawbot/CoreScope#2073). +func nodeAdvertRouteSchemas() map[string]*openAPISchema { + zero := 0 + count := &openAPISchema{Type: "integer", Minimum: &zero} + list := func(desc string) *openAPISchema { + return &openAPISchema{Type: "array", Items: openAPIRef("NodeAdvert"), Description: desc} + } + return map[string]*openAPISchema{ + "NodeAdvertsByRoute": { + Type: "object", + Description: "Node detail with include=advertRoutes only (#2073): the newest ADVERTs of the node per route class (see NodeAdvert.route_class), newest ingest first. The class is filtered before the per-class limit, so frequent zero-hop adverts cannot push rare flood adverts out; a mixed advert is listed only under mixed. Rows are the NodeAdvert shape without the observations array (observation_count and the best observation's observer/snr/rssi/path fields are kept); route_class is the class the row was listed under. Absent without include=advertRoutes and when the node's identity is hidden (node or observer blacklist, hidden-name prefix). Cached per node for up to 30 s (refreshed once the node has a newer transmission, at most every 5 s).", + Properties: map[string]*openAPISchema{ + "limit": {Type: "integer", Description: "Maximum rows per class (20)."}, + "flood": list("Adverts seen only on flood routes (0/1)."), + "zero_hop": list("Adverts seen only on zero-hop routes (2/3)."), + "mixed": list("Adverts seen on both flood and zero-hop routes."), + "unknown": list("Adverts with no usable route (Relay Airtime Share's legacy bucket); present only when the node has any."), + }, + }, + "NodeAdvertCounts": { + Type: "object", + Description: "Node detail with include=advertRoutes only (#2073): distinct ADVERTs (by content hash) per route class whose first_seen - when the advert was first heard, the axis flood_advert_count_7d also uses - lies in the last 24 hours / 7 days. Classified like NodeAdvert.route_class. Unlike Node.flood_advert_count_7d (route_type 1 only, unchanged external contract), 7d.flood also counts transport flood (route 0) and never counts a mixed advert. Rows whose first_seen cannot be parsed are skipped, as for flood_advert_count_7d. unknown is the bucket Relay Airtime Share calls legacy (same classifier, see NodeAdvert.route_class). Absent without include=advertRoutes and when the node's identity is hidden.", + Properties: map[string]*openAPISchema{ + "24h": openAPIRef("AdvertRouteCounts"), + "7d": openAPIRef("AdvertRouteCounts"), + "truncated": {Type: "boolean", Description: "true when the node had more adverts at or after the 7d date floor than the per-request row cap (50000); the counts then cover the newest rows only."}, + "route_mask_backfill": openAPIRef("RouteMaskBackfillStatus"), + }, + }, + "AdvertRouteCounts": { + Type: "object", + Properties: map[string]*openAPISchema{"flood": count, "zero_hop": count, "mixed": count, "unknown": count}, + }, + "RouteMaskBackfillStatus": { + Type: "object", + Description: "The ingestor's transmissions.route_mask backfill (#89). Until complete, rows without a mask are classified by their first-inserted route_type, so route classes are provisional.", + Properties: map[string]*openAPISchema{ + "status": {Type: "string", Enum: []string{"pending", "backfilling", "complete"}}, + "remaining": {Type: "integer", Nullable: true, Description: "Rows still without a mask; null when it cannot be counted cheaply."}, + }, + }, + } +} + // componentSchemas returns the reusable OpenAPI schemas surfaced under // components/schemas. The Node schema documents the per-node usefulness // metrics (issue #672) that the /api/nodes handlers attach to repeater/room @@ -226,7 +292,7 @@ func componentSchemas() map[string]interface{} { } return m } - return map[string]interface{}{ + schemas := map[string]interface{}{ "Node": map[string]interface{}{ "type": "object", // additionalProperties:true — the node object carries more fields @@ -287,8 +353,10 @@ func componentSchemas() map[string]interface{} { "NodeDetailResponse": map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ - "node": schemaRef("Node"), - "recentAdverts": map[string]interface{}{"type": "array", "items": schemaRef("NodeAdvert"), "description": "Up to 20 most recent transmissions from this node (newest first)."}, + "node": schemaRef("Node"), + "recentAdverts": map[string]interface{}{"type": "array", "items": schemaRef("NodeAdvert"), "description": "Up to 20 most recent transmissions from this node (newest ingest first, #1345), all route classes together."}, + "recentAdvertsByRoute": openAPIRef("NodeAdvertsByRoute"), + "advertCounts": openAPIRef("NodeAdvertCounts"), }, }, "NodeAdvert": map[string]interface{}{ @@ -301,6 +369,7 @@ func componentSchemas() map[string]interface{} { "payload_type": map[string]interface{}{"type": "integer", "description": "MeshCore payload type."}, "first_seen": str("RFC3339 time the transmission was first observed."), "from_pubkey": str("Originating node public key."), + "route_class": &openAPISchema{Type: "string", Enum: []string{advertClassFlood, advertClassZeroHop, advertClassMixed, advertClassUnknown}, Description: "Node detail with include=advertRoutes, ADVERT rows only (absent on other payload types and without the opt-in): the advert's route class, classified like Relay Airtime Share's ADVERT rows. flood: only route 0/1 (transport flood/flood) seen for the content hash; zero_hop: only route 2/3 (a zero-hop advert is sent DIRECT with an empty path); mixed: both; unknown: no usable route - the same bucket Relay Airtime Share's route_class calls \"legacy\" (its historical plain ADVERT row); node detail names it for what it is, both come from one classifier. Omitted when the node's identity is hidden. From transmissions.route_mask, falling back to the first-inserted route_type while the mask is not backfilled (see advertCounts.route_mask_backfill)."}, }, }, "CandidateEntry": map[string]interface{}{ @@ -711,6 +780,10 @@ func componentSchemas() map[string]interface{} { }, }, } + for name, schema := range nodeAdvertRouteSchemas() { + schemas[name] = schema + } + return schemas } // buildOpenAPISpec constructs an OpenAPI 3.0 spec by walking the mux router. diff --git a/cmd/server/relay_airtime_share.go b/cmd/server/relay_airtime_share.go index 688e06af5..546e0ba86 100644 --- a/cmd/server/relay_airtime_share.go +++ b/cmd/server/relay_airtime_share.go @@ -214,17 +214,19 @@ type relayAirtimeBucketKey struct { advertRoute relayAirtimeAdvertRoute } -// advertRouteClass is the single place that classifies an ADVERT's route. -// With a known route_mask (#89) it uses every raw route observed for the -// hash, so the result does not depend on ingest order: only routes 0/1 is -// flood, only routes 2/3 is zero-hop, both groups is mixed. Without usable -// mask bits (column absent, row not backfilled yet, or no valid route ever -// recorded) it falls back to the legacy first-inserted route_type, and -// NULL/out-of-range values stay unknown (the historical ADVERT bucket). -func advertRouteClass(tx *StoreTx) relayAirtimeAdvertRoute { - if tx.routeMaskKnown && int64(tx.routeMask)&packetpath.RouteMaskAll != 0 { - flood := int64(tx.routeMask)&packetpath.RouteMaskFlood != 0 - direct := int64(tx.routeMask)&packetpath.RouteMaskDirect != 0 +// classifyAdvertRoute is the single place that classifies an ADVERT's route; +// Relay Airtime Share (advertRouteClass) and the node-detail advert lists +// (node_advert_routes.go, whose SQL CASE mirrors it) both use it. With a +// known route_mask (#89) it uses every raw route observed for the hash, so +// the result does not depend on ingest order: only routes 0/1 is flood, only +// routes 2/3 is zero-hop, both groups is mixed. Without usable mask bits +// (column absent, row not backfilled yet, or no valid route ever recorded) +// it falls back to the legacy first-inserted route_type, and NULL/out-of- +// range values stay unknown (the historical ADVERT bucket). +func classifyAdvertRoute(mask int64, maskKnown bool, routeType *int) relayAirtimeAdvertRoute { + if maskKnown && mask&packetpath.RouteMaskAll != 0 { + flood := mask&packetpath.RouteMaskFlood != 0 + direct := mask&packetpath.RouteMaskDirect != 0 switch { case flood && direct: return relayAirtimeAdvertMixed @@ -234,10 +236,10 @@ func advertRouteClass(tx *StoreTx) relayAirtimeAdvertRoute { return relayAirtimeAdvertZeroHop } } - if tx.RouteType == nil { + if routeType == nil { return relayAirtimeAdvertUnknown } - switch *tx.RouteType { + switch *routeType { case RouteTransportFlood, RouteFlood: return relayAirtimeAdvertFlood case RouteDirect, RouteTransportDirect: @@ -246,6 +248,11 @@ func advertRouteClass(tx *StoreTx) relayAirtimeAdvertRoute { return relayAirtimeAdvertUnknown } +// advertRouteClass classifies an in-memory ADVERT (see classifyAdvertRoute). +func advertRouteClass(tx *StoreTx) relayAirtimeAdvertRoute { + return classifyAdvertRoute(int64(tx.routeMask), tx.routeMaskKnown, tx.RouteType) +} + // relayAirtimeKey keeps non-ADVERT aggregation unchanged while separating the // ADVERT route classes (see advertRouteClass). tx.PayloadType must be non-nil. func relayAirtimeKey(tx *StoreTx) relayAirtimeBucketKey { @@ -280,10 +287,14 @@ func relayAirtimeBucketName(key relayAirtimeBucketKey) string { // part of the API contract and deliberately independent of the display // labels built by relayAirtimeBucketName. const ( - relayAirtimeRouteClassFlood = "flood" - relayAirtimeRouteClassZeroHop = "zero_hop" - relayAirtimeRouteClassMixed = "mixed" - relayAirtimeRouteClassLegacy = "legacy" + relayAirtimeRouteClassFlood = advertClassFlood + relayAirtimeRouteClassZeroHop = advertClassZeroHop + relayAirtimeRouteClassMixed = advertClassMixed + // Relay Airtime Share names the no-usable-route bucket "legacy" (its + // historical plain ADVERT row, kept for API compatibility); node detail + // (include=advertRoutes) calls the same bucket advertClassUnknown. Both + // come from classifyAdvertRoute's relayAirtimeAdvertUnknown. + relayAirtimeRouteClassLegacy = "legacy" ) // relayAirtimeRouteClass returns the route_class value for a row: one of the diff --git a/cmd/server/routes.go b/cmd/server/routes.go index efbaff8c9..30f1c5e70 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -73,6 +73,10 @@ type Server struct { neighborMu sync.Mutex neighborGraph *NeighborGraph + // #2073 node-detail advert route breakdown, per pubkey + // (node_advert_routes_cache.go). + advertRoutes nodeAdvertRouteCache + // Cached /api/scope-stats response — per-window, recomputed at most once every 30s scopeStatsMu sync.Mutex scopeStatsCache map[string]*ScopeStatsResponse @@ -2093,15 +2097,50 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) { } } + // #2073: adverts per route class and per-class counts, only on request + // (include=advertRoutes; the node page sends it, other callers of this + // endpoint do not pay for it). Same visibility rule as the rest of node + // detail plus identityHidden (#68), which also covers the observer + // blacklist and observer names, checked before the breakdown's cache; a + // failed lookup fails closed. A hidden identity gets neither the new + // fields nor route_class on recentAdverts (it is route_mask data too). + advertRoutes := wantsNodeAdvertRoutes(r) + if advertRoutes { + hidden, err := s.isIdentityHidden(r.Context(), pubkey) + if err != nil { + log.Printf("WARN isIdentityHidden(%s): %v", pubkey, err) + } + advertRoutes = err == nil && !hidden + } + // #1143: GetRecentTransmissionsForNode no longer accepts a name fallback; // attribution is strict exact-match on the indexed from_pubkey column. - recentAdverts, _ := s.db.GetRecentTransmissionsForNode(pubkey, 20) + recentAdverts, _ := s.db.GetRecentTransmissionsForNode(pubkey, 20, advertRoutes) + + resp := NodeDetailResponse{ + Node: node, + RecentAdverts: recentAdverts, + } + var floodFromScan *int + if advertRoutes { + if res, err := s.nodeAdvertRoutes(pubkey, time.Now()); err == nil { + resp.RecentAdvertsByRoute = &res.byRoute + resp.AdvertCounts = &res.counts + floodFromScan = res.floodAdvertCount7d + } else { + log.Printf("WARN nodeAdvertRoutes(%s): %v", pubkey, err) + } + } // Windowed flood-advert count (7d): only the mesh-wide-airtime advert kind, // separated from zero-hop adverts so a nearby observer hearing a node's // cheap local adverts does not inflate the number. Consumed by the ArcScope - // repeater advisor to rate advert hygiene. - if n, err := s.db.CountFloodAdvertsForNode(pubkey, 7*24, floodAdvertRowCap); err == nil { + // repeater advisor to rate advert hygiene. Always fresh, never cached: a + // breakdown scan this request ran itself yields the identical number + // (GetNodeAdvertRoutes), otherwise it is counted here. + if floodFromScan != nil { + node["flood_advert_count_7d"] = *floodFromScan + } else if n, err := s.db.CountFloodAdvertsForNode(pubkey, 7*24, floodAdvertRowCap); err == nil { node["flood_advert_count_7d"] = n } else { log.Printf("WARN CountFloodAdvertsForNode(%s): %v", pubkey, err) @@ -2135,10 +2174,7 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) { } } - writeJSON(w, NodeDetailResponse{ - Node: node, - RecentAdverts: recentAdverts, - }) + writeJSON(w, resp) } func (s *Server) handleNodeHealth(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/server/types.go b/cmd/server/types.go index f530fdb4b..980fd4b54 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -860,8 +860,53 @@ type NodeSearchResponse struct { } type NodeDetailResponse struct { - Node map[string]interface{} `json:"node"` - RecentAdverts []map[string]interface{} `json:"recentAdverts"` + Node map[string]interface{} `json:"node"` + RecentAdverts NodeAdvertRows `json:"recentAdverts"` + // #2073: omitted when the identity is hidden (identityHidden, #68). + RecentAdvertsByRoute *NodeAdvertsByRoute `json:"recentAdvertsByRoute,omitempty"` + AdvertCounts *NodeAdvertCounts `json:"advertCounts,omitempty"` +} + +// NodeAdvertRow is one transmission row on node detail: the /api/packets +// transmission shape of the shared scanTransmissionRow plus, depending on +// the list, observations and route_class (#2073). It stays map-shaped +// because that shared scanner is (#1383); the named type keeps the response +// fields typed. +type NodeAdvertRow map[string]interface{} + +// NodeAdvertRows is a list of NodeAdvertRow. +type NodeAdvertRows []NodeAdvertRow + +// NodeAdvertsByRoute is the newest adverts of a node per route class +// (#2073), classified like Relay Airtime Share (classifyAdvertRoute). The +// class is filtered in SQL before the per-class limit, so frequent zero-hop +// adverts cannot push rare flood adverts out. Unknown (no usable route) is +// only present when the node has such adverts. +type NodeAdvertsByRoute struct { + Limit int `json:"limit"` + Flood NodeAdvertRows `json:"flood"` + ZeroHop NodeAdvertRows `json:"zero_hop"` + Mixed NodeAdvertRows `json:"mixed"` + Unknown NodeAdvertRows `json:"unknown,omitempty"` +} + +// NodeAdvertCounts counts a node's distinct adverts (by hash) per route class +// whose first_seen lies in the last 24 hours and 7 days (#2073). +// RouteMaskBackfill tells whether legacy rows are still classified by their +// first-inserted route_type (anything but "complete": provisional). +type NodeAdvertCounts struct { + H24 AdvertRouteCounts `json:"24h"` + D7 AdvertRouteCounts `json:"7d"` + Truncated bool `json:"truncated"` + RouteMaskBackfill RouteMaskBackfillStatus `json:"route_mask_backfill"` +} + +// AdvertRouteCounts is one window of NodeAdvertCounts. +type AdvertRouteCounts struct { + Flood int `json:"flood"` + ZeroHop int `json:"zero_hop"` + Mixed int `json:"mixed"` + Unknown int `json:"unknown"` } type NodeStatsResp struct { diff --git a/docs/api-spec.md b/docs/api-spec.md index a2481e45c..e38c72403 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -464,6 +464,18 @@ Node detail page data. |----------|--------|----------------------| | `pubkey` | string | Node public key (hex)| +### Query Parameters + +| Param | Type | Description | +|-----------|--------|-------------| +| `include` | string | Opt-in extras, comma-separated (may also repeat). `advertRoutes` adds `recentAdvertsByRoute`, `advertCounts` and `route_class` on the `recentAdverts` ADVERT rows (see [Advert route classes](#advert-route-classes)). Unknown values are ignored. | + +Without `include=advertRoutes` the response is exactly the pre-#2073 one: +no `recentAdvertsByRoute`, no `advertCounts`, no `route_class`. The breakdown +scans all of the node's ADVERT rows, so only the node page (full view and +side panel) asks for it; the packets, live, channels and route views and the +claimed-nodes lookups do not. + ### Response `200` ```jsonc @@ -479,14 +491,73 @@ Node detail page data. "advert_count": number, "hash_size": number | null, "hash_size_inconsistent": boolean, - "hash_sizes_seen": [number] | undefined + "hash_sizes_seen": [number] | undefined, + "flood_advert_count_7d": number // route_type 1 only (see below) }, - "recentAdverts": [Packet] // last 20 packets for this node, newest first + "recentAdverts": [Packet], // last 20 packets for this node, newest ingest first; + // with include=advertRoutes ADVERT rows also carry route_class + "recentAdvertsByRoute": { // include=advertRoutes only; absent when the identity is hidden + "limit": 20, // max rows per class + "flood": [Packet], // newest adverts per route class, newest ingest first; + // rows without the observations array + "zero_hop": [Packet], + "mixed": [Packet], + "unknown": [Packet] // only present when the node has such adverts + }, + "advertCounts": { // include=advertRoutes only; absent when the identity is hidden + "24h": { "flood": number, "zero_hop": number, "mixed": number, "unknown": number }, + "7d": { "flood": number, "zero_hop": number, "mixed": number, "unknown": number }, + "truncated": boolean, // more adverts than the 50,000-row cap in the 7d floor + "route_mask_backfill": { "status": "pending" | "backfilling" | "complete", "remaining": number | null } + } } ``` Where `Packet` is a transmission object (see [Packet Object](#packet-object)). +#### Advert route classes + +`recentAdvertsByRoute`, `advertCounts` and `route_class` (port/extension of +upstream `Kpa-clawbot/CoreScope#2073`) classify ADVERTs exactly like the +ADVERT rows of Relay Airtime Share (#89), from `transmissions.route_mask` +(every raw route type observed for the content hash): + +| `route_class` | Meaning | +|---------------|---------| +| `flood` | only route 0/1 (transport flood / flood) seen | +| `zero_hop` | only route 2/3 — a zero-hop advert is sent DIRECT with an empty path | +| `mixed` | both flood and zero-hop routes seen for the same advert | +| `unknown` | no usable route — the same bucket Relay Airtime Share's `route_class` calls `legacy` (its historical plain ADVERT row); one classifier, two names kept for API compatibility | + +Rows whose mask is not backfilled yet (and databases without the column) +fall back to the first-inserted `route_type`; `route_mask_backfill` says +whether that fallback is still in use (anything but `complete`: provisional). + +- The class is filtered before the per-class limit, so frequent + zero-hop adverts cannot push rare flood adverts out of `flood`. A mixed + advert appears only under `mixed`. +- `advertCounts` counts distinct adverts (by hash) whose `first_seen` — when + the advert was first heard, the axis `flood_advert_count_7d` uses too — + lies in the window. Rows whose `first_seen` cannot be parsed are skipped, + as for `flood_advert_count_7d`. +- `flood_advert_count_7d` is unchanged (an external contract): it counts + `route_type` 1 only. `advertCounts["7d"].flood` differs from it: it also + counts transport flood (route 0) and never counts a mixed advert, while + `flood_advert_count_7d` counts a mixed advert whose first-inserted route + was 1. +- Both new fields follow the node-detail visibility rules (blacklisted and + hidden-name nodes are 404) and are omitted when the identity is hidden by + the observer blacklist or an observer name (#68); `route_class` is then + left off the `recentAdverts` rows as well. +- The per-class rows omit the `observations` array (they repeat rows of + `recentAdverts`); `observation_count` and the best observation's fields + stay. Their `route_class` is the class they were listed under. +- The breakdown is cached per node for up to 30 s, and refreshed once the + node has a newer transmission (at most every 5 s). `flood_advert_count_7d` + is never cached: it is counted on every request (a request that scanned + the node for the breakdown itself takes the identical number from that + scan). + ### Response `404` ```json diff --git a/public/index.html b/public/index.html index 423bdfe0f..0d2d3f980 100644 --- a/public/index.html +++ b/public/index.html @@ -215,6 +215,7 @@ + diff --git a/public/node-adverts.js b/public/node-adverts.js new file mode 100644 index 000000000..dc0822ca7 --- /dev/null +++ b/public/node-adverts.js @@ -0,0 +1,225 @@ +/* === CoreScope — node-adverts.js === + * + * The Recent Adverts panel, shared by the node detail page and the node side + * panel. The "Recent Adverts" name and its tooltip port upstream + * `Kpa-clawbot/CoreScope#2071`; the per-route tabs and counts extend upstream + * `Kpa-clawbot/CoreScope#2073` over the node-detail fields + * recentAdvertsByRoute and advertCounts (see docs/api-spec.md). + * + * render() returns an HTML string built only from escaped values; bind() + * wires the tab bar through the app's initTabBar (roles, arrow keys) and + * reports tab changes so the caller can keep ?adverts= in the URL. Nothing + * here fetches: everything comes from the one node-detail response, which + * carries the breakdown only when asked for (the advertRoutes opt-in, see + * docs/api-spec.md) - nodes.js fetchNodeDetail does, other callers of + * /api/nodes/{pubkey} do not pay for it. + */ +(function (root) { + 'use strict'; + + var PARAM = 'adverts'; + var TITLE_TIP = 'Adverts this node originated. The section is limited to adverts because they are the only packet type attributable to an originating node: transmissions.from_pubkey is populated for ADVERTs only, so a relayed CHAN or TXT packet cannot be traced back to its sender without path resolution.'; + var CLASSES = [ + { key: 'flood', label: 'Flood' }, + { key: 'zero_hop', label: 'Zero-hop' }, + { key: 'mixed', label: 'Mixed' }, + { key: 'unknown', label: 'Unknown' } + ]; + var TYPE_ICONS = { 4: 'ph-broadcast', 5: 'ph-chat-circle', 2: 'ph-envelope' }; + var TYPE_LABELS = { 4: 'Advert', 5: 'Channel', 2: 'DM' }; + + function esc(s) { + return String(s == null ? '' : s) + .replace(/&/g, '&').replace(//g, '>') + .replace(/"/g, '"').replace(/'/g, '''); + } + + function truncate(s, len) { + s = String(s || ''); + return s.length > len ? s.slice(0, len) + '…' : s; + } + + function icon(name) { + return ''; + } + + function classLabel(key) { + for (var i = 0; i < CLASSES.length; i++) { + if (CLASSES[i].key === key) return CLASSES[i].label; + } + return 'Unknown'; + } + + function isTabKey(key) { + return key === 'all' || CLASSES.some(function (c) { return c.key === key; }); + } + + function splitHash(hash) { + var s = String(hash || ''); + var i = s.indexOf('?'); + return { path: i < 0 ? s : s.slice(0, i), params: i < 0 || i === s.length - 1 ? [] : s.slice(i + 1).split('&') }; + } + + function paramName(pair) { + var eq = pair.indexOf('='); + try { return decodeURIComponent(eq < 0 ? pair : pair.slice(0, eq)); } catch (e) { return ''; } + } + + /** The tab named by ?adverts= in a location hash; 'all' when absent or invalid. */ + function parseTab(hash) { + var params = splitHash(hash).params; + for (var i = 0; i < params.length; i++) { + if (paramName(params[i]) !== PARAM) continue; + var eq = params[i].indexOf('='); + var value = ''; + try { value = decodeURIComponent(eq < 0 ? '' : params[i].slice(eq + 1)); } catch (e) { value = ''; } + return isTabKey(value) ? value : 'all'; + } + return 'all'; + } + + /** hash with ?adverts= set to tab (dropped for 'all'); path and other params kept. */ + function hashWithTab(hash, tab) { + var parts = splitHash(hash); + var out = []; + var placed = false; + for (var i = 0; i < parts.params.length; i++) { + if (paramName(parts.params[i]) !== PARAM) { out.push(parts.params[i]); continue; } + if (!placed && tab !== 'all') out.push(PARAM + '=' + encodeURIComponent(tab)); + placed = true; + } + if (!placed && tab !== 'all') out.push(PARAM + '=' + encodeURIComponent(tab)); + return parts.path + (out.length ? '?' + out.join('&') : ''); + } + + function num(v) { return Number(v) || 0; } + + function countsRow(label, w) { + w = w || {}; + var parts = ['Flood ' + num(w.flood), 'Zero-hop ' + num(w.zero_hop), 'Mixed ' + num(w.mixed)]; + if (num(w.unknown) > 0) parts.push('Unknown ' + num(w.unknown)); + return '' + label + ' ' + parts.join(' · ') + ''; + } + + function countsHtml(counts) { + if (!counts) return ''; + var html = '
' + + countsRow('24h', counts['24h']) + countsRow('7d', counts['7d']); + if (counts.truncated) html += '(newest 50,000 adverts only)'; + return html + '
'; + } + + function noteHtml(counts) { + var st = counts && counts.route_mask_backfill && counts.route_mask_backfill.status; + if (!counts || st === 'complete') return ''; + return '

Route classes are provisional: the route-mask backfill is still running, so older adverts are classified by the route they were first stored with.

'; + } + + function routeBadge(p) { + if (!p.route_class) return ''; + return ' '; + } + + // Hash-size badge for nodes whose adverts disagree on it: path_len's upper + // two bits of the stored frame (byte 1), as before. + function hashSizeBadge(p) { + if (p.payload_type !== 4 || !p.raw_hex) return ''; + var pb = parseInt(String(p.raw_hex).slice(2, 4), 16); + if (!(pb & 0x3F)) return ''; + var hs = ((pb >> 6) & 0x3) + 1; + return ' '; + } + + function entryHtml(p, opts, withRoute) { + var decoded = null; + try { decoded = JSON.parse(p.decoded_json); } catch (e) { decoded = null; } + var detail = decoded && decoded.text ? ': ' + esc(truncate(decoded.text, 50)) + : decoded && decoded.name ? ' — ' + esc(decoded.name) : ''; + var type = icon(TYPE_ICONS[p.payload_type] || 'ph-package') + ' ' + (TYPE_LABELS[p.payload_type] || 'Packet'); + var count = num(p.observation_count); + var obsBadge = count > 1 ? ' ' + icon('ph-eye') + ' ' + count + '' : ''; + var obs = p.observer_name || p.observer_id; + var signal = (p.snr != null ? ' · SNR ' + esc(p.snr) + 'dB' : '') + (p.rssi != null ? ' · RSSI ' + esc(p.rssi) + 'dBm' : ''); + var body = type + (withRoute ? routeBadge(p) : '') + detail + (opts.hashSizeInconsistent ? hashSizeBadge(p) : '') + + obsBadge + (obs ? ' via ' + esc(obs) : '') + signal; + var ts = opts.timestampHtml ? opts.timestampHtml(p.timestamp) : esc(p.timestamp); + var link = '' + + ''; + } + return '
' + ts + '' + body + '' + + link + ' style="margin-left:8px;font-size:0.8em">Analyze →
'; + } + + function byTimestampDesc(a, b) { + return (Date.parse(b.timestamp) || 0) - (Date.parse(a.timestamp) || 0); + } + + // Newest first by timestamp, like the node pages always showed the list; + // the server picks the rows by ingest id (#1345). + function listHtml(rows, key, opts) { + var valid = (rows || []).filter(function (p) { return p && p.hash && p.timestamp; }).sort(byTimestampDesc); + var cls = opts.variant === 'pane' ? 'advert-timeline' : 'node-activity-list'; + if (!valid.length) { + var empty = key === 'all' ? 'No recent adverts' : 'No ' + classLabel(key).toLowerCase() + ' adverts from this node on record'; + return '
' + empty + '
'; + } + return '
' + valid.map(function (p) { return entryHtml(p, opts, key === 'all'); }).join('') + '
'; + } + + /** + * HTML for the panel. detail is the /api/nodes/{pubkey} response; + * opts: { variant: 'full'|'pane', tab, idPrefix, timestampHtml(iso), + * hashSizeInconsistent, roleColor }. Without recentAdvertsByRoute (older + * server, hidden identity) only the chronological list is shown. + */ + function render(detail, opts) { + detail = detail || {}; + opts = opts || {}; + var prefix = opts.idPrefix || 'nodeAdverts'; + var all = (detail.recentAdverts || []).filter(function (p) { return p && p.hash && p.timestamp; }); + var byRoute = detail.recentAdvertsByRoute; + var html = '

Recent Adverts (' + all.length + ')

'; + if (!byRoute) return html + listHtml(all, 'all', opts); + + var tabs = [{ key: 'all', label: 'All', rows: all }]; + CLASSES.forEach(function (c) { + var rows = byRoute[c.key] || []; + if (c.key !== 'unknown' || rows.length) tabs.push({ key: c.key, label: c.label, rows: rows }); + }); + var tab = tabs.some(function (t) { return t.key === opts.tab; }) ? opts.tab : 'all'; + + html += countsHtml(detail.advertCounts) + noteHtml(detail.advertCounts); + // initTabBar (bind) adds role="tablist"; it skips a bar that has one. + html += '
' + tabs.map(function (t) { + var on = t.key === tab; + return ''; + }).join('') + '
'; + html += tabs.map(function (t) { + return '
' + listHtml(t.rows, t.key, opts) + '
'; + }).join(''); + return html; + } + + /** Wire the tab bar rendered into el; opts.onTabChange(key) on every switch. */ + function bind(el, opts) { + var bar = el && el.querySelector('.node-adverts-tabs'); + if (!bar || typeof root.initTabBar !== 'function') return; + root.initTabBar(bar, function (btn) { + var btns = bar.querySelectorAll('[data-adverts-tab]'); + for (var i = 0; i < btns.length; i++) { + var on = btns[i] === btn; + btns[i].classList.toggle('active', on); + var p = document.getElementById(btns[i].getAttribute('aria-controls')); + if (p) p.hidden = !on; + } + if (opts && opts.onTabChange) opts.onTabChange(btn.getAttribute('data-adverts-tab')); + }); + } + + root.NodeAdverts = { parseTab: parseTab, hashWithTab: hashWithTab, render: render, bind: bind }; +})(typeof window !== 'undefined' ? window : this); diff --git a/public/nodes.js b/public/nodes.js index 8c9ca7d57..c91de427e 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -3,7 +3,6 @@ (function () { let nodes = []; - const PAYLOAD_TYPES = {0:'Request',1:'Response',2:'Direct Msg',3:'ACK',4:'Advert',5:'Channel Msg',7:'Anon Req',8:'Path',9:'Trace'}; function syncClaimedToFavorites() { const myNodes = JSON.parse(localStorage.getItem('meshcore-my-nodes') || '[]'); @@ -684,10 +683,13 @@ * Fetch node detail + health data in parallel. * Both selectNode() and loadFullNode() need the same data — * this shared helper avoids duplicating the fetch logic (fixes #391). + * It is the only caller that asks for the Recent Adverts route breakdown + * (#2073, the advertRoutes opt-in, docs/api-spec.md); other node-detail + * fetches stay on the plain URL, which the server answers as before. */ async function fetchNodeDetail(pubkey) { const [nodeData, healthData] = await Promise.all([ - api('/nodes/' + encodeURIComponent(pubkey), { ttl: CLIENT_TTL.nodeDetail }), + api('/nodes/' + encodeURIComponent(pubkey) + '?include=advertRoutes', { ttl: CLIENT_TTL.nodeDetail }), api('/nodes/' + encodeURIComponent(pubkey) + '/health', { ttl: CLIENT_TTL.nodeDetail }).catch(() => null) ]); nodeData.healthData = healthData; @@ -833,36 +835,10 @@
- ${(() => { const validPackets = adverts.filter(p => p.hash && p.timestamp); return ` -

Recent Packets (${validPackets.length})

-
- ${validPackets.length ? validPackets.map(p => { - let decoded; try { decoded = JSON.parse(p.decoded_json); } catch {} - const typeLabel = p.payload_type === 4 ? ' Advert' : p.payload_type === 5 ? ' Channel' : p.payload_type === 2 ? ' DM' : ' Packet'; - const detail = decoded?.text ? ': ' + escapeHtml(truncate(decoded.text, 50)) : decoded?.name ? ' — ' + escapeHtml(decoded.name) : ''; - const obs = p.observer_name || p.observer_id; - const snr = p.snr != null ? ` · SNR ${p.snr}dB` : ''; - const rssi = p.rssi != null ? ` · RSSI ${p.rssi}dBm` : ''; - const obsBadge = p.observation_count > 1 ? ` ${p.observation_count}` : ''; - // Show hash size per advert if inconsistent - let hashSizeBadge = ''; - if (n.hash_size_inconsistent && p.payload_type === 4 && p.raw_hex) { - const pb = parseInt(p.raw_hex.slice(2, 4), 16); - if ((pb & 0x3F) !== 0) { - const hs = ((pb >> 6) & 0x3) + 1; - const hsColor = hs >= 3 ? '#16a34a' : hs === 2 ? '#86efac' : '#f97316'; - const hsFg = hs === 2 ? '#064e3b' : '#fff'; - hashSizeBadge = ` ${hs}B`; - } - } - return `
- ${renderNodeTimestampHtml(p.timestamp)} - ${typeLabel}${detail}${hashSizeBadge}${obsBadge}${obs ? ' via ' + escapeHtml(obs) : ''}${snr}${rssi} - Analyze → -
`; - }).join('') : '
No recent packets
'} -
- `; })()} + ${NodeAdverts.render({ recentAdverts: adverts, recentAdvertsByRoute: nodeData.recentAdvertsByRoute, advertCounts: nodeData.advertCounts }, { + variant: 'full', idPrefix: 'nodeFullAdverts', tab: NodeAdverts.parseTab(location.hash), + timestampHtml: renderNodeTimestampHtml, hashSizeInconsistent: !!n.hash_size_inconsistent, + })}
${observers.length ? `
@@ -964,6 +940,8 @@ }); }); + bindNodeAdverts(document.getElementById('node-packets')); + // Deep-link scroll: ?section=node-packets or ?section=node-packets const hashParams = location.hash.split('?')[1] || ''; const urlParams = new URLSearchParams(hashParams); @@ -1827,6 +1805,14 @@ } } + // #2073: the Recent Adverts tab lives in the URL (?adverts=) so it survives a + // reload and can be shared; replaceState, so switching never re-inits. + function bindNodeAdverts(el) { + NodeAdverts.bind(el, { + onTabChange: function (tab) { history.replaceState(null, '', NodeAdverts.hashWithTab(location.hash, tab)); }, + }); + } + function renderDetail(panel, data) { const n = nodeWithHealthActivity(data.node, (data.healthData && data.healthData.stats) || {}, data.recentAdverts || []); const adverts = (data.recentAdverts || []).sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp)); @@ -1884,30 +1870,11 @@
-
- ${(() => { const validPackets = adverts.filter(a => a.hash && a.timestamp); return ` -

Recent Packets (${validPackets.length})

-
- ${validPackets.length ? validPackets.map(a => { - let decoded; - try { decoded = JSON.parse(a.decoded_json); } catch {} - const pType = PAYLOAD_TYPES[a.payload_type] || 'Packet'; - const icon = a.payload_type === 4 ? '' : a.payload_type === 5 ? '' : a.payload_type === 2 ? '' : ''; - const detail = decoded?.text ? ': ' + escapeHtml(truncate(decoded.text, 50)) : decoded?.name ? ' — ' + escapeHtml(decoded.name) : ''; - const obs = a.observer_name || a.observer_id; - return ``; - }).join('') : '
No recent packets
'} -
- `; })()} +
+ ${NodeAdverts.render({ recentAdverts: adverts, recentAdvertsByRoute: data.recentAdvertsByRoute, advertCounts: data.advertCounts }, { + variant: 'pane', idPrefix: 'nodePaneAdverts', tab: NodeAdverts.parseTab(location.hash), + timestampHtml: renderNodeTimestampHtml, roleColor: roleColor, + })}
${observers.length ? `
@@ -1938,6 +1905,7 @@
`; + bindNodeAdverts(document.getElementById('node-pane-adverts')); // Init map -- same real+estimate side-by-side treatment as loadFullNode. if (hasLoc || hasEstLoc) { diff --git a/public/style.css b/public/style.css index 7cf997e50..aeae50c51 100644 --- a/public/style.css +++ b/public/style.css @@ -2844,6 +2844,29 @@ button.ch-item.ch-item-encrypted .ch-badge { filter: grayscale(0.6); } .node-activity-item:last-child { border-bottom: none; } .node-activity-time { color: var(--text-muted); white-space: nowrap; min-width: 70px; font-size: 12px; } +/* #2073 Recent Adverts (node-adverts.js): per-route counts, tabs, route badge. + Colors reuse the status tokens; badge text stays --text for contrast. */ +.node-adverts-counts { display: flex; flex-wrap: wrap; gap: 2px 16px; margin: 0 0 8px; font-size: 12px; color: var(--text-muted); } +.node-adverts-count-row strong { margin-right: 4px; color: var(--text); } +.node-adverts-note { margin: 0 0 8px; font-size: 12px; } +.node-adverts-tabs { display: flex; flex-wrap: wrap; gap: 4px; margin: 0 0 8px; } +.node-adverts-tabs .tab-btn { padding: 4px 10px; } +.node-adverts-empty { font-size: 13px; } +.advert-timeline .node-adverts-empty { padding: 8px; } +.advert-route-badge { + display: inline-flex; align-items: center; gap: 4px; padding: 0 6px; + border: 1px solid var(--border); border-radius: 4px; + font-size: 11px; font-weight: var(--fw-semibold); color: var(--text); white-space: nowrap; +} +.advert-route-badge::before { content: ''; width: 7px; height: 7px; border-radius: 50%; background: var(--text-muted); } +.advert-route-badge[data-route-class="flood"]::before { background: var(--status-info); } +.advert-route-badge[data-route-class="zero_hop"]::before { background: var(--status-green); } +.advert-route-badge[data-route-class="mixed"]::before { background: var(--status-purple); } +/* Per-advert hash size when a node's adverts disagree (was inline colors). */ +.advert-hs-badge { font-size: 9px; font-family: var(--mono); color: var(--text-on-accent); background: var(--palette-green-600); } +.advert-hs-badge.hs-1 { background: var(--status-orange); } +.advert-hs-badge.hs-2 { background: var(--palette-green-300); color: var(--palette-gray-900); } + /* Analytics page */ .analytics-page { padding: 16px 24px; max-width: 1600px; margin: 0 auto; } .analytics-header { margin-bottom: 20px; } diff --git a/test-all.sh b/test-all.sh index 73be5ebdc..884db1880 100755 --- a/test-all.sh +++ b/test-all.sh @@ -31,6 +31,7 @@ node test-repeater-metric-scatter.js node test-top-routes-overlay.js node test-important-links-byte-filter.js node test-url-state.js +node test-node-adverts.js node test-perf-go-runtime.js node test-channel-psk-ux.js node test-channel-sidebar-layout.js diff --git a/test-fixtures/seed-2073-route-adverts.sql b/test-fixtures/seed-2073-route-adverts.sql new file mode 100644 index 000000000..5e3d7a33a --- /dev/null +++ b/test-fixtures/seed-2073-route-adverts.sql @@ -0,0 +1,45 @@ +-- #2073 E2E seed (test-issue-2073-recent-adverts-e2e.js): one repeater whose +-- adverts cover every route class of the node-detail Recent Adverts panel. +-- Applied by CI after the fixture is migrated (route_mask must exist): +-- sqlite3 test-fixtures/e2e-fixture.db < test-fixtures/seed-2073-route-adverts.sql +-- +-- route_mask bit r = raw route r seen for the hash (#89): 1 = transport +-- flood, 2 = flood, 4 = direct (zero-hop), 8 = transport direct. +-- Out-of-band negative ids and old observation timestamps keep these rows +-- last in the packets views (same approach as the #1486 / #1791 seeds); +-- first_seen is relative to now so the 24h / 7d windows are stable. +-- +-- Expected on /api/nodes/2073e2e0...01: +-- Flood : e2e2073flood0001 (route 1), e2e2073flood0002 (route 0), +-- e2e2073legacy001 (NULL mask, route_type 1 fallback) +-- Zero-hop : e2e2073zerohop01, e2e2073zerohop02 +-- Mixed : e2e2073mixed0001 (first stored as zero-hop), +-- e2e2073mixed0002 (first stored as flood) +-- 24h: flood 2, zero_hop 1, mixed 1 7d: flood 3, zero_hop 2, mixed 2 +-- and /api/nodes/2073e2e0...02 ("Zero Hop Only E2E"): one zero-hop advert, +-- so its Flood and Mixed tabs show the empty state. +INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES + ('2073e2e000000000000000000000000000000000000000000000000000000001', 'Route Mix E2E', 'repeater', + strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-1 hours'), strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-3 days'), 7), + ('2073e2e000000000000000000000000000000000000000000000000000000002', 'Zero Hop Only E2E', 'repeater', + strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-2 hours'), strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-3 days'), 1); + +INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, channel_hash, from_pubkey, route_mask) VALUES + (-2073001, '1200e2e207300001', 'e2e2073zerohop01', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-1 hours'), 2, 4, 0, '{"type":"ADVERT","name":"Route Mix E2E","pubKey":"2073e2e000000000000000000000000000000000000000000000000000000001"}', NULL, '2073e2e000000000000000000000000000000000000000000000000000000001', 4), + (-2073002, '1100e2e207300002', 'e2e2073flood0001', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-2 hours'), 1, 4, 0, '{"type":"ADVERT","name":"Route Mix E2E","pubKey":"2073e2e000000000000000000000000000000000000000000000000000000001"}', NULL, '2073e2e000000000000000000000000000000000000000000000000000000001', 2), + (-2073003, '1000000000e2e207300003', 'e2e2073flood0002', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-3 hours'), 0, 4, 0, '{"type":"ADVERT","name":"Route Mix E2E","pubKey":"2073e2e000000000000000000000000000000000000000000000000000000001"}', NULL, '2073e2e000000000000000000000000000000000000000000000000000000001', 1), + (-2073004, '1200e2e207300004', 'e2e2073mixed0001', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-4 hours'), 2, 4, 0, '{"type":"ADVERT","name":"Route Mix E2E","pubKey":"2073e2e000000000000000000000000000000000000000000000000000000001"}', NULL, '2073e2e000000000000000000000000000000000000000000000000000000001', 6), + (-2073005, '1200e2e207300005', 'e2e2073zerohop02', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-30 hours'), 2, 4, 0, '{"type":"ADVERT","name":"Route Mix E2E","pubKey":"2073e2e000000000000000000000000000000000000000000000000000000001"}', NULL, '2073e2e000000000000000000000000000000000000000000000000000000001', 4), + (-2073006, '1100e2e207300006', 'e2e2073legacy001', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-50 hours'), 1, 4, 0, '{"type":"ADVERT","name":"Route Mix E2E","pubKey":"2073e2e000000000000000000000000000000000000000000000000000000001"}', NULL, '2073e2e000000000000000000000000000000000000000000000000000000001', NULL), + (-2073007, '1100e2e207300007', 'e2e2073mixed0002', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-60 hours'), 1, 4, 0, '{"type":"ADVERT","name":"Route Mix E2E","pubKey":"2073e2e000000000000000000000000000000000000000000000000000000001"}', NULL, '2073e2e000000000000000000000000000000000000000000000000000000001', 6), + (-2073008, '1200e2e207300008', 'e2e2073zhonly001', strftime('%Y-%m-%dT%H:%M:%SZ', 'now', '-2 hours'), 2, 4, 0, '{"type":"ADVERT","name":"Zero Hop Only E2E","pubKey":"2073e2e000000000000000000000000000000000000000000000000000000002"}', NULL, '2073e2e000000000000000000000000000000000000000000000000000000002', 4); + +INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) VALUES + (-2073001, 1, 'rx', 6.5, -92, 0, '[]', CAST(strftime('%s', '2026-05-15T00:00:00Z') AS INTEGER)), + (-2073002, 1, 'rx', 5.0, -97, 0, '[]', CAST(strftime('%s', '2026-05-15T00:00:00Z') AS INTEGER)), + (-2073003, 1, 'rx', 4.0, -99, 0, '[]', CAST(strftime('%s', '2026-05-15T00:00:00Z') AS INTEGER)), + (-2073004, 1, 'rx', 3.5, -101, 0, '[]', CAST(strftime('%s', '2026-05-15T00:00:00Z') AS INTEGER)), + (-2073005, 1, 'rx', 6.0, -93, 0, '[]', CAST(strftime('%s', '2026-05-15T00:00:00Z') AS INTEGER)), + (-2073006, 1, 'rx', 2.0, -104, 0, '[]', CAST(strftime('%s', '2026-05-15T00:00:00Z') AS INTEGER)), + (-2073007, 1, 'rx', 1.5, -106, 0, '[]', CAST(strftime('%s', '2026-05-15T00:00:00Z') AS INTEGER)), + (-2073008, 1, 'rx', 5.5, -95, 0, '[]', CAST(strftime('%s', '2026-05-15T00:00:00Z') AS INTEGER)); diff --git a/test-issue-1147-section-order-e2e.js b/test-issue-1147-section-order-e2e.js index 732133872..57ab87a8a 100644 --- a/test-issue-1147-section-order-e2e.js +++ b/test-issue-1147-section-order-e2e.js @@ -1,13 +1,14 @@ /** - * #1147 — Side panel + full node detail page must render "Recent Packets" + * #1147 — Side panel + full node detail page must render "Recent Adverts" + * (named "Recent Packets" until upstream `Kpa-clawbot/CoreScope#2071`) * directly under Overview, BEFORE Paths/Neighbors/Heard By/Clock Skew. * * Operator mental order: identity → packets they originated → paths they - * relay → adverts → meta. Recent Packets currently appears LAST; this is + * relay → adverts → meta. Recent Adverts currently appears LAST; this is * the regression guard that proves the new ordering. * * Acceptance: - * - Full node detail page (#/nodes/): index of "Recent Packets" + * - Full node detail page (#/nodes/): index of "Recent Adverts" * section header < index of "Paths Through This Node" header AND * < index of "Heard By" header AND < index of "Neighbors" header * AND < index of "Clock Skew" header (when present). @@ -61,7 +62,7 @@ function indexOfStarts(headers, prefix) { const pubkey = await page.evaluate(async () => { const r = await fetch('/api/nodes?limit=20'); const d = await r.json(); - // Prefer a node with an advert_count > 0 so Recent Packets has content, + // Prefer a node with an advert_count > 0 so Recent Adverts has content, // but ANY node should render the section header (with an empty state). const cand = (d.nodes || []).find(n => (n.advert_count || 0) > 0) || (d.nodes || [])[0]; @@ -71,42 +72,42 @@ function indexOfStarts(headers, prefix) { console.log(' → using probe pubkey: ' + pubkey.slice(0, 12) + '…'); // ─── Case 1: Full node detail page ─── - await step('full page: Recent Packets appears before Paths/Heard By/Neighbors/Clock Skew', async () => { + await step('full page: Recent Adverts appears before Paths/Heard By/Neighbors/Clock Skew', async () => { await page.goto(BASE + '/#/nodes/' + encodeURIComponent(pubkey), { waitUntil: 'domcontentloaded' }); // Wait for the body container to render (the full detail uses .node-full-card). await page.waitForSelector('.node-full-card', { timeout: 10000 }); - // Wait until at least one "Recent Packets" header is in the DOM. + // Wait until at least one "Recent Adverts" header is in the DOM. await page.waitForFunction(() => { return Array.from(document.querySelectorAll('h4')) - .some(h => (h.textContent || '').trim().startsWith('Recent Packets')); + .some(h => (h.textContent || '').trim().startsWith('Recent Adverts')); }, { timeout: 10000 }); const headers = await sectionHeaders(page, 'body'); console.log(' headers (full page): ' + JSON.stringify(headers)); - const iRecent = indexOfStarts(headers, 'Recent Packets'); + const iRecent = indexOfStarts(headers, 'Recent Adverts'); const iPaths = indexOfStarts(headers, 'Paths Through This Node'); const iHeard = indexOfStarts(headers, 'Heard By'); const iNeigh = indexOfStarts(headers, 'Neighbors'); // Clock Skew is hidden by default but rendered later; only enforce ordering // when its container has visible content. Skip if absent. - assert(iRecent !== -1, 'Recent Packets header not found on full page'); + assert(iRecent !== -1, 'Recent Adverts header not found on full page'); assert(iPaths !== -1, 'Paths Through This Node header not found on full page'); assert(iRecent < iPaths, - `Recent Packets (idx ${iRecent}) must appear BEFORE Paths Through This Node (idx ${iPaths}) on full page`); + `Recent Adverts (idx ${iRecent}) must appear BEFORE Paths Through This Node (idx ${iPaths}) on full page`); if (iHeard !== -1) { assert(iRecent < iHeard, - `Recent Packets (idx ${iRecent}) must appear BEFORE Heard By (idx ${iHeard}) on full page`); + `Recent Adverts (idx ${iRecent}) must appear BEFORE Heard By (idx ${iHeard}) on full page`); } if (iNeigh !== -1) { assert(iRecent < iNeigh, - `Recent Packets (idx ${iRecent}) must appear BEFORE Neighbors (idx ${iNeigh}) on full page`); + `Recent Adverts (idx ${iRecent}) must appear BEFORE Neighbors (idx ${iNeigh}) on full page`); } }); // ─── Case 2: Side panel (opened from /nodes list) ─── - await step('side panel: Recent Packets appears before Paths/Heard By/Neighbors', async () => { + await step('side panel: Recent Adverts appears before Paths/Heard By/Neighbors', async () => { await page.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded' }); await page.waitForSelector('[data-loaded="true"]', { timeout: 15000 }); await page.waitForSelector('table tbody tr:not([id^=vscroll])', { timeout: 15000 }); @@ -126,33 +127,33 @@ function indexOfStarts(headers, prefix) { // Wait for side panel to render the detail container. await page.waitForSelector('.node-detail', { timeout: 10000 }); - // Wait until Recent Packets header lands in the side panel scope. + // Wait until Recent Adverts header lands in the side panel scope. await page.waitForFunction(() => { const root = document.querySelector('.node-detail'); if (!root) return false; return Array.from(root.querySelectorAll('h4')) - .some(h => (h.textContent || '').trim().startsWith('Recent Packets')); + .some(h => (h.textContent || '').trim().startsWith('Recent Adverts')); }, { timeout: 10000 }); const headers = await sectionHeaders(page, '.node-detail'); console.log(' headers (side panel): ' + JSON.stringify(headers)); - const iRecent = indexOfStarts(headers, 'Recent Packets'); + const iRecent = indexOfStarts(headers, 'Recent Adverts'); const iPaths = indexOfStarts(headers, 'Paths Through This Node'); const iHeard = indexOfStarts(headers, 'Heard By'); const iNeigh = indexOfStarts(headers, 'Neighbors'); - assert(iRecent !== -1, 'Recent Packets header not found in side panel'); + assert(iRecent !== -1, 'Recent Adverts header not found in side panel'); assert(iPaths !== -1, 'Paths Through This Node header not found in side panel'); assert(iRecent < iPaths, - `Recent Packets (idx ${iRecent}) must appear BEFORE Paths Through This Node (idx ${iPaths}) in side panel`); + `Recent Adverts (idx ${iRecent}) must appear BEFORE Paths Through This Node (idx ${iPaths}) in side panel`); if (iHeard !== -1) { assert(iRecent < iHeard, - `Recent Packets (idx ${iRecent}) must appear BEFORE Heard By (idx ${iHeard}) in side panel`); + `Recent Adverts (idx ${iRecent}) must appear BEFORE Heard By (idx ${iHeard}) in side panel`); } if (iNeigh !== -1) { assert(iRecent < iNeigh, - `Recent Packets (idx ${iRecent}) must appear BEFORE Neighbors (idx ${iNeigh}) in side panel`); + `Recent Adverts (idx ${iRecent}) must appear BEFORE Neighbors (idx ${iNeigh}) in side panel`); } }); diff --git a/test-issue-1206-resize-observer-leak-e2e.js b/test-issue-1206-resize-observer-leak-e2e.js index 018f9f24c..0d13b8662 100644 --- a/test-issue-1206-resize-observer-leak-e2e.js +++ b/test-issue-1206-resize-observer-leak-e2e.js @@ -156,6 +156,12 @@ async function viewText(page, view) { }, view); } +// The node page's detail request: nodes.js fetchNodeDetail opts in to the +// Recent Adverts route breakdown (#2073), so api() keys, in-flight entries and +// error messages carry the query. Routes below match on the pathname. +const DETAIL_QUERY = '?include=advertRoutes'; +const detailApiPath = (key) => '/nodes/' + key + DETAIL_QUERY; + // Holds the first request for `node` until released and then answers it with // `firstResponse` (an HTTP status for a failure, otherwise a node payload). // Later requests for the node are answered at once with `laterResponse`. @@ -164,7 +170,7 @@ async function holdFirstDetail(page, node, firstResponse, laterResponse) { const gate = new Promise((resolve) => { release = resolve; }); let requests = 0; const path = '/api/nodes/' + node.public_key; - await page.route('**' + path, async (route) => { + await page.route((url) => url.pathname === path, async (route) => { const first = ++requests === 1; if (first) await gate; const response = first ? firstResponse : laterResponse; @@ -183,11 +189,10 @@ async function holdFirstDetail(page, node, firstResponse, laterResponse) { // request for the same node is a separate response and the held one can // arrive last. async function trackHeldDetail(page, node, detach) { - await page.evaluate(([key, detachHeld]) => { - const path = '/nodes/' + key; + await page.evaluate(([path, detachHeld]) => { window.__heldDetail = api(path).then(() => null, (error) => error.message); if (detachHeld) _inflight.delete(path); - }, [node.public_key, detach]); + }, [detailApiPath(node.public_key), detach]); } async function releaseHeldDetail(page, held) { @@ -394,7 +399,7 @@ async function releaseHeldDetail(page, held) { { view: 'side', fails: true }, { view: 'side', fails: false }, ]) { - const expectedError = scenario.fails ? 'API 500: /nodes/' + nodeA.public_key : null; + const expectedError = scenario.fails ? 'API 500: ' + detailApiPath(nodeA.public_key) : null; await step(scenario.view + ' node map: stale ' + (scenario.fails ? 'failure' : 'no-location response') + ' preserves the replacement', () => withNodeMaps(browser, async (page, open) => { const held = await holdFirstDetail(page, nodeA, scenario.fails ? 500 : { ...nodeA, lat: null, lon: null }, nodeA); @@ -455,7 +460,7 @@ async function releaseHeldDetail(page, held) { for (const view of ['side', 'full']) { for (const fails of [false, true]) { - const expectedError = fails ? 'API 500: /nodes/' + nodeA.public_key : null; + const expectedError = fails ? 'API 500: ' + detailApiPath(nodeA.public_key) : null; await step(view + ' node map: A -> B -> A ignores the first A ' + (fails ? 'failure' : 'response') + ' arriving last', () => withNodeMaps(browser, async (page, open) => { const held = await holdFirstDetail(page, nodeA, fails ? 500 : staleA, nodeA); diff --git a/test-issue-2073-recent-adverts-e2e.js b/test-issue-2073-recent-adverts-e2e.js new file mode 100644 index 000000000..55223fedc --- /dev/null +++ b/test-issue-2073-recent-adverts-e2e.js @@ -0,0 +1,240 @@ +/** + * #2073 — Recent Adverts split by route class (port/extension of upstream + * `Kpa-clawbot/CoreScope#2073`, title from `#2071`), on the full node page, + * in the side panel and at phone width. + * + * Needs the fixture seeded with test-fixtures/seed-2073-route-adverts.sql + * (CI applies it after migrating the fixture). Never run against prod. + * + * Usage: BASE_URL=http://localhost:13581 node test-issue-2073-recent-adverts-e2e.js + * SCREENSHOT_DIR= also saves screenshots of each state. + */ +'use strict'; + +const path = require('path'); +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; +const SHOTS = process.env.SCREENSHOT_DIR || ''; +const MIX = '2073e2e000000000000000000000000000000000000000000000000000000001'; +const ZH_ONLY = '2073e2e000000000000000000000000000000000000000000000000000000002'; + +let passed = 0, failed = 0; +async function step(name, fn) { + try { await fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.error(' ✗ ' + name + ': ' + e.message); } +} +function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } +// Screenshots wait out the page fade-in and the .tab-btn color transition. +async function shot(page, name, selector) { + if (!SHOTS) return; + await page.waitForTimeout(800); + if (selector) await page.$eval(selector, (el) => el.scrollIntoView({ block: 'start' })); + await page.screenshot({ path: path.join(SHOTS, '2073-' + name + '.png'), fullPage: false }); +} + +// State of the panel inside root: title, counts text, tabs, visible entries. +async function panelState(page, root) { + return page.$eval(root, (el) => { + const bar = el.querySelector('.node-adverts-tabs'); + const tabs = bar ? Array.from(bar.querySelectorAll('[role="tab"]')) : []; + const visible = Array.from(el.querySelectorAll('.node-adverts-panel')).filter(p => !p.hidden); + const entries = visible.length ? Array.from(visible[0].querySelectorAll('.node-activity-item, .advert-entry')) : []; + const h4 = el.querySelector('h4'); + const active = document.activeElement; + return { + title: h4 ? h4.textContent.trim() : '', + tip: h4 ? h4.getAttribute('title') || '' : '', + counts: (el.querySelector('.node-adverts-counts') || { textContent: '' }).textContent.replace(/\s+/g, ' ').trim(), + note: !!el.querySelector('.node-adverts-note'), + barRole: bar ? bar.getAttribute('role') : null, + tabs: tabs.map(t => ({ key: t.dataset.advertsTab, selected: t.getAttribute('aria-selected'), tabindex: t.getAttribute('tabindex') })), + visiblePanels: visible.map(p => p.id), + entries: entries.length, + badges: visible.length ? Array.from(visible[0].querySelectorAll('.advert-route-badge')).map(b => b.dataset.routeClass) : [], + empty: visible.length ? (visible[0].querySelector('.node-adverts-empty') || { textContent: '' }).textContent.trim() : '', + hrefs: entries.map(e => (e.querySelector('a.ch-analyze-link') || {}).getAttribute ? e.querySelector('a.ch-analyze-link').getAttribute('href') : ''), + focusedTab: active && active.dataset ? active.dataset.advertsTab || null : null, + focusVisible: !!(active && active.matches && active.matches(':focus-visible')), + }; + }); +} +const selectedTab = (s) => (s.tabs.find(t => t.selected === 'true') || {}).key; + +// The first theme load re-renders the node views once (theme-refresh → +// loadFullNode / selectNode, same as on master), replacing the tab bar. Wait +// for that and for the tab bar to stay the same element before interacting. +const THEME_FLAG = () => { window.addEventListener('theme-refresh', () => { window.__2073ThemeReady = true; }, { once: true }); }; +async function waitSettled(page, root) { + const sel = root + ' .node-adverts-tabs[role="tablist"]'; + await page.waitForFunction(() => window.__2073ThemeReady === true); + await page.waitForFunction((s) => { + const el = document.querySelector(s); + if (!el) return false; + if (window.__2073Stable !== el) { window.__2073Stable = el; window.__2073Since = performance.now(); return false; } + return performance.now() - window.__2073Since > 400; + }, sel, { polling: 100 }); +} + +(async () => { + const browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'], + }); + const ctx = await browser.newContext({ viewport: { width: 1440, height: 1000 } }); + await ctx.addInitScript(THEME_FLAG); + const page = await ctx.newPage(); + page.setDefaultTimeout(15000); + const errors = []; + page.on('pageerror', (e) => errors.push(e.message)); + // Node-detail requests (GET /api/nodes/{pubkey}, no sub-path) the node + // page makes: all of them must opt in to the breakdown. + const detailRequests = []; + page.on('request', (req) => { + const u = new URL(req.url()); + if (/^\/api\/nodes\/[0-9a-f]{64}$/.test(u.pathname)) detailRequests.push(u.pathname + u.search); + }); + + console.log(`\n=== #2073 Recent Adverts E2E against ${BASE} ===`); + const full = '#node-packets'; + + await step('full page: title, tooltip, counts and default All tab', async () => { + await page.goto(BASE + '/#/nodes/' + MIX, { waitUntil: 'domcontentloaded' }); + await waitSettled(page, full); + const s = await panelState(page, full); + assert(s.title === 'Recent Adverts (7)', 'title: ' + s.title); + assert(s.tip.includes('from_pubkey'), 'tooltip explains the advert-only list'); + assert(s.counts.includes('24h Flood 2 · Zero-hop 1 · Mixed 1'), 'counts: ' + s.counts); + assert(s.counts.includes('7d Flood 3 · Zero-hop 2 · Mixed 2'), 'counts: ' + s.counts); + assert(!/Unknown/.test(s.counts), 'unknown hidden at zero'); + assert(s.note, 'provisional note while the route_mask backfill is pending'); + assert(s.tabs.map(t => t.key).join() === 'all,flood,zero_hop,mixed', 'tabs: ' + JSON.stringify(s.tabs)); + assert(selectedTab(s) === 'all' && s.entries === 7, 'All selected with 7 adverts'); + assert(s.badges.length === 7 && s.badges.filter(b => b === 'mixed').length === 2, 'route badge per advert: ' + s.badges); + await shot(page, 'full-all-light', full); + }); + + await step('full page: Flood tab lists every flood advert and deep-links', async () => { + await page.click(full + ' [data-adverts-tab="flood"]'); + const s = await panelState(page, full); + assert(selectedTab(s) === 'flood' && s.visiblePanels.join() === 'nodeFullAdverts-panel-flood', 'flood panel shown'); + assert(s.entries === 3, 'flood adverts: ' + s.entries); + assert(s.hrefs.some(h => h.endsWith('e2e2073flood0002')), 'transport-flood advert listed'); + assert(s.hrefs.some(h => h.endsWith('e2e2073legacy001')), 'legacy NULL-mask advert listed via route_type'); + assert(!s.hrefs.some(h => /mixed/.test(h)), 'mixed advert never under Flood'); + assert(new URL(page.url()).hash === '#/nodes/' + MIX + '?adverts=flood', 'hash: ' + new URL(page.url()).hash); + await shot(page, 'full-flood-light', full); + }); + + await step('full page: reload restores the tab from the URL', async () => { + await page.reload({ waitUntil: 'domcontentloaded' }); + await waitSettled(page, full); + const s = await panelState(page, full); + assert(selectedTab(s) === 'flood' && s.entries === 3, 'restored: ' + JSON.stringify(s.tabs)); + }); + + await step('full page: arrow keys move between tabs with visible focus', async () => { + await page.focus(full + ' [data-adverts-tab="flood"]'); + await page.keyboard.press('ArrowRight'); + let s = await panelState(page, full); + assert(selectedTab(s) === 'zero_hop' && s.focusedTab === 'zero_hop' && s.entries === 2, 'ArrowRight → Zero-hop'); + assert(s.focusVisible, 'focused tab matches :focus-visible'); + assert(s.tabs.filter(t => t.tabindex === '0').length === 1, 'roving tabindex'); + await page.keyboard.press('ArrowRight'); + s = await panelState(page, full); + assert(selectedTab(s) === 'mixed' && s.entries === 2, 'ArrowRight → Mixed'); + assert(new URL(page.url()).hash.endsWith('?adverts=mixed'), 'hash follows the keyboard'); + await shot(page, 'full-mixed-light', full); + await page.keyboard.press('Home'); + s = await panelState(page, full); + assert(selectedTab(s) === 'all' && new URL(page.url()).hash === '#/nodes/' + MIX, 'Home → All drops the param'); + }); + + await step('full page: empty state explains itself', async () => { + await page.goto(BASE + '/#/nodes/' + ZH_ONLY + '?adverts=flood', { waitUntil: 'domcontentloaded' }); + await waitSettled(page, full); + const s = await panelState(page, full); + assert(selectedTab(s) === 'flood' && s.entries === 0, 'flood tab empty'); + assert(s.empty === 'No flood adverts from this node on record', 'empty text: ' + s.empty); + await shot(page, 'full-flood-empty-light', full); + }); + + await step('full page: dark theme renders', async () => { + // The app keeps the chosen theme in localStorage (app.js). + await page.evaluate(() => localStorage.setItem('meshcore-theme', 'dark')); + await page.goto(BASE + '/#/nodes/' + MIX + '?adverts=mixed', { waitUntil: 'domcontentloaded' }); + await page.reload({ waitUntil: 'domcontentloaded' }); + await waitSettled(page, full); + assert(await page.evaluate(() => document.documentElement.getAttribute('data-theme')) === 'dark', 'dark theme active'); + const s = await panelState(page, full); + assert(selectedTab(s) === 'mixed' && s.entries === 2, 'mixed in dark'); + await shot(page, 'full-mixed-dark', full); + await page.click(full + ' [data-adverts-tab="all"]'); + await shot(page, 'full-all-dark', full); + await page.evaluate(() => localStorage.setItem('meshcore-theme', 'light')); + }); + + await step('side panel: same component, tabs and deep link', async () => { + await page.goto(BASE + '/#/nodes?search=' + encodeURIComponent('Route Mix E2E'), { waitUntil: 'domcontentloaded' }); + await page.click('tr[data-key="' + MIX + '"]'); + const pane = '#node-pane-adverts'; + await waitSettled(page, pane); + let s = await panelState(page, pane); + assert(s.title === 'Recent Adverts (7)' && s.counts.includes('24h Flood 2 · Zero-hop 1 · Mixed 1'), 'pane title/counts'); + assert(selectedTab(s) === 'all' && s.entries === 7, 'pane All'); + await shot(page, 'pane-all-light'); + await page.click(pane + ' [data-adverts-tab="mixed"]'); + s = await panelState(page, pane); + assert(selectedTab(s) === 'mixed' && s.entries === 2, 'pane Mixed'); + assert(new URL(page.url()).hash === '#/nodes/' + MIX + '?adverts=mixed', 'pane hash: ' + new URL(page.url()).hash); + await shot(page, 'pane-mixed-light'); + }); + + await step('opt-in: the node page asks for the breakdown, the plain URL stays master-shaped', async () => { + assert(detailRequests.length > 0 && detailRequests.every(u => /[?&]include=advertRoutes(&|$)/.test(u)), + 'node page detail requests: ' + detailRequests.join(' ')); + const r = await page.evaluate(async (pk) => { + const plain = await (await fetch('/api/nodes/' + pk)).text(); + const opted = await (await fetch('/api/nodes/' + pk + '?include=advertRoutes')).text(); + return { + plainKeys: Object.keys(JSON.parse(plain)).sort().join(), + plainRouteClass: plain.includes('route_class'), + optedKeys: Object.keys(JSON.parse(opted)).sort().join(), + optedRouteClass: opted.includes('"route_class"'), + }; + }, MIX); + assert(r.plainKeys === 'node,recentAdverts' && !r.plainRouteClass, 'plain: ' + JSON.stringify(r)); + assert(r.optedKeys === 'advertCounts,node,recentAdverts,recentAdvertsByRoute' && r.optedRouteClass, 'opted in: ' + JSON.stringify(r)); + }); + + await step('mobile 390×844: no horizontal overflow, tabs usable', async () => { + const m = await browser.newContext({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true }); + await m.addInitScript(THEME_FLAG); + const mp = await m.newPage(); + mp.on('pageerror', (e) => errors.push(e.message)); + await mp.goto(BASE + '/#/nodes/' + MIX, { waitUntil: 'domcontentloaded' }); + await waitSettled(mp, full); + await mp.$eval(full, (el) => el.scrollIntoView()); + for (const tab of ['all', 'flood', 'zero_hop', 'mixed']) { + await mp.tap(full + ' [data-adverts-tab="' + tab + '"]'); + const o = await mp.evaluate((sel) => { + const card = document.querySelector(sel); + return { doc: document.documentElement.scrollWidth - document.documentElement.clientWidth, card: card.scrollWidth - card.clientWidth }; + }, full); + assert(o.doc <= 0 && o.card <= 0, tab + ': horizontal overflow ' + JSON.stringify(o)); + if (tab === 'all' || tab === 'flood') await shot(mp, 'mobile-' + tab, full); + } + const s = await panelState(mp, full); + assert(selectedTab(s) === 'mixed' && s.entries === 2, 'mobile Mixed'); + await m.close(); + }); + + await step('no page errors', async () => { + assert(errors.length === 0, errors.join(' | ')); + }); + + await browser.close(); + console.log(`\n${passed}/${passed + failed} passed`); + process.exit(failed === 0 ? 0 : 1); +})().catch(e => { console.error(e); process.exit(2); }); diff --git a/test-node-adverts.js b/test-node-adverts.js new file mode 100644 index 000000000..7adf066db --- /dev/null +++ b/test-node-adverts.js @@ -0,0 +1,269 @@ +/* Unit tests for public/node-adverts.js — the Recent Adverts panel shared by + * the node detail page and the side panel (port/extension of upstream + * `Kpa-clawbot/CoreScope#2071` / `#2073`). Loads the real file in a vm context. + * Run: node test-node-adverts.js + */ +'use strict'; +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const src = fs.readFileSync(path.join(__dirname, 'public', 'node-adverts.js'), 'utf8'); +const ctx = { window: {}, console }; +vm.createContext(ctx); +vm.runInContext(src, ctx, { filename: 'node-adverts.js' }); +const NA = ctx.window.NodeAdverts; + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(' ✅ ' + name); } + catch (e) { failed++; console.log(' ❌ ' + name + ': ' + e.message); } +} + +function row(id, cls, extra) { + return Object.assign({ + id: id, hash: 'h' + id, timestamp: '2026-09-25T10:00:0' + (id % 10) + 'Z', first_seen: '2026-09-25T10:00:00Z', + payload_type: 4, route_type: cls === 'zero_hop' ? 2 : 1, route_class: cls, + decoded_json: JSON.stringify({ type: 'ADVERT', name: 'Node ' + id }), observation_count: 1, observations: [], + }, extra || {}); +} +const counts = (h24, d7, extra) => Object.assign({ '24h': h24, '7d': d7, truncated: false, route_mask_backfill: { status: 'complete', remaining: 0 } }, extra || {}); +const c = (flood, zh, mixed, unknown) => ({ flood: flood, zero_hop: zh, mixed: mixed, unknown: unknown || 0 }); + +function detail(over) { + return Object.assign({ + recentAdverts: [row(5, 'zero_hop'), row(4, 'zero_hop'), row(3, 'mixed')], + recentAdvertsByRoute: { limit: 20, flood: [row(1, 'flood'), row(0, 'flood')], zero_hop: [row(5, 'zero_hop'), row(4, 'zero_hop')], mixed: [row(3, 'mixed')] }, + advertCounts: counts(c(2, 94, 1), c(9, 600, 3)), + }, over || {}); +} +const opts = (over) => Object.assign({ variant: 'full', idPrefix: 'na', timestampHtml: (iso) => '' }, over || {}); +// The markup of one tab panel: from its id to the next panel (or the end). +function panel(html, key) { + const start = html.indexOf('id="na-panel-' + key + '"'); + assert.ok(start >= 0, 'panel ' + key + ' missing'); + const next = html.indexOf('id="na-panel-', start + 1); + return html.slice(start, next < 0 ? html.length : next); +} + +console.log('── node-adverts.js ──'); + +test('parseTab: reads ?adverts= and falls back to all', () => { + assert.strictEqual(NA.parseTab('#/nodes/abc?adverts=flood'), 'flood'); + assert.strictEqual(NA.parseTab('#/nodes/abc?section=node-packets&adverts=zero_hop'), 'zero_hop'); + assert.strictEqual(NA.parseTab('#/nodes/abc?adverts=mixed'), 'mixed'); + assert.strictEqual(NA.parseTab('#/nodes/abc?adverts=unknown'), 'unknown'); + assert.strictEqual(NA.parseTab('#/nodes/abc?adverts=' }), + observer_name: 'obs', + route_class: '">', + }); + const html = NA.render(detail({ recentAdverts: [evil], recentAdvertsByRoute: { limit: 20, flood: [evil], zero_hop: [], mixed: [] } }), opts()); + assert.ok(!html.includes(' { + const r = row(8, 'flood', { observation_count: 3, snr: 7.5, rssi: -91, observer_name: 'ObsA', raw_hex: '11c1' }); + const d = detail({ recentAdverts: [r] }); + const full = panel(NA.render(d, opts({ hashSizeInconsistent: true })), 'all'); + assert.ok(full.includes('badge-obs') && full.includes('SNR 7.5dB') && full.includes('RSSI -91dBm') && full.includes('via ObsA')); + assert.ok(/class="badge advert-hs-badge hs-\d"/.test(full), 'hash-size badge kept on the full page'); + assert.ok(full.includes('node-activity-item')); + const pane = NA.render(d, opts({ variant: 'pane', roleColor: 'var(--role-repeater)' })); + assert.ok(pane.includes('advert-entry') && pane.includes('advert-dot') && pane.includes('badge-obs')); +}); + +// PR #97 review P3: every node-controlled or caller-supplied value that +// reaches the markup is escaped - one test per value, so dropping a single +// esc() fails it. +function evilRender(extra, o) { + const r = row(9, 'flood', extra); + return NA.render(detail({ recentAdverts: [r], recentAdvertsByRoute: { limit: 20, flood: [r], zero_hop: [], mixed: [] } }), opts(o)); +} + +test('escaping: decoded text', () => { + const html = evilRender({ decoded_json: JSON.stringify({ text: '' }) }); + assert.ok(!html.includes(' { + const html = evilRender({ snr: '1', rssi: '2' }); + assert.ok(!html.includes(''), 'snr reached the HTML raw'); + assert.ok(!html.includes(''), 'rssi reached the HTML raw'); + assert.ok(html.includes('SNR <b id=snr>1</b>dB') && html.includes('RSSI <b id=rssi>2</b>dBm'), html); +}); + +test('escaping: role colour in the pane dot', () => { + const html = evilRender({}, { variant: 'pane', roleColor: 'red">' }); + assert.ok(!html.includes('