From 491ca2936afa07f9c278c742f0b1625b11199c82 Mon Sep 17 00:00:00 2001 From: dborup Date: Sat, 3 Oct 2026 15:52:05 +0200 Subject: [PATCH 1/2] feat(nodes): make neighbor estimates conservative and evidence-aware --- .github/workflows/deploy.yml | 1 + cmd/server/db.go | 180 ++++---------- cmd/server/db_test.go | 10 +- cmd/server/neighbor_position_estimate.go | 217 +++++++++++++++++ cmd/server/neighbor_position_estimate_test.go | 230 ++++++++++++++++++ cmd/server/openapi.go | 22 +- cmd/server/ping_score_bulk.go | 94 ++----- cmd/server/ping_score_bulk_test.go | 15 +- cmd/server/routes.go | 30 +-- cmd/server/routes_test.go | 20 +- docs/api-spec.md | 47 +++- docs/user-guide/nodes.md | 80 ++++++ public/nodes.js | 115 ++++++--- test-all.sh | 1 + test-neighbor-estimate-e2e.js | 178 ++++++++++++++ test-neighbor-estimate.js | 95 ++++++++ 16 files changed, 1070 insertions(+), 265 deletions(-) create mode 100644 cmd/server/neighbor_position_estimate.go create mode 100644 cmd/server/neighbor_position_estimate_test.go create mode 100644 test-neighbor-estimate-e2e.js create mode 100644 test-neighbor-estimate.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 495d06c5f..0e6ea96e6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -390,6 +390,7 @@ jobs: - name: Run Playwright E2E tests (fail-fast) run: | node test-node-liveness-e2e.js 2>&1 | tee node-liveness-e2e-output.txt + node test-neighbor-estimate-e2e.js 2>&1 | tee neighbor-estimate-e2e-output.txt BASE_URL=http://localhost:13581 node test-e2e-playwright.js 2>&1 | tee e2e-output.txt # M5+M6 of #1668 — axe-core CI gate. # M5: color-contrast on desktop dark+light. diff --git a/cmd/server/db.go b/cmd/server/db.go index 8401081dc..0724a46d2 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2868,68 +2868,30 @@ func (db *DB) GetPacketPath(hash string, maxEdgeKm float64) (*PacketPathResponse return buildPacketPathResponseFromReduction(hash, red.txID, red.first, red.best, nodeByPK, nodeByName, neighborLookup), nil } -// EstimateMaxEdgeKm is the geo-sanity threshold nearestPositionedNeighbor's -// callers pass -- deliberately much tighter than Config.NeighborMaxEdgeKm() -// (500km, default), which governs a different, coarser question (see -// nearestPositionedNeighbor's doc comment for the real-world case that -// motivated this). Genuine LoRa RF adjacency rarely exceeds a few tens of -// km, so 30km errs toward excluding a real-but-unusually-long neighbor -// link over including an MQTT-bridge artifact that happens to have -// accumulated a large edge count. +// EstimateMaxEdgeKm is an initial neighborhood-selection heuristic, distinct +// from Config.NeighborMaxEdgeKm's general graph filter. It is neither an RF +// range guarantee nor an error radius; selected neighbors may span 2x this. const EstimateMaxEdgeKm = 30.0 -// nearestPositionedNeighbor estimates pubkey's position from its -// neighbor_edges neighbors that themselves have a real, known position, -// for use as an approximate stand-in when pubkey has none of its own -- -// e.g. a node that's never advertised a GPS fix, but is almost -// certainly physically near wherever its neighbors are. Weighted by -// each edge's observation count (a neighbor seen relaying to/from -// pubkey many times pulls the estimate harder than one seen once), so -// with several positioned neighbors this settles somewhere among them -// rather than collapsing onto a single neighbor's exact coordinates -- -// each neighbor's OWN position is a real, precise fix; only pubkey's -// position relative to them is unknown, so more of them narrows it -// down. With exactly one positioned neighbor this is identical to -// using that neighbor's position outright. -// -// contributorCount is how many positioned neighbors fed the estimate, -// and spreadKm is the widest distance between any two of them (0 when -// there's only one) -- together a rough confidence signal callers can -// use to size an "uncertainty" marker: more contributors that broadly -// agree (small spread) means a tighter estimate than a single neighbor -// or several that disagree (large spread). -// -// maxEdgeKm geo-sanity-filters the candidate pool before averaging: -// neighbor_edges isn't purely RF adjacency -- it also carries -// observer↔last-hop edges (cmd/ingestor/neighbor_builder.go), and an -// MQTT-bridged remote observer can be genuinely hundreds of km from a -// repeater it merely "heard" over the bridge, not next to it. Anchored -// on the single highest-weight (most-observed) candidate as the most -// trustworthy data point, any other candidate further than maxEdgeKm -// from it is dropped before the centroid/spread math runs -- otherwise -// one rare distant outlier both skews the estimate and inflates -// spreadKm into something like "800km", which reads as "this whole -// estimate is garbage" when in practice the real local neighbors -// dominate by weight. -// -// Callers pass EstimateMaxEdgeKm, NOT Config.NeighborMaxEdgeKm() (500km) -- -// that default is tuned for deciding which edges are implausible enough to -// exclude from the general-purpose NeighborGraph entirely, a much coarser -// question than "is this specific candidate close enough to trust for a -// single-point position estimate". A real case (dborup, #Bornholm test -// repeater) showed why 500km is far too loose here: the correct anchor -// (DK_Bornholm_Olsker, on the island) had a Swedish MQTT-bridge neighbor -// 177km away with a MASSIVE accumulated count (6403, vs the anchor's -// 11917) -- big enough on its own to drag the weighted centroid out into -// the sea between Bornholm and Sweden. Genuine LoRa RF range rarely -// exceeds a few tens of km even with favorable terrain, so a much tighter -// cap catches this class of bug. maxEdgeKm <= 0 disables the filter. -// -// Returns ok=false when pubkey has no neighbor with a position at all. +// nearestPositionedNeighbor wraps the shared neighbor-position estimator +// for legacy approximate path/analytics consumers. A clear singleton keeps +// its historical path proxy, but an ambiguous result never does. Node detail +// uses neighborPositionEstimate directly and requires status "estimated". +// spreadKm measures neighbor separation, not uncertainty in the target. +// maxEdgeKm<=0 disables geographic filtering, not bounded count/freshness +// weighting. Candidate SQL remains top-20 lifetime counts before GPS filtering: +// many unpositioned or stale high-count neighbors can still mask better ones. func (db *DB) nearestPositionedNeighbor(pubkey string, maxEdgeKm float64) (name string, lat, lon float64, contributorCount int, spreadKm float64, ok bool) { + r := db.neighborPositionEstimate(pubkey, maxEdgeKm, time.Now()) + e := r.Legacy + return e.Name, e.Lat, e.Lon, e.ContributorCount, e.SpreadKm, r.LegacyOK +} + +func (db *DB) neighborPositionEstimate(pubkey string, maxEdgeKm float64, now time.Time) neighborPositionResult { + unavailable := estimateNeighborPosition(nil, maxEdgeKm, now) pk := strings.ToLower(strings.TrimSpace(pubkey)) if pk == "" { - return "", 0, 0, 0, 0, false + return unavailable } // Secondary sort `neighbor ASC` is a deterministic tie-break for // candidates with exactly equal count -- previously undefined (whichever @@ -2939,29 +2901,29 @@ func (db *DB) nearestPositionedNeighbor(pubkey string, maxEdgeKm float64) (name // a tie. This determinizes it; it does not preserve any order that was // ever guaranteed before. Both queries must stay in lockstep here. rows, err := db.conn.Query(` - SELECT CASE WHEN node_a = ? THEN node_b ELSE node_a END AS neighbor, count + SELECT CASE WHEN node_a = ? THEN node_b ELSE node_a END AS neighbor, count, last_seen FROM neighbor_edges WHERE node_a = ? OR node_b = ? ORDER BY count DESC, neighbor ASC LIMIT 20`, pk, pk, pk) if err != nil { - return "", 0, 0, 0, 0, false - } - type candidate struct { - pubkey string - weight float64 + return unavailable } - var candidates []candidate + var candidates []neighborPositionCandidate for rows.Next() { var neighborPK string var count float64 - if rows.Scan(&neighborPK, &count) == nil { - candidates = append(candidates, candidate{pubkey: neighborPK, weight: count}) + var lastSeen sql.NullString + if rows.Scan(&neighborPK, &count, &lastSeen) != nil { + rows.Close() + return unavailable } + candidates = append(candidates, neighborPositionCandidate{Pubkey: neighborPK, Count: count, LastSeen: parseTimestamp(lastSeen.String)}) } + iterationErr := rows.Err() rows.Close() - if len(candidates) == 0 { - return "", 0, 0, 0, 0, false + if iterationErr != nil || len(candidates) == 0 { + return unavailable } placeholders := make([]byte, 0, len(candidates)*2) @@ -2971,7 +2933,7 @@ func (db *DB) nearestPositionedNeighbor(pubkey string, maxEdgeKm float64) (name placeholders = append(placeholders, ',') } placeholders = append(placeholders, '?') - args[i] = c.pubkey + args[i] = c.Pubkey } type posInfo struct { name string @@ -2979,73 +2941,35 @@ func (db *DB) nearestPositionedNeighbor(pubkey string, maxEdgeKm float64) (name } posByPK := make(map[string]posInfo, len(candidates)) nodeRows, err := db.conn.Query( - "SELECT public_key, name, lat, lon FROM nodes WHERE public_key IN ("+string(placeholders)+") AND lat IS NOT NULL AND lon IS NOT NULL AND lat != 0 AND lon != 0", args...) - if err == nil { - for nodeRows.Next() { - var candPK string - var candName sql.NullString - var candLat, candLon float64 - if nodeRows.Scan(&candPK, &candName, &candLat, &candLon) == nil { - posByPK[candPK] = posInfo{name: candName.String, lat: candLat, lon: candLon} - } + "SELECT public_key, name, lat, lon FROM nodes WHERE public_key IN ("+string(placeholders)+") AND lat IS NOT NULL AND lon IS NOT NULL AND NOT (lat = 0 AND lon = 0)", args...) + if err != nil { + return unavailable + } + for nodeRows.Next() { + var candPK string + var candName sql.NullString + var candLat, candLon float64 + if nodeRows.Scan(&candPK, &candName, &candLat, &candLon) != nil { + nodeRows.Close() + return unavailable } - nodeRows.Close() + posByPK[candPK] = posInfo{name: candName.String, lat: candLat, lon: candLon} } - - // candidates is count-DESC ordered, so the first resolved contributor - // is the strongest (most-observed) -- both the geo-sanity anchor - // below and, for the returned name, the presumed most trustworthy. - type weighted struct { - posInfo - weight float64 + iterationErr = nodeRows.Err() + nodeRows.Close() + if iterationErr != nil { + return unavailable } - var contributors []weighted + var contributors []neighborPositionCandidate for _, c := range candidates { - p, found := posByPK[c.pubkey] + p, found := posByPK[c.Pubkey] if !found { continue } - w := c.weight - if w <= 0 { - w = 1 - } - contributors = append(contributors, weighted{posInfo: p, weight: w}) - } - if len(contributors) == 0 { - return "", 0, 0, 0, 0, false - } - - if maxEdgeKm > 0 && len(contributors) > 1 { - anchor := contributors[0] - filtered := contributors[:1:1] // anchor always survives (distance to itself is 0) - for _, c := range contributors[1:] { - if haversineKm(anchor.lat, anchor.lon, c.lat, c.lon) <= maxEdgeKm { - filtered = append(filtered, c) - } - } - contributors = filtered - } - - var sumLat, sumLon, sumWeight float64 - var strongestName string - for _, c := range contributors { - sumLat += c.lat * c.weight - sumLon += c.lon * c.weight - sumWeight += c.weight - if strongestName == "" { - strongestName = c.name - } - } - var spread float64 - for i := 0; i < len(contributors); i++ { - for j := i + 1; j < len(contributors); j++ { - d := haversineKm(contributors[i].lat, contributors[i].lon, contributors[j].lat, contributors[j].lon) - if d > spread { - spread = d - } - } + c.Name, c.Lat, c.Lon = p.name, p.lat, p.lon + contributors = append(contributors, c) } - return strongestName, sumLat / sumWeight, sumLon / sumWeight, len(contributors), spread, true + return estimateNeighborPosition(contributors, maxEdgeKm, now) } // channelHashIndex is the ingestor's partial index diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 77ea6f88a..b5d7f0c59 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -969,7 +969,7 @@ func TestGetPacketPath_FallsBackToSingleNeighborPosition(t *testing.T) { // TestGetPacketPath_FallsBackToWeightedNeighborCentroid covers a hop // with TWO positioned neighbors of different edge strength: the -// approximate position must be a count-weighted average of both real +// approximate position must be a bounded-count-weighted average of both real // positions -- not just the stronger neighbor's exact coordinates -- // since each neighbor's own GPS is precise even though the hop's // position relative to them isn't. @@ -983,8 +983,8 @@ func TestGetPacketPath_FallsBackToWeightedNeighborCentroid(t *testing.T) { db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES ('pkghost', 'GhostRepeater', 'repeater')`) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon) VALUES ('pkanchor', 'AnchorRepeater', 'repeater', 55.5, 9.5)`) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon) VALUES ('pkweak', 'WeakRepeater', 'repeater', 60.0, 15.0)`) - // pkanchor is a 10x stronger edge than pkweak -- weighted centroid: - // lat = (55.5*10 + 60.0*1) / 11 = 55.90909..., lon = (9.5*10 + 15.0*1) / 11 = 10.0 + // Lifetime counts now receive bounded logarithmic weights, not 10:1 + // physical influence. With filtering disabled both still contribute. db.conn.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count) VALUES ('pkanchor', 'pkghost', 10)`) db.conn.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count) VALUES ('pkghost', 'pkweak', 1)`) @@ -1013,7 +1013,9 @@ func TestGetPacketPath_FallsBackToWeightedNeighborCentroid(t *testing.T) { if p.Lat == nil || p.Lon == nil { t.Fatalf("Lat/Lon = %v/%v, want a computed centroid, not nil", p.Lat, p.Lon) } - const wantLat, wantLon = 55.90909090909091, 10.0 + // Golden values for w(10)=1+log(11)/log(21), w(1)=1+log(2)/log(21). + // Unknown freshness scales both equally, so cancels in the centroid. + const wantLat, wantLon = 57.33217355999426, 11.739323239992985 const epsilon = 1e-9 if diff := *p.Lat - wantLat; diff > epsilon || diff < -epsilon { t.Errorf("Lat = %v, want weighted centroid %v (not AnchorRepeater's exact 55.5, since WeakRepeater also has a real position)", *p.Lat, wantLat) diff --git a/cmd/server/neighbor_position_estimate.go b/cmd/server/neighbor_position_estimate.go new file mode 100644 index 000000000..ff85b8998 --- /dev/null +++ b/cmd/server/neighbor_position_estimate.go @@ -0,0 +1,217 @@ +package main + +import ( + "math" + "sort" + "time" +) + +// Initial heuristic limits, not calibrated probabilities or RF range bounds. +// Keep the SQL candidate limit and the fixed-size distance matrix in sync. +const neighborPositionCandidateLimit = 20 +const neighborPositionHalfLife = 7 * 24 * time.Hour +const neighborPositionAmbiguityRatio = 0.8 +const neighborPositionMinimumRelativeWeight = 0.1 + +// NeighborPositionEstimate describes evidence sufficiency, not positional +// accuracy. SpreadKm is the maximum separation of contributing neighbors; +// it is NOT an error radius. Persisted edges have no source/prefix confidence. +type NeighborPositionEstimate struct { + Status string `json:"status"` + Method string `json:"method"` + ContributorCount int `json:"contributor_count"` + CandidateCount int `json:"candidate_count"` + SpreadKm float64 `json:"spread_km"` + Lat *float64 `json:"lat,omitempty"` + Lon *float64 `json:"lon,omitempty"` + NewestSeen string `json:"newest_seen,omitempty"` + OldestSeen string `json:"oldest_seen,omitempty"` + UnknownFreshnessCount int `json:"unknown_freshness_count"` +} + +type neighborPositionCandidate struct { + Pubkey, Name string + Lat, Lon float64 + Count float64 + LastSeen time.Time +} + +type neighborPositionResult struct { + Estimate NeighborPositionEstimate + // A single-neighbor path proxy remains available for legacy approx + // markers, but never as a supported node position or an ambiguity bypass. + Legacy neighborEstimate + LegacyOK bool +} + +func validNeighborPosition(lat, lon float64) bool { + return !math.IsNaN(lat) && !math.IsInf(lat, 0) && !math.IsNaN(lon) && !math.IsInf(lon, 0) && + lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180 && !(lat == 0 && lon == 0) +} + +func neighborPositionWeight(c neighborPositionCandidate, now time.Time) float64 { + count := c.Count + if math.IsNaN(count) || count < 1 { + count = 1 + } + // Traffic is only a weak tie-break: one edge has at most twice the + // support of another equally fresh edge, regardless of lifetime volume. + w := 1 + math.Log1p(math.Min(count, 20))/math.Log(21) + if c.LastSeen.IsZero() || c.LastSeen.After(now.Add(5*time.Minute)) { + return w * 0.25 + } + age := now.Sub(c.LastSeen) + if age < 0 { + age = 0 + } + return w * math.Exp2(-float64(age)/float64(neighborPositionHalfLife)) +} + +// estimateNeighborPosition is shared by single-node and bulk path lookups. +// Inputs are bounded to 20 before distance work. Each candidate seeds a +// radius neighborhood; evaluating all seeds avoids highest-count anchoring. +// The distance matrix and neighborhood support take O(20²) bounded work. +// Two disjoint, similarly supported neighborhoods cause abstention. This +// heuristic does not prove physical adjacency or independently known sources. +// maxEdgeKm<=0 retains the legacy opt-out: all valid candidates contribute. +func estimateNeighborPosition(input []neighborPositionCandidate, maxEdgeKm float64, now time.Time) neighborPositionResult { + r := neighborPositionResult{Estimate: NeighborPositionEstimate{Status: "unavailable", Method: "neighbor_cluster_v1"}} + // SQL already supplies a deterministically ranked top 20. Defensively + // cap callers before copying/sorting as well; this is not an all-node API. + if len(input) > neighborPositionCandidateLimit { + input = input[:neighborPositionCandidateLimit] + } + candidates := append([]neighborPositionCandidate(nil), input...) + for i := range candidates { + if math.IsNaN(candidates[i].Count) || candidates[i].Count < 1 { + candidates[i].Count = 1 + } + } + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].Count != candidates[j].Count { + return candidates[i].Count > candidates[j].Count + } + if candidates[i].Pubkey != candidates[j].Pubkey { + return candidates[i].Pubkey < candidates[j].Pubkey + } + return candidates[i].LastSeen.After(candidates[j].LastSeen) + }) + if len(candidates) > neighborPositionCandidateLimit { + candidates = candidates[:neighborPositionCandidateLimit] + } + valid := candidates[:0] + seen := make(map[string]bool, len(candidates)) + strongestWeight := 0.0 + for _, c := range candidates { + if !seen[c.Pubkey] && validNeighborPosition(c.Lat, c.Lon) && neighborPositionWeight(c, now) > 0 { + valid = append(valid, c) + seen[c.Pubkey] = true + strongestWeight = math.Max(strongestWeight, neighborPositionWeight(c, now)) + } + } + // A negligible stale edge must not turn one meaningful neighbor into a + // supposedly supported pair. This relative evidence floor is a heuristic, + // not an absolute freshness guarantee: equally old evidence remains old, + // and its timestamps remain visible to callers. + candidates = valid[:0] + for _, c := range valid { + if neighborPositionWeight(c, now) >= strongestWeight*neighborPositionMinimumRelativeWeight { + candidates = append(candidates, c) + } + } + n := len(candidates) + r.Estimate.CandidateCount = n + if n == 0 { + return r + } + var distance [neighborPositionCandidateLimit][neighborPositionCandidateLimit]float64 + var weights [neighborPositionCandidateLimit]float64 + var groups [neighborPositionCandidateLimit]uint32 + var support [neighborPositionCandidateLimit]float64 + var size [neighborPositionCandidateLimit]int + for i, c := range candidates { + weights[i] = neighborPositionWeight(c, now) + for j := 0; j < i; j++ { + distance[i][j] = haversineKm(c.Lat, c.Lon, candidates[j].Lat, candidates[j].Lon) + distance[j][i] = distance[i][j] + } + } + best := 0 + for i := range candidates { + for j := range candidates { + if maxEdgeKm <= 0 || distance[i][j] <= maxEdgeKm { + groups[i] |= 1 << j + support[i] += weights[j] + size[i]++ + } + } + if support[i] > support[best] || (support[i] == support[best] && size[i] > size[best]) { + best = i + } + } + if support[best] <= 0 { + return r + } + for i := range candidates { + if groups[i]&groups[best] == 0 && support[i] >= neighborPositionAmbiguityRatio*support[best] { + r.Estimate.Status = "ambiguous" + return r + } + } + var latSum, lonSum, totalWeight, spread float64 + var newest, oldest time.Time + name := "" + // Unwrap longitudes around a selected contributor to avoid averaging + // +179.9 and -179.9 into Greenwich. This remains a heuristic centroid. + referenceLon := candidates[best].Lon + for i, c := range candidates { + if groups[best]&(1< spread { + spread = distance[i][j] + } + } + } + lat := latSum / totalWeight + lon := math.Mod(lonSum/totalWeight+540, 360) - 180 + // Preserve the exact proxy coordinate for legacy one-neighbor callers. + if size[best] == 1 { + lat, lon = candidates[best].Lat, candidates[best].Lon + } + r.Legacy = neighborEstimate{Name: name, Lat: lat, Lon: lon, ContributorCount: size[best], SpreadKm: spread} + r.LegacyOK = true + r.Estimate.ContributorCount = size[best] + r.Estimate.SpreadKm = spread + if !newest.IsZero() { + r.Estimate.NewestSeen = newest.UTC().Format(time.RFC3339) + } + if !oldest.IsZero() { + r.Estimate.OldestSeen = oldest.UTC().Format(time.RFC3339) + } + if size[best] < 2 { + r.Estimate.Status = "insufficient" + return r + } + r.Estimate.Status = "estimated" + r.Estimate.Lat, r.Estimate.Lon = &lat, &lon + return r +} diff --git a/cmd/server/neighbor_position_estimate_test.go b/cmd/server/neighbor_position_estimate_test.go new file mode 100644 index 000000000..af8fe5a57 --- /dev/null +++ b/cmd/server/neighbor_position_estimate_test.go @@ -0,0 +1,230 @@ +package main + +import ( + "encoding/json" + "fmt" + "math" + "net/http/httptest" + "reflect" + "testing" + "time" +) + +// A heavily observed remote edge must not determine the reference location +// when two other positioned neighbors agree with each other. +func TestNeighborPositionEstimate_RemoteTrafficDoesNotAnchor(t *testing.T) { + for _, remoteCount := range []int{100, 10000} { + t.Run(fmt.Sprint(remoteCount), func(t *testing.T) { + db := setupPacketPathCountingDB(t) + defer db.Close() + for _, q := range []string{ + `INSERT INTO nodes VALUES ('local1', 'Local1', 'repeater', 55.00, 9.40)`, + `INSERT INTO nodes VALUES ('local2', 'Local2', 'repeater', 55.05, 9.45)`, + `INSERT INTO nodes VALUES ('remote', 'Remote', 'repeater', 48.00, 11.50)`, + `INSERT INTO neighbor_edges (node_a,node_b,count) VALUES ('target','local1',60), ('target','local2',60)`, + } { + if _, err := db.conn.Exec(q); err != nil { + t.Fatal(err) + } + } + if _, err := db.conn.Exec(`INSERT INTO neighbor_edges (node_a,node_b,count) VALUES ('target','remote',?)`, remoteCount); err != nil { + t.Fatal(err) + } + _, lat, lon, count, _, ok := db.nearestPositionedNeighbor("target", EstimateMaxEdgeKm) + if !ok || count != 2 || haversineKm(lat, lon, 55.025, 9.425) > 1 { + t.Fatalf("remote count %d: got ok=%v, contributors=%d, location=(%v,%v); want coherent local pair", remoteCount, ok, count, lat, lon) + } + }) + } +} + +func TestNeighborPositionEstimate_EvidenceAndFreshness(t *testing.T) { + now := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC) + near := []neighborPositionCandidate{ + {Pubkey: "a", Lat: 55, Lon: 9.4, Count: 60, LastSeen: now}, + {Pubkey: "b", Lat: 55.05, Lon: 9.45, Count: 60, LastSeen: now.Add(-time.Hour)}, + } + t.Run("competing clusters abstain", func(t *testing.T) { + cs := append(append([]neighborPositionCandidate(nil), near...), + neighborPositionCandidate{Pubkey: "c", Lat: 48, Lon: 11.5, Count: 10000, LastSeen: now}, + neighborPositionCandidate{Pubkey: "d", Lat: 48.05, Lon: 11.55, Count: 10000, LastSeen: now}) + r := estimateNeighborPosition(cs, 30, now) + if r.Estimate.Status != "ambiguous" || r.LegacyOK || r.Estimate.Lat != nil { + t.Fatalf("must abstain, got %+v", r) + } + }) + t.Run("old traffic loses to fresh cluster", func(t *testing.T) { + cs := append(append([]neighborPositionCandidate(nil), near...), + neighborPositionCandidate{Pubkey: "c", Lat: 48, Lon: 11.5, Count: 10000, LastSeen: now.Add(-30 * 24 * time.Hour)}, + neighborPositionCandidate{Pubkey: "d", Lat: 48.05, Lon: 11.55, Count: 10000, LastSeen: now.Add(-30 * 24 * time.Hour)}) + r := estimateNeighborPosition(cs, 30, now) + if r.Estimate.Status != "estimated" || r.Estimate.ContributorCount != 2 || *r.Estimate.Lat < 54 { + t.Fatalf("want recent pair, got %+v", r) + } + if r.Estimate.NewestSeen != now.Format(time.RFC3339) || r.Estimate.OldestSeen != now.Add(-time.Hour).Format(time.RFC3339) { + t.Fatalf("freshness bounds: %+v", r.Estimate) + } + }) + t.Run("single is only a legacy proxy", func(t *testing.T) { + r := estimateNeighborPosition(near[:1], 30, now) + if r.Estimate.Status != "insufficient" || r.Estimate.Lat != nil || r.Estimate.ContributorCount != 1 || !r.LegacyOK { + t.Fatalf("got %+v", r) + } + }) + t.Run("duplicate is not a second neighbor", func(t *testing.T) { + r := estimateNeighborPosition([]neighborPositionCandidate{near[0], near[0]}, 30, now) + if r.Estimate.Status != "insufficient" || r.Estimate.ContributorCount != 1 { + t.Fatalf("got %+v", r) + } + }) + t.Run("zero weight is not supporting evidence", func(t *testing.T) { + cs := append([]neighborPositionCandidate(nil), near...) + cs[1].LastSeen = time.Date(1000, 1, 1, 0, 0, 0, 0, time.UTC) + r := estimateNeighborPosition(cs, 30, now) + if r.Estimate.Status != "insufficient" { + t.Fatalf("zero weight must not satisfy min2: %+v", r) + } + }) + t.Run("negligible stale partner is not supporting evidence", func(t *testing.T) { + cs := append([]neighborPositionCandidate(nil), near...) + cs[1].LastSeen = now.Add(-30 * 24 * time.Hour) + r := estimateNeighborPosition(cs, 30, now) + if r.Estimate.Status != "insufficient" || r.Estimate.ContributorCount != 1 { + t.Fatalf("stale partner must not upgrade a singleton: %+v", r) + } + }) + t.Run("input permutation deterministic", func(t *testing.T) { + a := estimateNeighborPosition(near, 30, now) + b := estimateNeighborPosition([]neighborPositionCandidate{near[1], near[0]}, 30, now) + if !reflect.DeepEqual(a, b) { + t.Fatalf("different output: %+v vs %+v", a, b) + } + }) + t.Run("bad and future timestamps unknown", func(t *testing.T) { + cs := append([]neighborPositionCandidate(nil), near...) + cs[0].LastSeen = parseTimestamp("not a timestamp") + cs[1].LastSeen = now.Add(24 * time.Hour) + r := estimateNeighborPosition(cs, 30, now) + if r.Estimate.UnknownFreshnessCount != 2 || r.Estimate.NewestSeen != "" || r.Estimate.Status != "estimated" { + t.Fatalf("got %+v", r.Estimate) + } + }) +} + +func TestNeighborPositionEstimate_CoordinateSafety(t *testing.T) { + now := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC) + for _, c := range []neighborPositionCandidate{ + {Lat: math.NaN(), Lon: 9}, {Lat: 55, Lon: math.Inf(1)}, {Lat: 91, Lon: 9}, {Lat: 55, Lon: 181}, {Lat: 0, Lon: 0}, + } { + if r := estimateNeighborPosition([]neighborPositionCandidate{c}, 30, now); r.Estimate.Status != "unavailable" || r.LegacyOK { + t.Fatalf("invalid coordinate accepted: %+v", r) + } + } + cs := []neighborPositionCandidate{{Pubkey: "a", Lat: 0, Lon: 179.95, Count: math.Inf(1)}, {Pubkey: "b", Lat: 0, Lon: -179.95, Count: math.NaN()}} + r := estimateNeighborPosition(cs, 30, now) + if r.Estimate.Status != "estimated" || math.IsNaN(*r.Estimate.Lat) || math.Abs(*r.Estimate.Lon) < 179 { + t.Fatalf("antimeridian/finite count safety: %+v", r) + } +} + +func BenchmarkNeighborPositionEstimate_MaxCandidates(b *testing.B) { + now := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC) + cs := make([]neighborPositionCandidate, 20) + for i := range cs { + cs[i] = neighborPositionCandidate{Pubkey: fmt.Sprint(i), Lat: 55 + float64(i)*0.002, Lon: 9.4, Count: 10000, LastSeen: now.Add(-time.Duration(i) * time.Hour)} + } + b.ReportAllocs() + for i := 0; i < b.N; i++ { + estimateNeighborPosition(cs, 30, now) + } +} + +func TestNodeDetail_NeighborEstimateAbstention(t *testing.T) { + for _, status := range []string{"insufficient", "ambiguous"} { + t.Run(status, func(t *testing.T) { + srv, router := setupTestServer(t) + for _, q := range []string{ + `INSERT INTO nodes (public_key,name,role) VALUES ('target','Target','repeater')`, + `INSERT INTO nodes (public_key,name,lat,lon) VALUES ('a','A',55,9.4)`, + `INSERT INTO neighbor_edges (node_a,node_b,count) VALUES ('target','a',60)`, + } { + if _, err := srv.db.conn.Exec(q); err != nil { + t.Fatal(err) + } + } + if status == "ambiguous" { + for _, q := range []string{ + `INSERT INTO nodes (public_key,name,lat,lon) VALUES ('b','B',55.01,9.41),('c','C',48,11.5),('d','D',48.01,11.51)`, + `INSERT INTO neighbor_edges (node_a,node_b,count) VALUES ('target','b',60),('target','c',10000),('target','d',10000)`, + } { + if _, err := srv.db.conn.Exec(q); err != nil { + t.Fatal(err) + } + } + } + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", "/api/nodes/target", nil)) + if w.Code != 200 { + t.Fatalf("HTTP %d: %s", w.Code, w.Body.String()) + } + var body struct { + Node map[string]json.RawMessage `json:"node"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + var estimate NeighborPositionEstimate + if err := json.Unmarshal(body.Node["neighbor_estimate"], &estimate); err != nil { + t.Fatal(err) + } + if estimate.Status != status || estimate.Lat != nil || estimate.Lon != nil { + t.Fatalf("unexpected metadata %+v", estimate) + } + for _, key := range []string{"estimated_lat", "estimated_lon", "estimated_distance_km", "estimated_contributor_count"} { + if _, exists := body.Node[key]; exists { + t.Errorf("unsupported estimate leaked %s", key) + } + } + }) + } +} + +func TestNeighborPositionEstimate_TargetPositionDoesNotInfluenceEstimate(t *testing.T) { + db := setupPacketPathCountingDB(t) + defer db.Close() + for _, q := range []string{ + `INSERT INTO nodes VALUES ('target','Target','repeater',48,11.5),('a','A','repeater',55,9.4),('b','B','repeater',55.01,9.41)`, + `INSERT INTO neighbor_edges (node_a,node_b,count,last_seen) VALUES ('target','a',60,'2026-10-03T11:00:00Z'),('target','b',60,'2026-10-02T11:00:00Z')`, + } { + if _, err := db.conn.Exec(q); err != nil { + t.Fatal(err) + } + } + now := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC) + before := db.neighborPositionEstimate("target", 30, now) + if _, err := db.conn.Exec(`UPDATE nodes SET lat=NULL,lon=NULL WHERE public_key='target'`); err != nil { + t.Fatal(err) + } + after := db.neighborPositionEstimate("target", 30, now) + if !reflect.DeepEqual(before, after) || after.Estimate.Status != "estimated" { + t.Fatalf("target GPS affected estimate: %+v vs %+v", before, after) + } + bulk := make(map[string]neighborEstimate) + if err := db.nearestPositionedNeighborsChunk([]string{"target"}, 30, bulk, now); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(after.Legacy, bulk["target"]) { + t.Fatalf("freshness single/bulk mismatch: %+v vs %+v", after.Legacy, bulk["target"]) + } +} + +func TestNeighborPositionEstimate_CandidateBound(t *testing.T) { + cs := make([]neighborPositionCandidate, 100) + for i := range cs { + cs[i] = neighborPositionCandidate{Pubkey: fmt.Sprint(i), Lat: 55 + float64(i)*0.0001, Lon: 9.4, Count: 1} + } + r := estimateNeighborPosition(cs, 30, time.Now()) + if r.Estimate.CandidateCount != 20 || r.Estimate.ContributorCount != 20 { + t.Fatalf("expected bounded20, got %+v", r) + } +} diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 0afe3f571..415279902 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -226,6 +226,25 @@ func openAPIRef(name string) *openAPISchema { return &openAPISchema{Ref: "#/components/schemas/" + name} } +func neighborPositionEstimateSchema() *openAPISchema { + return &openAPISchema{ + Type: "object", + Description: "Node-detail evidence status from a bounded neighbor-cluster heuristic, not triangulation or a calibrated confidence probability. Positions are emitted only for an unambiguous group with at least two contributors. Initial parameters: 30 km seed radius, count capped at 20 with logarithmic weight in [1,2], seven-day age half-life, unknown freshness weight 0.25, minimum effective weight 10% of the strongest candidate, competing disjoint group support ratio 0.8. The relative weight floor is not an absolute freshness guarantee. Candidate selection still uses the top 20 lifetime edge counts before position filtering. Persisted edges have no source-independence or prefix-confidence metadata.", + Properties: map[string]*openAPISchema{ + "status": {Type: "string", Enum: []string{"estimated", "insufficient", "ambiguous", "unavailable"}}, + "method": {Type: "string", Enum: []string{"neighbor_cluster_v1"}}, + "contributor_count": {Type: "integer", Description: "Distinct positioned neighbors contributing to the selected group; zero when no group was selected."}, + "candidate_count": {Type: "integer", Description: "Valid distinct positioned candidates considered, at most 20."}, + "spread_km": {Type: "number", Description: "Maximum distance between selected neighbors, not a positional error radius or uncertainty bound."}, + "lat": {Type: "number", Description: "Estimated latitude, present only when status is estimated."}, + "lon": {Type: "number", Description: "Estimated longitude, present only when status is estimated."}, + "newest_seen": {Type: "string", Description: "Newest known contributor edge timestamp, RFC3339 UTC; omitted when all timestamps are unknown."}, + "oldest_seen": {Type: "string", Description: "Oldest known contributor edge timestamp, RFC3339 UTC; omitted when all timestamps are unknown."}, + "unknown_freshness_count": {Type: "integer", Description: "Selected contributors whose edge timestamp is absent, invalid, or more than five minutes in the future."}, + }, + } +} + // nodeAdvertRouteSchemas documents the #2073 node-detail advert route fields // (port of upstream Kpa-clawbot/CoreScope#2073). func nodeAdvertRouteSchemas() map[string]*openAPISchema { @@ -309,7 +328,8 @@ func componentSchemas() map[string]interface{} { "role": str("Node role (e.g. repeater, room, client, sensor)."), "lat": map[string]interface{}{"type": "number", "nullable": true}, "lon": map[string]interface{}{"type": "number", "nullable": true}, - "estimated_lat": map[string]interface{}{"type": "number", "nullable": true, "description": "Node detail endpoint only: an approximate position from the same neighbor-centroid estimate (geo-sanity-filtered via Config.NeighborMaxEdgeKm) that backs Position-Fix Coverage Gaps, View Path's approx markers, and Suspicious GPS Positions. Present whenever the node has a trustworthy neighbor cluster to estimate from, regardless of whether it also has a real (lat/lon) fix -- lets the detail page show both side by side to visually cross-check a node flagged by Suspicious GPS Positions."}, + "neighbor_estimate": neighborPositionEstimateSchema(), + "estimated_lat": &openAPISchema{Type: "number", Description: "Node detail only: legacy alias of neighbor_estimate.lat, present only when status is estimated (at least two contributors, no similarly supported disjoint competing group). A heuristic cross-check, not a measured position. The 30 km seed radius is not an error bound."}, "estimated_lon": map[string]interface{}{"type": "number", "nullable": true, "description": "Paired with estimated_lat."}, "estimated_contributor_count": map[string]interface{}{"type": "integer", "description": "Number of positioned neighbors the estimated_lat/estimated_lon centroid was averaged from. Present only alongside estimated_lat/estimated_lon."}, "estimated_distance_km": map[string]interface{}{"type": "number", "description": "Distance between the node's own reported lat/lon and estimated_lat/estimated_lon. Present only when the node has BOTH a real fix and an estimate -- absent when either is missing."}, diff --git a/cmd/server/ping_score_bulk.go b/cmd/server/ping_score_bulk.go index 514220b1a..ac448d38f 100644 --- a/cmd/server/ping_score_bulk.go +++ b/cmd/server/ping_score_bulk.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "strings" + "time" ) // This file holds additive, read-only bulk-query helpers for Ping Scores @@ -108,10 +109,11 @@ func (db *DB) observationFingerprintsBulk(txIDs []int64) (map[int64]observationF // previously-undefined ordering; it does not preserve any order that was // ever guaranteed before. // -// A pubkey with no result (no edges, or no positioned contributor within -// maxEdgeKm of the strongest one) is simply absent from the returned map, -// exactly matching nearestPositionedNeighbor's ok=false -- not a -// zero-value entry. +// Candidates feed the same capped-count/freshness cluster estimator as the +// single-item wrapper, using one clock snapshot for the entire batch. A pubkey +// with no result (no valid positioned evidence, or competing groups) is absent +// from the map, matching nearestPositionedNeighbor's ok=false. A clear singleton +// remains a legacy approximate path proxy, not a supported node position. // // The candidate-position lookup inside each chunk (nodes matching the up- // to-20-per-target neighbor pubkeys the ranked query returned) has its own @@ -119,6 +121,7 @@ func (db *DB) observationFingerprintsBulk(txIDs []int64) (map[int64]observationF // single 499-target chunk can produce up to 499*20 = 9980 distinct // candidate pubkeys, far past the 499-bind-parameter budget for one query. func (db *DB) nearestPositionedNeighborsBulk(pubkeys []string, maxEdgeKm float64) (map[string]neighborEstimate, error) { + now := time.Now() result := make(map[string]neighborEstimate, len(pubkeys)) if len(pubkeys) == 0 { return result, nil @@ -143,7 +146,7 @@ func (db *DB) nearestPositionedNeighborsBulk(pubkeys []string, maxEdgeKm float64 if end > len(unique) { end = len(unique) } - if err := db.nearestPositionedNeighborsChunk(unique[i:end], maxEdgeKm, result); err != nil { + if err := db.nearestPositionedNeighborsChunk(unique[i:end], maxEdgeKm, result, now); err != nil { return nil, err } } @@ -152,7 +155,7 @@ func (db *DB) nearestPositionedNeighborsBulk(pubkeys []string, maxEdgeKm float64 // nearestPositionedNeighborsChunk resolves one chunk (<=499 targets) of // nearestPositionedNeighborsBulk, writing results directly into result. -func (db *DB) nearestPositionedNeighborsChunk(targets []string, maxEdgeKm float64, result map[string]neighborEstimate) error { +func (db *DB) nearestPositionedNeighborsChunk(targets []string, maxEdgeKm float64, result map[string]neighborEstimate, now time.Time) error { placeholders := make([]byte, 0, len(targets)*4) args := make([]interface{}, len(targets)) for i, pk := range targets { @@ -167,22 +170,18 @@ func (db *DB) nearestPositionedNeighborsChunk(targets []string, maxEdgeKm float6 edges AS ( SELECT t.pk AS target, CASE WHEN ne.node_a = t.pk THEN ne.node_b ELSE ne.node_a END AS neighbor, - ne.count AS count + ne.count AS count, ne.last_seen AS last_seen FROM neighbor_edges ne JOIN targets t ON (ne.node_a = t.pk OR ne.node_b = t.pk) ), ranked AS ( - SELECT target, neighbor, count, + SELECT target, neighbor, count, last_seen, ROW_NUMBER() OVER (PARTITION BY target ORDER BY count DESC, neighbor ASC) AS rn FROM edges ) - SELECT target, neighbor, count FROM ranked WHERE rn <= 20 ORDER BY target, rn` + SELECT target, neighbor, count, last_seen FROM ranked WHERE rn <= 20 ORDER BY target, rn` - type candidate struct { - pubkey string - weight float64 - } - candidatesByTarget := make(map[string][]candidate, len(targets)) + candidatesByTarget := make(map[string][]neighborPositionCandidate, len(targets)) rows, err := db.conn.Query(query, args...) if err != nil { @@ -193,10 +192,11 @@ func (db *DB) nearestPositionedNeighborsChunk(targets []string, maxEdgeKm float6 for rows.Next() { var target, neighbor string var count float64 - if err := rows.Scan(&target, &neighbor, &count); err != nil { + var lastSeen sql.NullString + if err := rows.Scan(&target, &neighbor, &count, &lastSeen); err != nil { return fmt.Errorf("neighbor estimate bulk scan: %w", err) } - candidatesByTarget[target] = append(candidatesByTarget[target], candidate{pubkey: neighbor, weight: count}) + candidatesByTarget[target] = append(candidatesByTarget[target], neighborPositionCandidate{Pubkey: neighbor, Count: count, LastSeen: parseTimestamp(lastSeen.String)}) } return rows.Err() }() @@ -210,7 +210,7 @@ func (db *DB) nearestPositionedNeighborsChunk(targets []string, maxEdgeKm float6 candidatePubkeySet := make(map[string]bool) for _, cs := range candidatesByTarget { for _, c := range cs { - candidatePubkeySet[c.pubkey] = true + candidatePubkeySet[c.Pubkey] = true } } candidatePubkeys := make([]string, 0, len(candidatePubkeySet)) @@ -245,7 +245,7 @@ func (db *DB) nearestPositionedNeighborsChunk(targets []string, maxEdgeKm float6 nodeArgs[j] = pk } nodeRows, err := db.conn.Query( - "SELECT public_key, name, lat, lon FROM nodes WHERE public_key IN ("+string(nodePlaceholders)+") AND lat IS NOT NULL AND lon IS NOT NULL AND lat != 0 AND lon != 0", nodeArgs...) + "SELECT public_key, name, lat, lon FROM nodes WHERE public_key IN ("+string(nodePlaceholders)+") AND lat IS NOT NULL AND lon IS NOT NULL AND NOT (lat = 0 AND lon = 0)", nodeArgs...) if err != nil { return fmt.Errorf("neighbor estimate bulk node query: %w", err) } @@ -267,64 +267,18 @@ func (db *DB) nearestPositionedNeighborsChunk(targets []string, maxEdgeKm float6 } } - type weighted struct { - posInfo - weight float64 - } for target, candidates := range candidatesByTarget { - var contributors []weighted + var contributors []neighborPositionCandidate for _, c := range candidates { - p, found := posByPK[c.pubkey] + p, found := posByPK[c.Pubkey] if !found { continue } - w := c.weight - if w <= 0 { - w = 1 - } - contributors = append(contributors, weighted{posInfo: p, weight: w}) - } - if len(contributors) == 0 { - continue - } - - // candidates is count-DESC ordered (via the ORDER BY target, rn - // clause above, mirroring rn's PARTITION BY target ORDER BY count - // DESC), so the first resolved contributor is the strongest -- - // both the geo-sanity anchor below and the returned name. - if maxEdgeKm > 0 && len(contributors) > 1 { - anchor := contributors[0] - filtered := contributors[:1:1] // anchor always survives (distance to itself is 0) - for _, c := range contributors[1:] { - if haversineKm(anchor.lat, anchor.lon, c.lat, c.lon) <= maxEdgeKm { - filtered = append(filtered, c) - } - } - contributors = filtered - } - - var sumLat, sumLon, sumWeight float64 - var strongestName string - for _, c := range contributors { - sumLat += c.lat * c.weight - sumLon += c.lon * c.weight - sumWeight += c.weight - if strongestName == "" { - strongestName = c.name - } - } - var spread float64 - for i := 0; i < len(contributors); i++ { - for j := i + 1; j < len(contributors); j++ { - d := haversineKm(contributors[i].lat, contributors[i].lon, contributors[j].lat, contributors[j].lon) - if d > spread { - spread = d - } - } + c.Name, c.Lat, c.Lon = p.name, p.lat, p.lon + contributors = append(contributors, c) } - result[target] = neighborEstimate{ - Name: strongestName, Lat: sumLat / sumWeight, Lon: sumLon / sumWeight, - ContributorCount: len(contributors), SpreadKm: spread, + if estimate := estimateNeighborPosition(contributors, maxEdgeKm, now); estimate.LegacyOK { + result[target] = estimate.Legacy } } return nil diff --git a/cmd/server/ping_score_bulk_test.go b/cmd/server/ping_score_bulk_test.go index cea572bb7..bc0113661 100644 --- a/cmd/server/ping_score_bulk_test.go +++ b/cmd/server/ping_score_bulk_test.go @@ -579,8 +579,8 @@ func TestNearestPositionedNeighborsBulk_MatchesSingleItem(t *testing.T) { db.conn.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count) VALUES ('targeta', 'nba1', 9)`) db.conn.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count) VALUES ('nba2', 'targeta', 4)`) - // Target B: strongest neighbor close by, a second neighbor far enough - // away that a tight maxEdgeKm excludes it. + // Target B: two distant singleton neighborhoods. Under bounded weights + // their support is similar enough to abstain, not anchor on count 8. db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon) VALUES ('nbb1', 'NbB1', 'repeater', 40.0, 10.0)`) db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon) VALUES ('nbb2', 'NbB2', 'repeater', 60.0, 30.0)`) db.conn.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count) VALUES ('targetb', 'nbb1', 8)`) @@ -595,7 +595,7 @@ func TestNearestPositionedNeighborsBulk_MatchesSingleItem(t *testing.T) { t.Fatal(err) } - for _, pk := range []string{"targeta", "targetb"} { + for _, pk := range []string{"targeta"} { wantName, wantLat, wantLon, wantCount, wantSpread, wantOK := db.nearestPositionedNeighbor(pk, maxEdgeKm) if !wantOK { t.Fatalf("single-item nearestPositionedNeighbor(%s) returned ok=false unexpectedly", pk) @@ -617,10 +617,11 @@ func TestNearestPositionedNeighborsBulk_MatchesSingleItem(t *testing.T) { t.Fatalf("single-item nearestPositionedNeighbor(targetc) returned ok=true unexpectedly") } - // Confirm the geo-filter actually did something in this fixture (else - // the test wouldn't be exercising what it claims to). - if bulk["targetb"].ContributorCount != 1 { - t.Errorf("bulk[targetb].ContributorCount = %d, want 1 -- nbb2 should be dropped by the 50km geo-filter", bulk["targetb"].ContributorCount) + if _, ok := bulk["targetb"]; ok { + t.Error("ambiguous targetb must not get a bulk proxy") + } + if _, _, _, _, _, ok := db.nearestPositionedNeighbor("targetb", maxEdgeKm); ok { + t.Error("ambiguous targetb must not get a single proxy") } } diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 30f1c5e70..6d18fb557 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -2146,29 +2146,21 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) { log.Printf("WARN CountFloodAdvertsForNode(%s): %v", pubkey, err) } - // Every node gets an approximate position cross-check via the same - // neighbor-centroid estimate (and geo-sanity filter) that backs - // Position-Fix Coverage Gaps, View Path's approx markers, and - // Suspicious GPS Positions. For a node with no real fix this FILLS IN - // a position (see the "real fix" convention note below); for a node - // that already reports one, this lets the detail page show both side - // by side, and a distance between them, so a node flagged by - // Suspicious GPS Positions can be visually cross-checked here instead - // of just trusting the flag. - // - // Same "real fix" convention as - // GetNodesForAreaAnalytics/GetNodesForScopeAdoption: lat/lon both - // present AND non-zero -- some nodes advertise (0,0) as a "no GPS lock - // yet" sentinel rather than omitting lat/lon entirely, and without this - // check those nodes never got an estimate (nor a good real map either, - // since (0,0) plots off the coast of Africa). + // Compute a neighbor-only cross-check without consulting the target's + // reported position. Unlike legacy single-neighbor path proxies, node + // detail emits coordinates only for a supported multi-neighbor group. + // Metadata makes abstention and unknown edge freshness explicit. Neither + // neighbor spread nor distance from the reported fix is an error bound. nodeLat, hasLat := node["lat"].(float64) nodeLon, hasLon := node["lon"].(float64) - hasRealFix := hasLat && hasLon && !(nodeLat == 0 && nodeLon == 0) - if _, lat, lon, contributorCount, _, ok := s.db.nearestPositionedNeighbor(pubkey, EstimateMaxEdgeKm); ok { + hasRealFix := hasLat && hasLon && validNeighborPosition(nodeLat, nodeLon) + estimate := s.db.neighborPositionEstimate(pubkey, EstimateMaxEdgeKm, time.Now()).Estimate + node["neighbor_estimate"] = estimate + if estimate.Status == "estimated" { + lat, lon := *estimate.Lat, *estimate.Lon node["estimated_lat"] = lat node["estimated_lon"] = lon - node["estimated_contributor_count"] = contributorCount + node["estimated_contributor_count"] = estimate.ContributorCount if hasRealFix { node["estimated_distance_km"] = haversineKm(nodeLat, nodeLon, lat, lon) } diff --git a/cmd/server/routes_test.go b/cmd/server/routes_test.go index 9eae00520..1e9e4cd11 100644 --- a/cmd/server/routes_test.go +++ b/cmd/server/routes_test.go @@ -344,15 +344,15 @@ func TestNodeDetailEndpoint(t *testing.T) { } // TestNodeDetail_NoRealFix_IncludesEstimatedPosition covers a node with no -// GPS fix that has a positioned neighbor -- the detail endpoint should fall -// back to the same neighbor-centroid estimate Position-Fix Coverage Gaps and -// View Path's approx markers already use, so the node's page can still show -// a (dashed/approximate) map instead of nothing. +// GPS fix that has a coherent positioned pair. A single neighbor is no +// longer sufficient for a node position (legacy path proxies are separate). func TestNodeDetail_NoRealFix_IncludesEstimatedPosition(t *testing.T) { srv, router := setupTestServer(t) srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES ('nofix00000000001', 'NoFixNode', 'repeater', NULL, NULL, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`) srv.db.conn.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count) VALUES ('nofix00000000001', 'aabbccdd11223344', 5)`) + srv.db.conn.Exec(`INSERT INTO nodes (public_key,name,lat,lon) VALUES ('secondnear','Second',37.51,-122.01)`) + srv.db.conn.Exec(`INSERT INTO neighbor_edges (node_a,node_b,count) VALUES ('nofix00000000001','secondnear',5)`) req := httptest.NewRequest("GET", "/api/nodes/nofix00000000001", nil) w := httptest.NewRecorder() @@ -373,8 +373,12 @@ func TestNodeDetail_NoRealFix_IncludesEstimatedPosition(t *testing.T) { if node["estimated_lat"] == nil || node["estimated_lon"] == nil { t.Fatalf("expected estimated_lat/estimated_lon to be set, got node=%+v", node) } - if cc, _ := node["estimated_contributor_count"].(float64); cc < 1 { - t.Errorf("expected estimated_contributor_count >= 1, got %v", node["estimated_contributor_count"]) + if cc, _ := node["estimated_contributor_count"].(float64); cc != 2 { + t.Errorf("expected estimated_contributor_count = 2, got %v", node["estimated_contributor_count"]) + } + meta := node["neighbor_estimate"].(map[string]interface{}) + if meta["status"] != "estimated" || meta["method"] != "neighbor_cluster_v1" || meta["lat"] != node["estimated_lat"] { + t.Fatalf("bad estimate metadata: %+v", meta) } } @@ -388,6 +392,8 @@ func TestNodeDetail_ZeroZeroFix_IncludesEstimatedPosition(t *testing.T) { srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES ('zerofix0000000001', 'ZeroFixNode', 'repeater', 0, 0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`) srv.db.conn.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count) VALUES ('zerofix0000000001', 'aabbccdd11223344', 5)`) + srv.db.conn.Exec(`INSERT INTO nodes (public_key,name,lat,lon) VALUES ('secondnear','Second',37.51,-122.01)`) + srv.db.conn.Exec(`INSERT INTO neighbor_edges (node_a,node_b,count) VALUES ('zerofix0000000001','secondnear',5)`) req := httptest.NewRequest("GET", "/api/nodes/zerofix0000000001", nil) w := httptest.NewRecorder() @@ -439,6 +445,8 @@ func TestNodeDetail_RealFixWithNeighbors_IncludesBothPositions(t *testing.T) { srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES ('farneighbor00000001', 'FarNeighbor', 'repeater', 40.0, -122.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`) srv.db.conn.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count) VALUES ('aabbccdd11223344', 'farneighbor00000001', 5)`) + srv.db.conn.Exec(`INSERT INTO nodes (public_key,name,lat,lon) VALUES ('secondnear','Second',40.01,-122.01)`) + srv.db.conn.Exec(`INSERT INTO neighbor_edges (node_a,node_b,count) VALUES ('aabbccdd11223344','secondnear',5)`) req := httptest.NewRequest("GET", "/api/nodes/aabbccdd11223344", nil) w := httptest.NewRecorder() diff --git a/docs/api-spec.md b/docs/api-spec.md index 6ce429d45..8a584f3ba 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -492,7 +492,19 @@ claimed-nodes lookups do not. "hash_size": number | null, "hash_size_inconsistent": boolean, "hash_sizes_seen": [number] | undefined, - "flood_advert_count_7d": number // route_type 1 only (see below) + "flood_advert_count_7d": number, // route_type 1 only (see below) + "neighbor_estimate": { // see Neighbor estimate below + "status": "estimated" | "insufficient" | "ambiguous" | "unavailable", + "method": "neighbor_cluster_v1", + "contributor_count": number, + "candidate_count": number, + "spread_km": number, + "lat": number, // estimated only; otherwise omitted + "lon": number, // estimated only; otherwise omitted + "oldest_seen": string (ISO), // omitted if no valid contributor timestamps + "newest_seen": string (ISO), + "unknown_freshness_count": number + } }, "recentAdverts": [Packet], // last 20 packets for this node, newest ingest first; // with include=advertRoutes ADVERT rows also carry route_class @@ -515,6 +527,39 @@ claimed-nodes lookups do not. Where `Packet` is a transmission object (see [Packet Object](#packet-object)). +#### Neighbor estimate + +`node.neighbor_estimate` describes heuristic evidence sufficiency, not a +calibrated positioning confidence. Only `estimated` supplies coordinates. +`insufficient` means the selected group has fewer than two contributors; +`ambiguous` means a disjoint group has similar support; `unavailable` means +there is no usable estimate. An abstention must not be plotted using stale +legacy coordinates. Reported `node.lat` / `node.lon` remain unchanged. + +`candidate_count` is the valid positioned candidate pool (at most 20) after +excluding candidates whose capped, age-adjusted weight is below 10% of the +strongest candidate's weight; it is not the total graph degree. Excluded links +also do not count toward the minimum of two contributors. `contributor_count` +counts the selected group's neighbors; it does not count independently verified +radio sources. `spread_km` is the +largest pairwise separation of selected neighbors, **not an error radius**. +`oldest_seen` and `newest_seen` summarize their valid stored edge `last_seen` +timestamps; they are not GPS-fix timestamps or the entire observation window. +`unknown_freshness_count` includes missing or implausibly future timestamps. +Unknown freshness uses a 0.25 weight multiplier. The evidence floor is relative, +so similarly old links may still support an estimate; status is not proof of a +current position. Consumers should retain the sighting dates and freshness +caveat when presenting the result. +For an ambiguous/unavailable result no group is selected, so contributor and +spread fields are zero rather than a confidence statement. + +For compatibility, `estimated_lat`, `estimated_lon`, +`estimated_contributor_count`, and (when reported GPS exists) +`estimated_distance_km` remain on the node only when the status is `estimated`. +That distance compares the estimate to the reported coordinates; it is not +validated error. See the [node guide](user-guide/nodes.md#approximate-area-neighbor-estimate) +for the algorithm's limits and a separate, leakage-free validation procedure. + #### Advert route classes `recentAdvertsByRoute`, `advertCounts` and `route_class` (port/extension of diff --git a/docs/user-guide/nodes.md b/docs/user-guide/nodes.md index c995d5224..0b5303bab 100644 --- a/docs/user-guide/nodes.md +++ b/docs/user-guide/nodes.md @@ -63,6 +63,86 @@ Click a node row to open the **detail pane** on the right. It shows: Click the node name in the detail pane to open the **full node page** with complete history, analytics, and health data. +### Approximate area (neighbor estimate) + +An **Approximate area** is a heuristic position inferred from neighboring nodes +with reported coordinates. It is not triangulation, a GPS fix, or a measurement +of the node's location. Accuracy has not been field-validated. A reported GPS +position is displayed separately and is never replaced by the estimate. + +The estimate considers a bounded pool of up to 20 positioned neighbor candidates. +It scores geographic groups using their combined support rather than choosing +the single busiest neighbor as an anchor. The lifetime observation count has a +capped, logarithmic weight; older link sightings receive less weight. One busy +link therefore cannot have unlimited influence. Candidates whose capped, +age-adjusted weight is below 10% of the strongest candidate's weight are +excluded from both the weighted center and the minimum-two-contributor check. +An almost negligible old link cannot turn a single strong neighbor into an +apparently supported pair. This uses persisted edge counts and `last_seen`, +**not** verified independent receivers, per-edge RF confidence, +RSSI ranging, or signal triangulation. Inferred links and neighbors' reported +positions can themselves be wrong. + +The database still selects the top 20 links by lifetime count before filtering +for usable positions and freshness. Busy links without coordinates or with old +sightings can therefore keep better candidates outside the pool. Improving that +candidate selection is a separate follow-up, not an accuracy claim of this change. + +- **Approximate area:** at least two contributors support the selected group. + The displayed point is a weighted center, not proof the node lies there. +- **Insufficient neighbor evidence:** the selected group contains fewer than two + contributors, so no estimated position marker is shown. +- **Conflicting neighbor groups:** geographically separate groups have similar + support; the estimator abstains instead of choosing a misleading point. +- **Unavailable:** there is no usable estimate. + +The contributor count is relative to the selected candidate pool after this +relative-weight filter, **not all neighbors in the network**. Neighbor spread is +the greatest distance between the selected contributors; it is **not a location +error radius**. The link +sighting dates summarize contributors' stored `last_seen` values, not the dates +of every observation or when their GPS was measured. Missing or implausible +future timestamps are flagged as unknown freshness. The distance from a reported +position is a comparison between two points, not a measured positioning error. +The map uses a fixed-size dashed symbol, not an uncertainty circle. + +This is a **relative** evidence rule, not an absolute freshness guarantee. Two +similarly old links can still support an estimate; read the displayed dates. +Neither an `estimated` status nor a pair of neighbors proves the node's current +position. + +Clients connected to an older server can show a **Legacy estimate; evidence +quality unavailable** fallback when coordinates exist but metadata does not. +Known single-neighbor legacy estimates are withheld. The conservative display +rules above apply to node detail; other legacy approximate path proxies are not +claimed to have the same evidence sufficiency. + +The initial method (`neighbor_cluster_v1`) uses a 7-day freshness half-life, a +count cap of 20, an unknown-freshness multiplier of 0.25, a minimum relative +weight of 10%, and an ambiguity cutoff of 80% of the leading group's support. +These are heuristics, not calibrated probabilities. The distance limit uses the +existing 30 km estimate constant, measured from a group seed, not a bound on +location error or on every pair of contributors. Exposing these policy choices in the customizer +is a later milestone; this change does not add settings or a confidence score. + +#### How to validate accuracy separately + +1. Reserve a separate set of nodes with independently trustworthy positions and + time windows before choosing thresholds. Do not tune on this held-out set. +2. Hide each target's position from the **entire** inference pipeline, including + upstream geographic filtering and neighbor/prefix resolution. Merely omitting + its coordinates from the final centroid would still leak the answer. If + stored edges were already resolved with target GPS, rebuild them without that + input or label the evaluation as contaminated. +3. Run old and new methods on exactly the same inputs and record abstentions. + Compare median and p90 geodesic error, severe-outlier counts and distances, + and coverage (fraction of eligible targets receiving an estimate). Report + both common-target accuracy and overall coverage, so rejecting hard cases + cannot masquerade as increased accuracy. +4. Stratify by neighbor count, age, competing groups, region, and observation + density. Publish the sample sizes and limitations. Synthetic regression tests + establish behavior, not real-world positioning accuracy. + ## Favorites Nodes you've claimed on the Home page appear as favorites. You can also star nodes directly from the Nodes page. diff --git a/public/nodes.js b/public/nodes.js index c91de427e..1ea5da633 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -696,6 +696,81 @@ return nodeData; } + // Shared by the full node page and side pane. This is an evidence summary, + // not a calibrated confidence interval or a GPS fix. Keep all API values + // behind validation before using them in either HTML or Leaflet. + function neighborEstimateView(n) { + function number(value) { + if (typeof value !== 'number' && typeof value !== 'string') return null; + if (typeof value === 'string' && !value.trim()) return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + } + function count(value) { + const parsed = number(value); + return parsed != null && Number.isInteger(parsed) && parsed >= 0 ? parsed : null; + } + function date(value) { + if (typeof value !== 'string' || !value.trim()) return null; + const parsed = new Date(value); + return Number.isFinite(parsed.getTime()) ? parsed.toISOString().replace('.000Z', 'Z') : null; + } + const typed = n.neighbor_estimate != null; + const evidence = typed ? n.neighbor_estimate : {}; + let status = typed ? evidence.status : 'estimated'; + if (!['estimated', 'insufficient', 'ambiguous', 'unavailable'].includes(status)) status = 'unavailable'; + const lat = number(typed ? evidence.lat : n.estimated_lat); + const lon = number(typed ? evidence.lon : n.estimated_lon); + const contributors = count(typed ? evidence.contributor_count : n.estimated_contributor_count); + const candidates = count(evidence.candidate_count); + const validPosition = lat != null && lon != null && Math.abs(lat) <= 90 && Math.abs(lon) <= 180; + if (!typed && n.estimated_lat == null && n.estimated_lon == null) return { visible: false, hasPosition: false, html: '' }; + if (status === 'estimated' && (contributors != null && contributors < 2)) status = 'insufficient'; + if (status === 'estimated' && (!validPosition || (typed && contributors == null))) status = 'unavailable'; + const messages = { + insufficient: 'Insufficient neighbor evidence for a position. At least two supported neighbors are needed.', + ambiguous: 'Conflicting neighbor groups; no position shown.', + unavailable: 'Neighbor estimate unavailable.', + }; + if (status !== 'estimated') { + return { visible: true, status: status, hasPosition: false, html: '' + (messages[status] || messages.unavailable) + '' }; + } + const details = []; + if (typed) { + details.push(contributors + (candidates != null && candidates >= contributors ? ' of ' + candidates + ' candidate neighbors' : ' neighbors')); + const spread = number(evidence.spread_km); + if (spread != null && spread >= 0) details.push('Neighbor spread: ' + spread.toFixed(1) + ' km (not a location error radius)'); + const oldest = date(evidence.oldest_seen), newest = date(evidence.newest_seen); + if (oldest && newest && oldest <= newest) details.push('Neighbor links last seen: ' + oldest + ' – ' + newest); + else if (newest) details.push('Newest neighbor link sighting: ' + newest); + const unknown = count(evidence.unknown_freshness_count); + if (unknown > 0) details.push(unknown + ' neighbor' + (unknown === 1 ? ' has' : 's have') + ' unknown freshness'); + } else { + details.push('Legacy estimate; evidence quality unavailable'); + } + const reportedLat = number(n.lat), reportedLon = number(n.lon), distance = number(n.estimated_distance_km); + if (reportedLat != null && reportedLon != null && Math.abs(reportedLat) <= 90 && Math.abs(reportedLon) <= 180 && + !(reportedLat === 0 && reportedLon === 0) && distance != null && distance >= 0) { + details.push(distance.toFixed(1) + ' km from reported position (not an error bound)'); + } + details.push('Neighbor-based approximation, not triangulation; accuracy is not field-validated.'); + return { + visible: true, status: status, hasPosition: true, lat: lat, lon: lon, + html: '~' + lat.toFixed(2) + ', ~' + lon.toFixed(2) + '
' + details.map(escapeHtml).join('
') + '', + }; + } + + function addNeighborEstimateMarker(map, n, estimate) { + // A fixed-size point symbol, deliberately not an uncertainty/range circle. + const popup = escapeHtml(n.name || n.public_key.slice(0, 12)) + '
Approximate area
' + estimate.html; + L.circleMarker([estimate.lat, estimate.lon], { + radius: 8, color: getComputedStyle(document.documentElement).getPropertyValue('--surface-0'), + weight: 2, fillColor: getComputedStyle(document.documentElement).getPropertyValue('--accent'), + fillOpacity: 0.5, dashArray: '5,4', + }).addTo(map).bindPopup(popup); + return [estimate.lat, estimate.lon]; + } + async function loadFullNode(pubkey) { const body = document.getElementById('nodeFullBody'); const viewSeq = ++detailViewSeq; @@ -714,12 +789,8 @@ // real fix either -- otherwise it'd plot a bogus marker off the // coast of Africa instead of falling back to the estimate below. const hasLoc = n.lat != null && n.lon != null && !(n.lat === 0 && n.lon === 0); - // Neighbor-centroid estimate (same technique Position-Fix Coverage - // Gaps, View Path's approx markers, and Suspicious GPS Positions - // use). Shown alongside a real fix too, not just as a fallback when - // one's missing -- lets a node flagged by Suspicious GPS Positions - // be visually cross-checked against its own claimed position. - const hasEstLoc = n.estimated_lat != null && n.estimated_lon != null; + const estimate = neighborEstimateView(n); + const hasEstLoc = estimate.hasPosition; // Health stats const h = healthData || {}; @@ -830,7 +901,7 @@ Packets Today${stats.packetsToday || 0} ${stats.avgHops ? `Avg Hops${stats.avgHops}` : ''} ${hasLoc ? `Location${Number(n.lat).toFixed(5)}, ${Number(n.lon).toFixed(5)}` : ''} - ${hasEstLoc ? `${hasLoc ? 'Neighbor Estimate' : 'Location'} (estimated)~${Number(n.estimated_lat).toFixed(5)}, ~${Number(n.estimated_lon).toFixed(5)} (from ${n.estimated_contributor_count} neighbor${n.estimated_contributor_count === 1 ? '' : 's'}${hasLoc ? ', ' + Number(n.estimated_distance_km).toFixed(1) + ' km from reported position' : ', no real GPS fix'})` : ''} + ${estimate.visible ? `${hasEstLoc ? 'Approximate area' : 'Neighbor estimate'}${estimate.html}` : ''} Hash Prefix${n.hash_size ? '' + n.public_key.slice(0, n.hash_size * 2).toUpperCase() + ' (' + n.hash_size + '-byte)' : 'Unknown'}${n.hash_size_inconsistent ? ' varies' : ''} @@ -898,22 +969,13 @@ bounds.push([n.lat, n.lon]); } if (hasEstLoc) { - // Dashed pin, same convention as area-nodes-map.js -- this - // position is an estimate, not a reported GPS fix. - var estPopup = escapeHtml(n.name || n.public_key.slice(0, 12)) + ' (estimated position' + - (hasLoc ? ', ' + Number(n.estimated_distance_km).toFixed(1) + ' km from reported position' : ', no real GPS fix') + ')'; - L.circleMarker([n.estimated_lat, n.estimated_lon], { - radius: 8, color: getComputedStyle(document.documentElement).getPropertyValue('--surface-0') || '#fff', - weight: 2, fillColor: getComputedStyle(document.documentElement).getPropertyValue('--accent') || '#3b82f6', - fillOpacity: 0.5, dashArray: '5,4', - }).addTo(detailMap).bindPopup(estPopup); - bounds.push([n.estimated_lat, n.estimated_lon]); + bounds.push(addNeighborEstimateMarker(detailMap, n, estimate)); } if (hasLoc && hasEstLoc) { L.polyline(bounds, { color: getComputedStyle(document.documentElement).getPropertyValue('--text-muted') || '#888', weight: 2, dashArray: '4,4', opacity: 0.6 }).addTo(detailMap); detailMap.fitBounds(bounds, { padding: [30, 30] }); } else { - detailMap.setView(bounds[0], 13); + detailMap.setView(bounds[0], hasLoc ? 13 : 10); } resizeDetailMapAfterLayout(); } catch {} @@ -1823,7 +1885,8 @@ // Same "real fix" convention and estimate-alongside-real-fix behavior // as loadFullNode above -- see its comments. const hasLoc = n.lat != null && n.lon != null && !(n.lat === 0 && n.lon === 0); - const hasEstLoc = n.estimated_lat != null && n.estimated_lon != null; + const estimate = neighborEstimateView(n); + const hasEstLoc = estimate.hasPosition; const nodeUrl = location.origin + '/#/nodes/' + encodeURIComponent(n.public_key); // Status calculation via shared helper @@ -1866,7 +1929,7 @@
Packets Today
${stats.packetsToday || 0}
${stats.avgHops ? `
Avg Hops
${stats.avgHops}
` : ''} ${hasLoc ? `
Location
${Number(n.lat).toFixed(5)}, ${Number(n.lon).toFixed(5)}
` : ''} - ${hasEstLoc ? `
${hasLoc ? 'Neighbor Estimate' : 'Location'} (estimated)
~${Number(n.estimated_lat).toFixed(5)}, ~${Number(n.estimated_lon).toFixed(5)} (from ${n.estimated_contributor_count} neighbor${n.estimated_contributor_count === 1 ? '' : 's'}${hasLoc ? ', ' + Number(n.estimated_distance_km).toFixed(1) + ' km from reported position' : ', no real GPS fix'})
` : ''} + ${estimate.visible ? `
${hasEstLoc ? 'Approximate area' : 'Neighbor estimate'}
${estimate.html}
` : ''} @@ -1918,20 +1981,13 @@ panelBounds.push([n.lat, n.lon]); } if (hasEstLoc) { - var panelEstPopup = escapeHtml(n.name || n.public_key.slice(0, 12)) + ' (estimated position' + - (hasLoc ? ', ' + Number(n.estimated_distance_km).toFixed(1) + ' km from reported position' : ', no real GPS fix') + ')'; - L.circleMarker([n.estimated_lat, n.estimated_lon], { - radius: 8, color: getComputedStyle(document.documentElement).getPropertyValue('--surface-0') || '#fff', - weight: 2, fillColor: getComputedStyle(document.documentElement).getPropertyValue('--accent') || '#3b82f6', - fillOpacity: 0.5, dashArray: '5,4', - }).addTo(detailMap).bindPopup(panelEstPopup); - panelBounds.push([n.estimated_lat, n.estimated_lon]); + panelBounds.push(addNeighborEstimateMarker(detailMap, n, estimate)); } if (hasLoc && hasEstLoc) { L.polyline(panelBounds, { color: getComputedStyle(document.documentElement).getPropertyValue('--text-muted') || '#888', weight: 2, dashArray: '4,4', opacity: 0.6 }).addTo(detailMap); detailMap.fitBounds(panelBounds, { padding: [30, 30] }); } else { - detailMap.setView(panelBounds[0], 13); + detailMap.setView(panelBounds[0], hasLoc ? 13 : 10); } resizeDetailMapAfterLayout(); } catch {} @@ -2053,6 +2109,7 @@ }); // Test hooks + window._nodesNeighborEstimateView = neighborEstimateView; window._nodesIsAdvertMessage = isAdvertMessage; window._nodesGetAllNodes = function() { return _allNodes; }; window._nodesSetAllNodes = function(n) { _allNodes = n; }; diff --git a/test-all.sh b/test-all.sh index 1eb9ecff4..b15b893bb 100755 --- a/test-all.sh +++ b/test-all.sh @@ -55,6 +55,7 @@ run test-top-routes-overlay.js run test-important-links-byte-filter.js run test-url-state.js run test-node-adverts.js +run test-neighbor-estimate.js run test-perf-go-runtime.js run test-channel-psk-ux.js run test-channel-sidebar-layout.js diff --git a/test-neighbor-estimate-e2e.js b/test-neighbor-estimate-e2e.js new file mode 100644 index 000000000..9f7ac3fb1 --- /dev/null +++ b/test-neighbor-estimate-e2e.js @@ -0,0 +1,178 @@ +'use strict'; +// Real SPA, DOM, route navigation and node detail rendering against a local +// synthetic API. Leaflet's API is spied (no CDN or tiles); markers are checked +// as calls, not claimed as a geographic/real-radio accuracy validation. +// Run: node test-neighbor-estimate-e2e.js; optional SCREENSHOT_DIR for evidence. +// Optional LEAFLET_ASSET_DIR points to an unpacked Leaflet dist directory +// (leaflet.js, leaflet.css, images/) for real-map validation without CDN access. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const path = require('node:path'); +const { chromium } = require('playwright'); + +const publicDir = path.join(__dirname, 'public'); +const leafletDir = process.env.LEAFLET_ASSET_DIR; +const recent = new Date().toISOString(); +const estimate = { + status: 'estimated', method: 'neighbor_cluster_v1', lat: 55.12345, lon: 12.65432, + contributor_count: 3, candidate_count: 4, spread_km: 8.4, + newest_seen: recent, oldest_seen: recent, unknown_freshness_count: 1, +}; +const cases = [ + { name: 'GPS and approximate area', gps: true, estimate, supported: true }, + { name: 'Approximate area only', gps: false, estimate, supported: true }, + { name: 'One neighbor', gps: false, estimate: { status: 'insufficient', contributor_count: 1, candidate_count: 1 }, text: /Insufficient neighbor evidence/ }, + { name: 'Competing clusters', gps: true, estimate: { status: 'ambiguous', contributor_count: 0, candidate_count: 4 }, text: /Conflicting neighbor groups/ }, +].map((fixture, index) => ({ ...fixture, key: (index + 1).toString(16).padStart(2, '0') + '22'.repeat(31) })); +const nodes = cases.map(fixture => ({ + public_key: fixture.key, name: fixture.name, role: 'repeater', + lat: fixture.gps ? 55.23456 : null, lon: fixture.gps ? 12.34567 : null, + first_seen: recent, last_seen: recent, advert_count: 1, + neighbor_estimate: fixture.estimate, + // Deliberately stale legacy coordinates: typed abstention must override them. + estimated_lat: 54, estimated_lon: 11, estimated_contributor_count: 9, + estimated_distance_km: 23.4, +})); +function json(res, data) { + res.writeHead(200, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify(data)); +} +function recordLeafletCalls() { + // Wrap the real implementation without replacing maps, markers, or popups. + for (const kind of ['map', 'marker', 'circleMarker', 'polyline']) { + const original = window.L[kind]; + window.L[kind] = function (position, options) { + const call = kind === 'map' ? { kind, id: position } : { kind, position, options }; + window.__mapCalls.push(call); + const result = original.apply(this, arguments); + if (kind !== 'map') { + const bindPopup = result.bindPopup; + result.bindPopup = function (html) { call.popup = html; return bindPopup.apply(this, arguments); }; + } + return result; + }; + } +} +const server = http.createServer((req, res) => { + const pathname = new URL(req.url, 'http://127.0.0.1').pathname; + if (leafletDir && pathname.startsWith('/__fixture_leaflet/')) { + const relative = pathname.slice('/__fixture_leaflet/'.length); + if (!['leaflet.js', 'leaflet.css', 'images/marker-icon.png', 'images/marker-icon-2x.png', 'images/marker-shadow.png'].includes(relative)) { + res.writeHead(404); return res.end(); + } + const file = path.join(leafletDir, relative); + if (!fs.existsSync(file)) { res.writeHead(404); return res.end(); } + res.writeHead(200, { 'Content-Type': relative.endsWith('.js') ? 'application/javascript' : relative.endsWith('.css') ? 'text/css' : 'image/png' }); + const content = fs.readFileSync(file); + return res.end(relative === 'leaflet.js' ? content.toString() + '\n;(' + recordLeafletCalls.toString() + ')();' : content); + } + if (pathname === '/api/nodes') return json(res, { nodes, total: nodes.length, counts: { all: nodes.length, repeater: nodes.length } }); + const match = pathname.match(/^\/api\/nodes\/([^/]+)(?:\/(.*))?$/); + if (match) { + const node = nodes.find(n => n.public_key === match[1]); + if (node) { + if (match[2] === 'health') return json(res, { node, observers: [], recentPackets: [], stats: { lastAdvert: recent, lastHeard: recent } }); + if (match[2] === 'neighbors') return json(res, { neighbors: [] }); + if (match[2] === 'paths') return json(res, { paths: [], totalTransmissions: 0 }); + if (!match[2]) return json(res, { node, recentAdverts: [] }); + } + } + if (pathname === '/api/observers') return json(res, { observers: [] }); + if (pathname === '/api/channels') return json(res, { channels: [] }); + if (pathname.startsWith('/api/')) return json(res, {}); + const file = pathname === '/' ? path.join(publicDir, 'index.html') : path.resolve(publicDir, '.' + pathname); + if (!file.startsWith(publicDir + path.sep) || !fs.existsSync(file) || !fs.statSync(file).isFile()) { + res.writeHead(404); return res.end(); + } + const types = { '.js': 'application/javascript', '.css': 'text/css', '.html': 'text/html', '.svg': 'image/svg+xml' }; + res.writeHead(200, { 'Content-Type': types[path.extname(file)] || 'application/octet-stream' }); + // All first-party scripts stay real; remove only CDN libraries and the + // plugin that requires real Leaflet, since the test records that boundary. + const body = fs.readFileSync(file); + if (pathname !== '/') return res.end(body); + let html = body.toString().replace(/]*src="(?:https?:\/\/|vendor\/leaflet\.markercluster)[^>]*>[\s\S]*?<\/script>/g, ''); + if (leafletDir) html = html.replace('', ''); + res.end(html); +}); + +(async () => { + let browser, checks = 0; + try { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const origin = 'http://127.0.0.1:' + server.address().port; + browser = await chromium.launch({ headless: true, executablePath: process.env.CHROMIUM_PATH || undefined }); + for (const fixture of cases) { + for (const mode of ['full', 'pane']) { + const page = await browser.newPage({ viewport: { width: 1440, height: 1000 } }); + const pageErrors = []; + page.on('pageerror', error => pageErrors.push(error.message)); + page.setDefaultTimeout(10000); + await page.route('**/*', route => new URL(route.request().url()).origin === origin ? route.continue() : route.abort()); + await page.addInitScript(realLeaflet => { + window.__mapCalls = []; + if (realLeaflet) return; + const layer = kind => (position, options) => { + const call = { kind, position, options }; + window.__mapCalls.push(call); + return { addTo() { return this; }, bindPopup(html) { call.popup = html; return this; } }; + }; + window.L = { + map(id) { + window.__mapCalls.push({ kind: 'map', id }); + return { fitBounds() {}, setView() {}, invalidateSize() {}, remove() {}, getPane() { return document.getElementById(id); } }; + }, + tileLayer: () => ({ addTo() { return this; } }), + marker: layer('marker'), circleMarker: layer('circleMarker'), polyline: layer('polyline'), + }; + }, !!leafletDir); + await page.goto(origin + (mode === 'full' ? '/#/nodes/' + fixture.key : '/#/nodes'), { waitUntil: 'domcontentloaded' }); + if (mode === 'pane') await page.locator('tr[data-key="' + fixture.key + '"]').click(); + const body = page.locator(mode === 'full' ? '#nodeFullBody' : '#nodesRight'); + const row = body.locator('.neighbor-estimate'); + await row.waitFor(); + const text = await row.innerText(); + const calls = await page.evaluate(() => window.__mapCalls); + const markers = calls.filter(call => call.kind === 'circleMarker'); + assert.equal(markers.length, fixture.supported ? 1 : 0, fixture.name + ': estimated markers'); checks++; + const gps = calls.filter(call => call.kind === 'marker'); + assert.equal(gps.length, fixture.gps ? 1 : 0, fixture.name + ': GPS markers'); checks++; + if (leafletDir) { + assert.equal(await body.locator('.leaflet-container').count(), fixture.gps || fixture.supported ? 1 : 0); checks++; + assert.equal(await body.locator('.leaflet-marker-icon').count(), fixture.gps ? 1 : 0); checks++; + assert.equal(await body.locator('path.leaflet-interactive').count(), fixture.supported ? (fixture.gps ? 2 : 1) : 0); checks++; + } + if (fixture.gps) { + assert.deepEqual(gps[0].position, [55.23456, 12.34567]); checks++; + assert.match(await body.innerText(), /55\.23456, 12\.34567/); checks++; + } + if (fixture.supported) { + assert.match(text, /~55\.12, ~12\.65/); checks++; + assert.match(text, /3 of 4 candidate neighbors/); checks++; + assert.match(text, /not a location error radius/); checks++; + assert.match(text, /unknown freshness/); checks++; + assert.match(text, /not triangulation/); checks++; + assert.deepEqual(markers[0].position, [55.12345, 12.65432]); checks++; + assert.match(markers[0].popup, /Approximate area/); checks++; + assert.match(markers[0].popup, /not a location error radius/); checks++; + if (fixture.gps) { + assert.match(text, /23\.4 km from reported position \(not an error bound\)/); checks++; + } else { + assert.doesNotMatch(text, /from reported position/); checks++; + } + } else { + assert.match(text, fixture.text); checks++; + assert.doesNotMatch(text, /~54|~11|~55/); checks++; + } + assert.deepEqual(pageErrors, [], 'no browser JS errors'); checks++; + if (process.env.SCREENSHOT_DIR) await page.screenshot({ path: path.join(process.env.SCREENSHOT_DIR, 'neighbor-estimate-' + fixture.key.slice(0, 2) + '-' + mode + '.png') }); + await page.close(); + console.log('PASS ' + fixture.name + ' (' + mode + ')'); + } + } + console.log(checks + ' checks passed'); + } finally { + if (browser) await browser.close(); + await new Promise(resolve => server.close(resolve)); + } +})().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/test-neighbor-estimate.js b/test-neighbor-estimate.js new file mode 100644 index 000000000..a9a47a375 --- /dev/null +++ b/test-neighbor-estimate.js @@ -0,0 +1,95 @@ +'use strict'; +// Exercise the actual nodes.js helper, not a copied implementation. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const context = vm.createContext({ + window: {}, localStorage: { getItem: () => null }, registerPage: () => {}, + escapeHtml: value => String(value).replace(/[&<>"']/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[c]), +}); +vm.runInContext(fs.readFileSync(path.join(__dirname, 'public/nodes.js'), 'utf8'), context); +const view = context.window._nodesNeighborEstimateView; +let checks = 0; +function test(name, fn) { fn(); checks++; console.log('PASS ' + name); } +function node(estimate = {}) { + return { neighbor_estimate: { + status: 'estimated', method: 'neighbor_cluster_v1', lat: 55.123456, lon: 12.654321, + contributor_count: 3, candidate_count: 5, spread_km: 4.1234, + oldest_seen: '2026-01-01T10:00:00Z', newest_seen: '2026-01-03T10:00:00Z', + unknown_freshness_count: 0, ...estimate, + } }; +} +test('no data remains absent on old APIs', () => { + assert.equal(view({}).visible, false); +}); +test('supported estimate is approximate and explains the limits', () => { + const result = view(node()); + assert.equal(result.hasPosition, true); + assert.equal(result.lat, 55.123456); + assert.match(result.html, /~55\.12, ~12\.65/); + assert.doesNotMatch(result.html, /55\.12345/); + assert.match(result.html, /3 of 5 candidate neighbors/); + assert.match(result.html, /4\.1 km/); + assert.match(result.html, /not a location error radius/); + assert.match(result.html, /not triangulation/); + assert.match(result.html, /2026-01-01/); + assert.match(result.html, /2026-01-03/); +}); +test('strings are converted at the boundary, including zero coordinates', () => { + const result = view(node({ lat: '0', lon: '12.1', contributor_count: '2', candidate_count: '3', spread_km: '0' })); + assert.equal(result.hasPosition, true); + assert.equal(result.lat, 0); + assert.match(result.html, /2 of 3/); + assert.match(result.html, /0\.0 km/); +}); +for (const status of ['insufficient', 'ambiguous', 'unavailable']) { + test(status + ' cannot reuse stale typed or legacy coordinates', () => { + const result = view({ ...node({ status }), estimated_lat: 55, estimated_lon: 12, estimated_contributor_count: 7 }); + assert.equal(result.hasPosition, false); + assert.equal(result.visible, true); + assert.doesNotMatch(result.html, /~55/); + assert.match(result.html, status === 'insufficient' ? /Insufficient/ : status === 'ambiguous' ? /Conflicting/ : /unavailable/i); + }); +} +test('a single contributor cannot masquerade as a supported estimate', () => { + const result = view(node({ contributor_count: 1 })); + assert.equal(result.hasPosition, false); + assert.match(result.html, /Insufficient/); +}); +test('invalid coordinates, numbers and unknown statuses never make markers', () => { + for (const patch of [{ lat: null }, { lat: '' }, { lat: true }, { lat: 'NaN' }, { lon: 'Infinity' }, { lat: 91 }, { lon: -181 }, { contributor_count: -2 }, { contributor_count: 2.5 }, { status: '' }]) { + const result = view(node(patch)); + assert.equal(result.hasPosition, false, JSON.stringify(patch)); + assert.doesNotMatch(result.html, / { + const result = view(node({ oldest_seen: '', newest_seen: 'nonsense', spread_km: -1, candidate_count: '