diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 263aceea7..229aa62b7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -496,6 +496,21 @@ jobs: CHROMIUM_REQUIRE=1 CORESCOPE_SERVER_BIN=./corescope-server CORESCOPE_INGESTOR_BIN=./corescope-ingestor FIXTURE_DB=test-fixtures/e2e-fixture.db node test-channel-proposals-e2e.js 2>&1 | tee -a e2e-output.txt # Starts its own ingestor + server on a temp copy of the fixture; seeds client-decryptable packets itself. CHROMIUM_REQUIRE=1 CORESCOPE_SERVER_BIN=./corescope-server CORESCOPE_INGESTOR_BIN=./corescope-ingestor FIXTURE_DB=test-fixtures/e2e-fixture.db node test-channels-client-state-152-decrypt-e2e.js 2>&1 | tee -a e2e-output.txt + # #315 uses synthetic data and its own default/on/off Go servers. + # Pin the existing Leaflet runtime; browser network stays local-only. + leaflet_dir="$RUNNER_TEMP/corescope-315-leaflet" + mkdir -p "$leaflet_dir/images" + curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/leaflet.js -o "$leaflet_dir/leaflet.js" + curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/leaflet.css -o "$leaflet_dir/leaflet.css" + curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon.png -o "$leaflet_dir/images/marker-icon.png" + curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-icon-2x.png -o "$leaflet_dir/images/marker-icon-2x.png" + curl --fail --silent --show-error --location --retry 3 https://unpkg.com/leaflet@1.9.4/dist/images/marker-shadow.png -o "$leaflet_dir/images/marker-shadow.png" + echo "db49d009c841f5ca34a888c96511ae936fd9f5533e90d8b2c4d57596f4e5641a $leaflet_dir/leaflet.js" | sha256sum --check + echo "a7837102824184820dfa198d1ebcd109ff6d0ff9a2672a074b9a1b4d147d04c6 $leaflet_dir/leaflet.css" | sha256sum --check + echo "574c3a5cca85f4114085b6841596d62f00d7c892c7b03f28cbfa301deb1dc437 $leaflet_dir/images/marker-icon.png" | sha256sum --check + echo "00179c4c1ee830d3a108412ae0d294f55776cfeb085c60129a39aa6fc4ae2528 $leaflet_dir/images/marker-icon-2x.png" | sha256sum --check + echo "264f5c640339f042dd729062cfc04c17f8ea0f29882b538e3848ed8f10edb4da $leaflet_dir/images/marker-shadow.png" | sha256sum --check + CHROMIUM_REQUIRE=1 CORESCOPE_SERVER_BIN=./corescope-server CORESCOPE_MIGRATE_BIN=./corescope-migrate CORESCOPE_LEAFLET_DIR="$leaflet_dir" node test-issue-315-estimated-positions-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1236-map-mobile-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1329-map-controls-accordion-e2e.js 2>&1 | tee -a e2e-output.txt BASE_URL=http://localhost:13581 node test-issue-1273-qr-overlay-height-e2e.js 2>&1 | tee -a e2e-output.txt diff --git a/cmd/server/config.go b/cmd/server/config.go index c1c547c72..d28acf49f 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -149,6 +149,10 @@ type Config struct { DBPath string `json:"dbPath"` ListLimits *ListLimitsConfig `json:"listLimits"` + // EstimatedPositions is an operator-side startup policy. Missing means + // enabled for compatibility; changes require a server restart. + EstimatedPositions *EstimatedPositionsConfig `json:"estimatedPositions,omitempty"` + // ChannelProposals configures publicly suggested hashtag channels // (internal/channelregistry). Submissions open only when enabled AND a // strong apiKey is set; the same block is read by the ingestor. @@ -613,6 +617,9 @@ func LoadConfig(baseDirs ...string) (*Config, error) { if err != nil { continue } + if err := validateEstimatedPositionsConfig(data); err != nil { + return nil, fmt.Errorf("config %s: %w", p, err) + } if err := json.Unmarshal(data, cfg); err != nil { continue } diff --git a/cmd/server/db.go b/cmd/server/db.go index 5199ef5f9..5c8a88737 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2168,7 +2168,7 @@ type PacketPathPoint struct { } // PacketPathObserver is the station that produced a given branch's -// observation of a packet (see GetPacketPath), positioned from its own +// observation of a packet (see getPacketPath), positioned from its own // self-advertised GPS (the same source /api/observers uses) when known, // falling back to its configured IATA code, and finally its strongest // neighbor_edges neighbor's position (Approx=true), otherwise -- not a @@ -2276,7 +2276,7 @@ type PacketPathResponse struct { TxID int64 `json:"-"` } -// GetPacketPath resolves every distinct station that observed a packet to +// getPacketPath resolves every distinct station that observed a packet to // its own branch: hop count and (where resolvable) relay names/positions // in path order, plus that station's own position. A station can hear a // packet more than once as flood copies arrive via different routes; only @@ -2290,8 +2290,8 @@ type PacketPathResponse struct { // geo-sanity filter for the Approx position fallback (see its doc // comment) -- pass Config.NeighborMaxEdgeKm(). // obsBranch is one candidate branch of a packet's path: the deepest-hop -// observation attributed to a single observer. Shared by GetPacketPath -// (built from one hash's rows) and GetPacketPathsBulk (built the same way, +// observation attributed to a single observer. Shared by getPacketPath +// (built from one hash's rows) and getPacketPathsBulk (built the same way, // per hash, from a multi-hash result set) via parsePacketPathObsRow so the // two can never parse a row differently. type obsBranch struct { @@ -2317,8 +2317,8 @@ type packetPathNodeInfo struct { // packetPathReduction accumulates one hash's observation rows into the // deepest-hop branch per observer (best) and the single earliest-arriving -// branch overall (first), exactly as GetPacketPath's original inline loop -// did. GetPacketPathsBulk keeps one packetPathReduction per hash while +// branch overall (first), exactly as getPacketPath's original inline loop +// did. getPacketPathsBulk keeps one packetPathReduction per hash while // scanning a combined multi-hash result set. type packetPathReduction struct { best map[string]*obsBranch @@ -2336,8 +2336,8 @@ func newPacketPathReduction() *packetPathReduction { // (observations.id), and "earliest wins" ties on equal timestamp the same // way. obsID is a real, stable, monotonically-assigned DB identity (unlike // scan order, which the query planner is free to vary between the -// single-hash query GetPacketPath issues and the multi-hash query -// GetPacketPathsBulk issues) -- so both paths pick the identical branch on +// single-hash query getPacketPath issues and the multi-hash query +// getPacketPathsBulk issues) -- so both paths pick the identical branch on // a tie regardless of any difference in how their rows happen to arrive. // This determinizes previously-undefined behavior; it does not preserve // any order that was ever guaranteed before. @@ -2355,7 +2355,7 @@ func (r *packetPathReduction) fold(key string, branch *obsBranch, tsValid bool, } // parsePacketPathObsRow parses one row of the packet-path observations/ -// transmissions join (GetPacketPath and GetPacketPathsBulk use the same +// transmissions join (getPacketPath and getPacketPathsBulk use the same // column order, bulk with one leading `hash` column and both with a // trailing `o.id` column) into an obsBranch and its best-map key. ok is // false for rows that can't contribute a branch -- missing/unparsable @@ -2522,15 +2522,15 @@ func dedupPacketPathStrings(ss []string) []string { // (0,0) sentinel position the same way GetNodesForScopeAdoption and // geofilter.PassesFilter do. Input is deduped and chunked at // packetPathNodeLookupChunkSize bind parameters per query -- the caller may -// pass an arbitrarily large pubkey set (e.g. GetPacketPathsBulk's whole-batch +// pass an arbitrarily large pubkey set (e.g. getPacketPathsBulk's whole-batch // union). If any chunk's query or scan fails, the entire call fails -- // (nil, error), never a partial map, even though earlier chunks may have // already resolved cleanly; there is no cross-chunk aggregation logic // needed beyond that abort, since each pubkey is confined to exactly one // chunk (dedup happens before chunking) and therefore writes exactly one -// map entry regardless of chunk order. Shared by GetPacketPath (which +// map entry regardless of chunk order. Shared by getPacketPath (which // discards the error, preserving its existing tolerant-on-query-failure -// behavior unchanged) and GetPacketPathsBulk (which propagates it, per the +// behavior unchanged) and getPacketPathsBulk (which propagates it, per the // bulk helpers' explicit-error contract). func (db *DB) resolveNodesByPubkey(pubkeys []string) (map[string]packetPathNodeInfo, error) { nodeByPK := make(map[string]packetPathNodeInfo, len(pubkeys)) @@ -2594,8 +2594,8 @@ func (db *DB) resolveNodesByPubkey(pubkeys []string) (map[string]packetPathNodeI // unique name to exactly one chunk, but the logic doesn't rely on that) is // still detected correctly rather than only within its own chunk. Any // chunk's query/scan failure fails the entire call -- (nil, error), never a -// partial map. Shared by GetPacketPath (discards the error, preserving -// existing behavior) and GetPacketPathsBulk (propagates it). +// partial map. Shared by getPacketPath (discards the error, preserving +// existing behavior) and getPacketPathsBulk (propagates it). func (db *DB) resolveNodesByName(names []string) (map[string]packetPathNodeInfo, error) { nodeByName := make(map[string]packetPathNodeInfo, len(names)) if len(names) == 0 { @@ -2666,10 +2666,10 @@ type neighborEstimate struct { // hash, given already-resolved node position maps and a neighbor-estimate // lookup. This is the single shared implementation of branch assembly, // hop-point/observer position resolution, DistanceFromFirstKm, and sort -// order -- used identically by GetPacketPath (single hash, maps resolved +// order -- used identically by getPacketPath (single hash, maps resolved // via a per-hash query, neighborLookup calling nearestPositionedNeighbor // directly on demand, unchanged from before this refactor) and -// GetPacketPathsBulk (many hashes, maps resolved via one batched query +// getPacketPathsBulk (many hashes, maps resolved via one batched query // across the whole request, neighborLookup reading a pre-fetched map so no // per-point query happens here). Changing this function changes both paths // identically -- they cannot silently diverge. @@ -2815,7 +2815,11 @@ func buildPacketPathResponseFromReduction( return resp } -func (db *DB) GetPacketPath(hash string, maxEdgeKm float64) (*PacketPathResponse, error) { +// getPacketPath takes the caller's immutable operator policy explicitly; +// shared DB handles never hold mutable instance configuration. There is +// deliberately no always-estimating exported wrapper: a caller that did +// not state a policy would bypass the operator's #315 setting. +func (db *DB) getPacketPath(hash string, maxEdgeKm float64, estimatesEnabled bool) (*PacketPathResponse, error) { if !db.hasResolvedPath() { return nil, fmt.Errorf("resolved_path not available on this server") } @@ -2870,9 +2874,9 @@ func (db *DB) GetPacketPath(hash string, maxEdgeKm float64) (*PacketPathResponse for pk := range pubkeySet { pubkeys = append(pubkeys, pk) } - // Error discarded here on purpose -- preserves GetPacketPath's existing + // Error discarded here on purpose -- preserves getPacketPath's existing // tolerant-on-query-failure behavior (a failed lookup just leaves - // positions unresolved, same as before this refactor). GetPacketPathsBulk + // positions unresolved, same as before this refactor). getPacketPathsBulk // propagates this same helper's error instead; see its own call site. nodeByPK, _ := db.resolveNodesByPubkey(pubkeys) @@ -2884,6 +2888,9 @@ func (db *DB) GetPacketPath(hash string, maxEdgeKm float64) (*PacketPathResponse nodeByName, _ := db.resolveNodesByName(names) // discarded for the same reason as above neighborLookup := func(pk string) (neighborEstimate, bool) { + if !estimatesEnabled { + return neighborEstimate{}, false + } _, nLat, nLon, nCount, nSpread, ok := db.nearestPositionedNeighbor(pk, maxEdgeKm) if !ok { return neighborEstimate{}, false @@ -3632,7 +3639,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . // reply text -- the same farthest-from-first-hearer distance View // Path shows on its map, computed here as a cheap position-only // pass (no neighbor-centroid approximation) rather than reusing - // GetPacketPath's heavier per-branch query for every ping. + // getPacketPath's heavier per-branch query for every ping. observerPubkeys map[string]bool firstPubkey string firstTS int64 @@ -3815,7 +3822,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . // Bulk-resolve observer positions too, for the "spread up to Nkm" // part of the reply -- only for pings that could possibly show one // (a first-hearer plus at least one other distinct station), and - // deliberately WITHOUT GetPacketPath's neighbor-centroid fallback + // deliberately WITHOUT getPacketPath's neighbor-centroid fallback // for unpositioned stations: that's a per-node query each, too // expensive to run for every ping on a page of channel messages. // A station missing its own GPS fix just doesn't contribute here. @@ -3850,7 +3857,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . var pk string var lat, lon sql.NullFloat64 // (0,0) is the ocean off Ghana, not a real fix -- same - // exclusion GetPacketPath applies. + // exclusion getPacketPath applies. if posRows.Scan(&pk, &lat, &lon) == nil && lat.Valid && lon.Valid && !(lat.Float64 == 0 && lon.Float64 == 0) { posByPK[pk] = [2]float64{lat.Float64, lon.Float64} } @@ -5712,7 +5719,7 @@ func (db *DB) gpsByPubkeysExact(pubkeys []string) map[string][2]float64 { continue } // (0,0) is the ocean off Ghana, not a real fix -- same - // exclusion GetPacketPath/packetSpreadStats apply. + // exclusion getPacketPath/packetSpreadStats apply. if lat.Valid && lon.Valid && !(lat.Float64 == 0 && lon.Float64 == 0) { result[pk] = [2]float64{lat.Float64, lon.Float64} } diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 77ea6f88a..4294f55ea 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -676,7 +676,7 @@ func TestGetPacketPath(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, resolved_path, timestamp) VALUES (1, 2, 4.0, -95, '["aa","bb"]', '["pkAlpha","pkBravo"]', 1736935260)`) - resp, err := db.GetPacketPath("pathtest00000001", 0) + resp, err := db.testPacketPath("pathtest00000001", 0) if err != nil { t.Fatal(err) } @@ -737,7 +737,7 @@ func TestGetPacketPath_First(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 3, 6.0, -90, '["aa","bb"]', 300)`) - resp, err := db.GetPacketPath("pathtest00000007", 0) + resp, err := db.testPacketPath("pathtest00000007", 0) if err != nil { t.Fatal(err) } @@ -821,7 +821,7 @@ func TestGetPacketPath_DistanceOmittedWhenApprox(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 2, 4.0, -95, '["aa","bb"]', 200)`) - resp, err := db.GetPacketPath("pathtest00000011", 0) + resp, err := db.testPacketPath("pathtest00000011", 0) if err != nil { t.Fatal(err) } @@ -872,7 +872,7 @@ func TestGetPacketPath_ExcludesNullIsland(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, resolved_path, timestamp) VALUES (1, 1, 9.0, -88, '["aa"]', '["pkZero"]', 1736935200)`) - resp, err := db.GetPacketPath("pathtest00000008", 0) + resp, err := db.testPacketPath("pathtest00000008", 0) if err != nil { t.Fatal(err) } @@ -924,7 +924,7 @@ func TestGetPacketPath_FallsBackToSingleNeighborPosition(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, resolved_path, timestamp) VALUES (1, 1, 9.0, -88, '["aa"]', '["pkghost"]', 1736935200)`) - resp, err := db.GetPacketPath("pathtest00000009", 0) + resp, err := db.testPacketPath("pathtest00000009", 0) if err != nil { t.Fatal(err) } @@ -999,7 +999,7 @@ func TestGetPacketPath_FallsBackToWeightedNeighborCentroid(t *testing.T) { // (nearestPositionedNeighbor's maxEdgeKm) is a separate concern with // its own dedicated tests below and would otherwise drop WeakRepeater // here, breaking ApproxNeighborCount/ApproxSpreadKm's assertions. - resp, err := db.GetPacketPath("pathtest00000010", 0) + resp, err := db.testPacketPath("pathtest00000010", 0) if err != nil { t.Fatal(err) } @@ -1142,7 +1142,7 @@ func TestGetPacketPath_ObserverPositionPrefersOwnGPS(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 1, 9.0, -88, '[]', 1736935200)`) - resp, err := db.GetPacketPath("pathtest00000004", 0) + resp, err := db.testPacketPath("pathtest00000004", 0) if err != nil { t.Fatal(err) } @@ -1185,7 +1185,7 @@ func TestGetPacketPath_ObserverPositionFallsBackToNameMatch(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 1, 9.0, -88, '[]', 1736935200)`) - resp, err := db.GetPacketPath("pathtest00000005", 0) + resp, err := db.testPacketPath("pathtest00000005", 0) if err != nil { t.Fatal(err) } @@ -1220,7 +1220,7 @@ func TestGetPacketPath_ObserverPositionSkipsAmbiguousNameMatch(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 1, 9.0, -88, '[]', 1736935200)`) - resp, err := db.GetPacketPath("pathtest00000006", 0) + resp, err := db.testPacketPath("pathtest00000006", 0) if err != nil { t.Fatal(err) } @@ -1250,7 +1250,7 @@ func TestGetPacketPath_NoResolvedPath(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 1, 9.0, -88, '["aa"]', 1736935200)`) - resp, err := db.GetPacketPath("pathtest00000002", 0) + resp, err := db.testPacketPath("pathtest00000002", 0) if err != nil { t.Fatal(err) } @@ -1290,7 +1290,7 @@ func TestGetPacketPath_SameObserverMultipleObservations(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, resolved_path, timestamp) VALUES (1, 1, 6.0, -100, '["aa"]', '["pkAlpha"]', 1736935260)`) - resp, err := db.GetPacketPath("pathtest00000003", 0) + resp, err := db.testPacketPath("pathtest00000003", 0) if err != nil { t.Fatal(err) } @@ -1306,7 +1306,7 @@ func TestGetPacketPath_UnknownHash(t *testing.T) { db := setupTestDB(t) defer db.Close() - resp, err := db.GetPacketPath("doesnotexist0000", 0) + resp, err := db.testPacketPath("doesnotexist0000", 0) if err != nil { t.Fatal(err) } diff --git a/cmd/server/estimated_positions.go b/cmd/server/estimated_positions.go new file mode 100644 index 000000000..3124b55c2 --- /dev/null +++ b/cmd/server/estimated_positions.go @@ -0,0 +1,118 @@ +package main + +import ( + "bytes" + "encoding/json" + "net/http" +) + +type EstimatedPositionsConfig struct { + Enabled *bool `json:"enabled,omitempty"` +} + +type invalidEstimatedPositionsConfigError struct{ reason string } + +func (e *invalidEstimatedPositionsConfigError) Error() string { return e.reason } + +type EstimatedPositionsClientConfig struct { + Enabled bool `json:"enabled"` +} + +// Disabled analytics deliberately omit uncomputed measurements rather than +// report a misleading zero or "no neighbor evidence" conclusion. +type EstimatedPositionsDisabledResponse struct { + EstimatedPositionsEnabled bool `json:"estimatedPositionsEnabled"` +} + +type DisabledAreaAnalyticsResponse struct { + EstimatedPositionsDisabledResponse + Density []AreaDensity `json:"density"` + BridgeNodes []AreaBridgeNode `json:"bridgeNodes"` + UnpositionedTotal int `json:"unpositionedTotal"` +} + +func (cfg *Config) estimatedPositionsEnabled() bool { + return cfg == nil || cfg.EstimatedPositions == nil || cfg.EstimatedPositions.Enabled == nil || *cfg.EstimatedPositions.Enabled +} + +func (s *Server) estimatedPositionsEnabled() bool { + return !s.estimatedPositionsDisabled +} + +// LoadConfig historically tolerates malformed JSON. A syntactically valid +// operator policy with a wrong type must instead fail startup explicitly: +// silently treating "false" as true would defeat the operator's intent. +func validateEstimatedPositionsConfig(data []byte) error { + var envelope struct { + EstimatedPositions json.RawMessage `json:"estimatedPositions"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return nil + } + if len(envelope.EstimatedPositions) == 0 { + return nil + } + var block struct { + Enabled json.RawMessage `json:"enabled"` + } + if bytes.Equal(bytes.TrimSpace(envelope.EstimatedPositions), []byte("null")) { + return &invalidEstimatedPositionsConfigError{reason: "estimatedPositions must be an object"} + } + if err := json.Unmarshal(envelope.EstimatedPositions, &block); err != nil { + return &invalidEstimatedPositionsConfigError{reason: "estimatedPositions must be an object"} + } + if len(block.Enabled) == 0 { + return nil + } + raw := string(bytes.TrimSpace(block.Enabled)) + if raw != "true" && raw != "false" { + return &invalidEstimatedPositionsConfigError{reason: "estimatedPositions.enabled must be a boolean"} + } + return nil +} + +func (s *Server) estimateNodePosition(pubkey string) (string, float64, float64, int, float64, bool) { + if !s.estimatedPositionsEnabled() { + return "", 0, 0, 0, 0, false + } + return s.db.nearestPositionedNeighbor(pubkey, EstimateMaxEdgeKm) +} + +func (s *Server) writeAreaAnalytics(w http.ResponseWriter, resp *AreaAnalyticsResponse) { + if s.estimatedPositionsEnabled() { + writeJSON(w, resp) + return + } + writeJSON(w, DisabledAreaAnalyticsResponse{Density: resp.Density, BridgeNodes: resp.BridgeNodes, UnpositionedTotal: resp.UnpositionedTotal}) +} + +// stripEstimatedPositions only operates on request-owned paths (archived +// paths must be cloned first). Keep route identities, actual GPS and airtime. +// Distance is measured between observer endpoints, not relay coordinates: +// clearing an approximate relay must not erase a real endpoint measurement. +func stripEstimatedPositions(path *PacketPathResponse) { + firstApprox := path.First != nil && path.First.Observer != nil && path.First.Observer.Approx + strip := func(b *PacketPathBranch) { + for i := range b.Points { + p := &b.Points[i] + if p.Approx { + p.Lat, p.Lon, p.ApproxSpreadKm = nil, nil, nil + p.Approx, p.ApproxNeighborCount = false, 0 + } + } + if firstApprox || (b.Observer != nil && b.Observer.Approx) { + b.DistanceFromFirstKm = nil + } + if o := b.Observer; o != nil && o.Approx { + o.Lat, o.Lon, o.ApproxSpreadKm = nil, nil, nil + o.Approx, o.ApproxNeighborCount = false, 0 + } + } + for i := range path.Branches { + strip(&path.Branches[i]) + } + if path.First != nil { + strip(path.First) + } + path.TouchedAreas = nil +} diff --git a/cmd/server/estimated_positions_test.go b/cmd/server/estimated_positions_test.go new file mode 100644 index 000000000..e37d8fdec --- /dev/null +++ b/cmd/server/estimated_positions_test.go @@ -0,0 +1,547 @@ +package main + +import ( + "context" + "encoding/json" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" +) + +func TestEstimatedPositionsStartupHelper(t *testing.T) { + if os.Getenv("CORESCOPE_ESTIMATED_POSITIONS_HELPER") != "1" { + t.Skip("subprocess helper") + } + os.Args = []string{"corescope-server", "-config-dir", os.Getenv("CORESCOPE_ESTIMATED_POSITIONS_TEST_CONFIG")} + main() +} + +func TestEstimatedPositionsInvalidStartup(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"estimatedPositions":{"enabled":"false"}}`), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestEstimatedPositionsStartupHelper$", "-test.count=1") + cmd.Env = append(os.Environ(), "CORESCOPE_ESTIMATED_POSITIONS_HELPER=1", "CORESCOPE_ESTIMATED_POSITIONS_TEST_CONFIG="+dir, "ENABLE_PPROF=false") + out, err := cmd.CombinedOutput() + if err == nil || ctx.Err() != nil { + t.Fatalf("must reject startup promptly: %v, %s", err, out) + } + if !strings.Contains(string(out), "[config] fatal:") || !strings.Contains(string(out), "estimatedPositions.enabled must be a boolean") || strings.Contains(string(out), "panic:") { + t.Fatalf("expected clear startup rejection, got %s", out) + } +} + +func disabledEstimatedPositionsConfig() *Config { + enabled := false + return &Config{EstimatedPositions: &EstimatedPositionsConfig{Enabled: &enabled}} +} + +func TestEstimatedPositionsConfig(t *testing.T) { + for _, tc := range []struct { + name, raw string + want bool + invalid bool + }{ + {"missing", `{}`, true, false}, + {"empty", `{"estimatedPositions":{}}`, true, false}, + {"true", `{"estimatedPositions":{"enabled":true}}`, true, false}, + {"false", `{"estimatedPositions":{"enabled":false}}`, false, false}, + {"string", `{"estimatedPositions":{"enabled":"false"}}`, false, true}, + {"number", `{"estimatedPositions":{"enabled":0}}`, false, true}, + {"null flag", `{"estimatedPositions":{"enabled":null}}`, false, true}, + {"null section", `{"estimatedPositions":null}`, false, true}, + {"wrong section", `{"estimatedPositions":false}`, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "config.json"), []byte(tc.raw), 0600); err != nil { + t.Fatal(err) + } + cfg, err := LoadConfig(dir) + if tc.invalid { + if err == nil || !strings.Contains(err.Error(), "estimatedPositions") { + t.Fatalf("want explicit config error, got %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + s := NewServer(nil, cfg, nil) + if s.estimatedPositionsEnabled() != tc.want { + t.Fatalf("enabled=%v want=%v", s.estimatedPositionsEnabled(), tc.want) + } + // Mutating the config after startup must not silently change policy. + flip := !tc.want + cfg.EstimatedPositions = &EstimatedPositionsConfig{Enabled: &flip} + w := httptest.NewRecorder() + s.handleConfigClient(w, httptest.NewRequest("GET", "/api/config/client", nil)) + var resp struct { + EstimatedPositions struct { + Enabled bool `json:"enabled"` + } `json:"estimatedPositions"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.EstimatedPositions.Enabled != tc.want { + t.Fatalf("client flag changed without restart: %s", w.Body.String()) + } + }) + } +} + +func TestEstimatedPositionsPathsSkipNeighborQueries(t *testing.T) { + db := setupPacketPathCountingDB(t) + defer db.Close() + _, err := db.conn.Exec(` + INSERT INTO nodes(public_key,name,role,lat,lon) VALUES ('anchor','GPS','repeater',55.5,9.5),('ghost','No GPS','repeater',NULL,NULL),('observer','Observer','repeater',NULL,NULL); + INSERT INTO observers(id,name) VALUES ('observer','Observer'); + INSERT INTO neighbor_edges(node_a,node_b,count) VALUES ('anchor','ghost',10),('anchor','observer',10); + INSERT INTO transmissions(id,raw_hex,hash,first_seen) VALUES(1,'AA','estimatepolicy','2026-01-01T00:00:00Z'); + INSERT INTO observations(transmission_id,observer_idx,path_json,resolved_path,timestamp) VALUES(1,1,'["aa","bb"]','["ghost","anchor"]',1736935200);`) + if err != nil { + t.Fatal(err) + } + for _, enabled := range []bool{true, false, true} { + resetBulkTestQueryLog() + single, err := db.getPacketPath("estimatepolicy", 50, enabled) + if err != nil { + t.Fatal(err) + } + bulk, err := db.getPacketPathsBulk([]string{"estimatepolicy"}, 50, enabled) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(single, bulk["estimatepolicy"]) { + t.Fatal("single/bulk diverged") + } + b := single.Branches[0] + if len(b.Points) != 2 || b.Points[0].PublicKey != "ghost" || b.Observer.PublicKey != "observer" { + t.Fatal("route identities lost") + } + if b.Points[1].Lat == nil || *b.Points[1].Lat != 55.5 { + t.Fatal("reported GPS lost") + } + if (b.Points[0].Lat != nil) != enabled || b.Points[0].Approx != enabled || (b.Observer.Lat != nil) != enabled { + t.Fatalf("unexpected estimate policy %+v", b) + } + queries := 0 + for _, q := range bulkTestQueryLog() { + if strings.Contains(q.sql, "neighbor_edges") { + queries++ + } + } + if enabled && queries == 0 { + t.Fatal("fixture failed to exercise estimator") + } + if !enabled && queries != 0 { + t.Fatalf("disabled performed %d neighbor queries", queries) + } + } +} + +func TestEstimatedPositionsNodePreservesGPS(t *testing.T) { + on, _ := setupTestServer(t) + _, err := on.db.conn.Exec(`INSERT INTO nodes(public_key,name,role,lat,lon) VALUES('policyghost','No GPS','repeater',NULL,NULL); INSERT INTO neighbor_edges(node_a,node_b,count) VALUES('policyghost','aabbccdd11223344',10)`) + if err != nil { + t.Fatal(err) + } + off := NewServer(on.db, disabledEstimatedPositionsConfig(), nil) + for _, s := range []*Server{on, off, on} { + for _, key := range []string{"policyghost", "aabbccdd11223344"} { + w := httptest.NewRecorder() + r := mux.SetURLVars(httptest.NewRequest("GET", "/api/nodes/"+key, nil), map[string]string{"pubkey": key}) + s.handleNodeDetail(w, r) + if w.Code != 200 { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + var resp struct { + Node map[string]interface{} `json:"node"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if key == "policyghost" && (resp.Node["estimated_lat"] != nil) != s.estimatedPositionsEnabled() { + t.Fatalf("estimate policy wrong: %s", w.Body.String()) + } + if key != "policyghost" && resp.Node["lat"] == nil { + t.Fatal("real GPS removed") + } + if !s.estimatedPositionsEnabled() { + for k := range resp.Node { + if strings.HasPrefix(k, "estimated_") { + t.Fatalf("leaked %s", k) + } + } + } + } + } +} + +func TestEstimatedPositionsAnalyticsDisabled(t *testing.T) { + s := NewServer(nil, disabledEstimatedPositionsConfig(), nil) + // A nil DB would panic if node detail's estimator wrapper did any work. + if _, _, _, _, _, ok := s.estimateNodePosition("unpositioned"); ok { + t.Fatal("disabled node estimate returned a position") + } + // A pre-existing cache must never bypass the disabled response. + s.gpsSanityCache = &GPSSanityResponse{Evaluated: 1, Nodes: []SuspiciousGPSNode{{ClusterLat: 55, ClusterLon: 10}}} + s.gpsSanityCachedAt = time.Now() + w := httptest.NewRecorder() + s.handleGPSSanity(w, httptest.NewRequest("GET", "/api/analytics/gps-sanity", nil)) + if strings.TrimSpace(w.Body.String()) != `{"estimatedPositionsEnabled":false}` { + t.Fatalf("disabled GPS-sanity: %s", w.Body.String()) + } + s.areaAnalyticsCache = &AreaAnalyticsResponse{EstimatedNodes: []EstimatedAreaNode{{Lat: 55, Lon: 10}}} + s.areaAnalyticsCachedAt = time.Now() + w = httptest.NewRecorder() + s.handleAreaAnalytics(w, httptest.NewRequest("GET", "/api/analytics/areas", nil)) + var resp map[string]interface{} + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp["estimatedPositionsEnabled"] != false { + t.Fatalf("missing disabled status: %s", w.Body.String()) + } + for _, field := range []string{"estimatedNodes", "positionGaps", "unpositionedNoNeighborFix"} { + if _, ok := resp[field]; ok { + t.Fatalf("uncomputed %s emitted", field) + } + } +} + +func TestEstimatedPositionsArchivedPathImmutable(t *testing.T) { + on, _ := setupPingScoresFixture(t) + snap := testPingScorePathArchive(t, on) + archive := snap.pathArchives["allTime.farthestPing"] + archive.Path.Branches[0].Points[0].Approx = true + archive.Path.Branches[0].Points[0].ApproxNeighborCount = 2 + snap.pathArchives["allTime.farthestPing"] = archive + before := mustJSON(t, snap.pathArchives) + off := NewServer(on.db, disabledEstimatedPositionsConfig(), nil) + off.pingScores.Store(snap) + for _, s := range []*Server{off, on} { + w := httptest.NewRecorder() + r := mux.SetURLVars(httptest.NewRequest("GET", "/api/ping-scores/archive01/path?record=allTime.farthestPing", nil), map[string]string{"hash": "archive01"}) + s.handlePingScorePath(w, r) + var resp PingScorePathResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if w.Code != 200 || resp.Path == nil { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + point := resp.Path.Branches[0].Points[0] + if point.PublicKey != "RELAY" || (point.Lat != nil) != s.estimatedPositionsEnabled() || point.Approx != s.estimatedPositionsEnabled() { + t.Fatalf("bad archived point %+v", point) + } + if resp.Path.Branches[0].Observer.Lat == nil || resp.Path.Branches[0].DistanceFromFirstKm == nil { + t.Fatal("real GPS distance removed") + } + } + if mustJSON(t, snap.pathArchives) != before { + t.Fatal("mutated shared archive") + } +} + +func TestEstimatedPositionsAreasSkipQueriesAndPreserveDensity(t *testing.T) { + db := setupPacketPathCountingDB(t) + defer db.Close() + _, err := db.conn.Exec(`ALTER TABLE nodes ADD COLUMN last_seen TEXT; + INSERT INTO nodes(public_key,name,role,lat,lon) VALUES ('anchor','GPS','repeater',55.5,9.5),('ghost','No GPS','repeater',NULL,NULL); + INSERT INTO neighbor_edges(node_a,node_b,count) VALUES ('anchor','ghost',10);`) + if err != nil { + t.Fatal(err) + } + value := func(f float64) *float64 { return &f } + areas := map[string]AreaEntry{"test": {Label: "Test area", LatMin: value(55), LatMax: value(56), LonMin: value(9), LonMax: value(10)}} + for _, enabled := range []bool{true, false} { + cfg := &Config{Areas: areas, EstimatedPositions: &EstimatedPositionsConfig{Enabled: &enabled}} + s := NewServer(db, cfg, nil) + resetBulkTestQueryLog() + w := httptest.NewRecorder() + s.handleAreaAnalytics(w, httptest.NewRequest("GET", "/api/analytics/areas", nil)) + if w.Code != 200 { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + var resp struct { + Density []AreaDensity `json:"density"` + UnpositionedTotal int `json:"unpositionedTotal"` + EstimatedNodes []EstimatedAreaNode `json:"estimatedNodes"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Density) != 1 || resp.Density[0].Total != 1 || resp.UnpositionedTotal != 1 { + t.Fatalf("real metrics lost: %s", w.Body.String()) + } + if (len(resp.EstimatedNodes) > 0) != enabled { + t.Fatalf("bad estimate policy: %s", w.Body.String()) + } + queries := 0 + for _, q := range bulkTestQueryLog() { + if strings.Contains(q.sql, "neighbor_edges") { + queries++ + } + } + if enabled && queries == 0 { + t.Fatal("on fixture not exercised") + } + if !enabled && queries != 0 { + t.Fatal("disabled areas queried neighbor estimates") + } + resetBulkTestQueryLog() + w = httptest.NewRecorder() + s.handleAreaAnalytics(w, httptest.NewRequest("GET", "/api/analytics/areas", nil)) + if len(bulkTestQueryLog()) != 0 { + t.Fatal("area cache lost") + } + } +} + +func TestEstimatedPositionsArchivedApproxObserverDistance(t *testing.T) { + on, _ := setupPingScoresFixture(t) + snap := testPingScorePathArchive(t, on) + archive := snap.pathArchives["allTime.farthestPing"] + archive.Path.Branches[0].Observer.Approx = true + archive.Path.Branches[0].Observer.ApproxNeighborCount = 2 + snap.pathArchives["allTime.farthestPing"] = archive + before := mustJSON(t, snap.pathArchives) + off := NewServer(on.db, disabledEstimatedPositionsConfig(), nil) + off.pingScores.Store(snap) + w := httptest.NewRecorder() + r := mux.SetURLVars(httptest.NewRequest("GET", "/api/ping-scores/archive01/path?record=allTime.farthestPing", nil), map[string]string{"hash": "archive01"}) + off.handlePingScorePath(w, r) + var resp PingScorePathResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if w.Code != 200 || resp.Path == nil { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + branch := resp.Path.Branches[0] + if branch.Observer.PublicKey != "PINGOBSB" || branch.Observer.Lat != nil || branch.Observer.Approx || branch.DistanceFromFirstKm != nil { + t.Fatalf("approx endpoint leaked: %+v", branch) + } + if resp.Path.First.Observer.Lat == nil || branch.Points[0].Lat == nil { + t.Fatal("real coordinates lost") + } + if mustJSON(t, snap.pathArchives) != before { + t.Fatal("source archive mutated") + } +} + +// Ping metrics deliberately use only non-approximate observer endpoints; +// changing position policy must not change cached/history score records. +func TestEstimatedPositionsPingMetricsIndependent(t *testing.T) { + s, _ := setupPingScoresFixture(t) + txID := seedPingTrigger(t, s, "policyping", "#test", "Sender", "2026-01-15T10:00:00Z") + seedPingObservation(t, s, txID, "pingobsa", 9, `[]`, `[]`, 1736935200) + seedPingObservation(t, s, txID, "pingobsb", 9, `[]`, `[]`, 1736935201) + _, err := s.db.conn.Exec(`INSERT INTO nodes(public_key,name,role,lat,lon) VALUES ('policyobserver','No GPS','repeater',NULL,NULL); + INSERT INTO observers(id,name) VALUES ('policyobserver','No GPS'); + INSERT INTO neighbor_edges(node_a,node_b,count) VALUES ('policyobserver','pingobsa',10);`) + if err != nil { + t.Fatal(err) + } + // This ping already has real endpoints. Add an unpositioned + // observer and relay to that same transmission, backed by a neighbor. + triggers, err := s.db.fetchPingTriggers() + if err != nil || len(triggers) == 0 { + t.Fatalf("fixture triggers: %v", err) + } + trigger := triggers[0] + _, err = s.db.conn.Exec(`INSERT INTO observations(transmission_id,observer_idx,path_json,resolved_path,timestamp) + VALUES (?,(SELECT rowid FROM observers WHERE id='policyobserver'),'["aa"]','["policyobserver"]',1736935210)`, trigger.txID) + if err != nil { + t.Fatal(err) + } + var scores []*PingScore + for _, enabled := range []bool{true, false} { + path, err := s.db.getPacketPath(trigger.hash, 50, enabled) + if err != nil { + t.Fatal(err) + } + scores = append(scores, s.buildPingScoreFromPath(trigger, path)) + for _, b := range path.Branches { + if b.Observer != nil && b.Observer.Approx && b.DistanceFromFirstKm != nil { + t.Fatal("approx endpoint credited as GPS distance") + } + } + } + if mustJSON(t, scores[0]) != mustJSON(t, scores[1]) { + t.Fatalf("position policy changed real Ping metrics: %s vs %s", mustJSON(t, scores[0]), mustJSON(t, scores[1])) + } +} + +// seedEstimatedPositionsArchiveFixture captures one Ping Scores record +// archive whose route really does carry neighbor-estimated geometry: the +// relay hop has no GPS of its own, so a positioned neighbor supplies an +// approximate stand-in that ends up inside the persisted path_json. +func seedEstimatedPositionsArchiveFixture(t *testing.T) *engineFixture { + t.Helper() + fx := setupEngineFixture(t, pingScoreHistoryEngineConfig{SettleDebounce: time.Minute, DeepSweepBatchSize: 100, RetentionDuration: 30 * 24 * time.Hour}) + ts := fx.clock.Now().Add(-time.Hour) + id := seedPingTrigger(t, fx.srv, "estarchive0001", "#test", "sender", ts.UTC().Format(time.RFC3339)) + seedPingObservation(t, fx.srv, id, "pingobsa", 9, `[]`, `[]`, ts.Unix()) + seedPingObservation(t, fx.srv, id, "pingobsb", 7, `["aa"]`, `["relay"]`, ts.Unix()+10) + seedPingObservation(t, fx.srv, id, "pingobsc", 5, `["aa","bb"]`, `["relay","relay2"]`, ts.Unix()+20) + if _, err := fx.srv.db.conn.Exec(`INSERT INTO neighbor_edges(node_a,node_b,count) VALUES('relay','pingobsa',10)`); err != nil { + t.Fatal(err) + } + settleEntry(t, fx) + snap, err := fx.engine.QuickSnapshot() + if err != nil { + t.Fatal(err) + } + archive, ok := snap.pathArchives["allTime.farthestPing"] + if !ok { + t.Fatal("farthest archive not captured") + } + approx := false + for _, b := range archive.Path.Branches { + for _, p := range b.Points { + if p.Approx && p.Lat != nil { + approx = true + } + } + } + if !approx { + t.Fatalf("fixture captured no estimate geometry: %s", mustJSON(t, archive)) + } + return fx +} + +// Issue #315 point 5 and the PR's own promise: the operator policy filters +// request-owned copies. A disabled cycle must NOT rewrite the persisted +// archive -- once the raw observations expire, the recorded estimate +// geometry would be gone for good, and CapturedAt would be bumped without +// any new evidence. +func TestEstimatedPositionsDisabledCycleKeepsArchives(t *testing.T) { + fx := seedEstimatedPositionsArchiveFixture(t) + before, err := fx.store.LoadPathArchives() + if err != nil { + t.Fatal(err) + } + if len(before) == 0 { + t.Fatal("nothing persisted to protect") + } + beforeJSON := mustJSON(t, before) + // Report one record rather than all ten slots of the same path. + report := func(label string, got map[string]PingScorePathArchive) string { + return label + "=" + mustJSON(t, got["allTime.farthestPing"]) + } + + off := NewServer(fx.srv.db, disabledEstimatedPositionsConfig(), nil) + offEngine, err := newPingScoreHistoryEngine(off, fx.store, fx.clock.Now, fx.config) + if err != nil { + t.Fatal(err) + } + fx.clock.Advance(time.Hour) + snap, err := offEngine.Cycle() + if err != nil { + t.Fatal(err) + } + after, err := fx.store.LoadPathArchives() + if err != nil { + t.Fatal(err) + } + if mustJSON(t, after) != beforeJSON { + t.Fatalf("disabled cycle rewrote persisted archives:\n%s\n%s", report("before", before), report("after ", after)) + } + if mustJSON(t, snap.pathArchives) != beforeJSON { + t.Fatalf("disabled cycle published rewritten archives:\n%s\n%s", report("before", before), report("after ", snap.pathArchives)) + } + + // The request copy must still be stripped: preserving the archive is + // not a licence to serve estimates while the policy is off. + off.pingScores.Store(snap) + w := httptest.NewRecorder() + r := mux.SetURLVars(httptest.NewRequest("GET", "/api/ping-scores/estarchive0001/path?record=allTime.farthestPing", nil), map[string]string{"hash": "estarchive0001"}) + off.handlePingScorePath(w, r) + var resp PingScorePathResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if w.Code != 200 || resp.Path == nil { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + for _, b := range resp.Path.Branches { + for _, p := range b.Points { + if p.Approx || (p.Lat != nil && p.PublicKey == "relay") { + t.Fatalf("disabled response leaked estimate geometry: %+v", p) + } + } + } + if mustJSON(t, snap.pathArchives) != beforeJSON { + t.Fatalf("response filtering mutated the shared archive:\n%s\n%s", report("before", before), report("after ", snap.pathArchives)) + } + + // Restoring the policy must hand back the original evidence, not a + // stripped reconstruction. + onEngine, err := newPingScoreHistoryEngine(fx.srv, fx.store, fx.clock.Now, fx.config) + if err != nil { + t.Fatal(err) + } + fx.clock.Advance(time.Hour) + onSnap, err := onEngine.Cycle() + if err != nil { + t.Fatal(err) + } + restored, err := fx.store.LoadPathArchives() + if err != nil { + t.Fatal(err) + } + if mustJSON(t, restored) != beforeJSON { + t.Fatalf("re-enabling rewrote archives:\n%s\n%s", report("before", before), report("after ", restored)) + } + if mustJSON(t, onSnap.pathArchives) != beforeJSON { + t.Fatalf("re-enabled snapshot lost original evidence:\n%s\n%s", report("before", before), report("after ", onSnap.pathArchives)) + } +} + +// The archive guard above must not freeze archives: a real change to the +// route still has to be recaptured while the policy is off. +func TestEstimatedPositionsDisabledCycleStillRecordsRealChanges(t *testing.T) { + fx := seedEstimatedPositionsArchiveFixture(t) + before, err := fx.store.LoadPathArchives() + if err != nil { + t.Fatal(err) + } + // A hop that was only known by pubkey now has a real node row, so the + // recorded route genuinely differs from the capture. + if _, err := fx.srv.db.conn.Exec(`INSERT INTO nodes(public_key,name) VALUES('relay','Relay One')`); err != nil { + t.Fatal(err) + } + off := NewServer(fx.srv.db, disabledEstimatedPositionsConfig(), nil) + offEngine, err := newPingScoreHistoryEngine(off, fx.store, fx.clock.Now, fx.config) + if err != nil { + t.Fatal(err) + } + fx.clock.Advance(time.Hour) + if _, err := offEngine.Cycle(); err != nil { + t.Fatal(err) + } + after, err := fx.store.LoadPathArchives() + if err != nil { + t.Fatal(err) + } + old, next := before["allTime.farthestPing"], after["allTime.farthestPing"] + if next.Path.Branches[0].Points[0].Name != "Relay One" { + t.Fatalf("real route change was not recaptured: %s", mustJSON(t, next)) + } + if next.CapturedAt == old.CapturedAt { + t.Fatalf("recaptured evidence kept the old capture time: %s", next.CapturedAt) + } +} diff --git a/cmd/server/main.go b/cmd/server/main.go index e2026ac35..ff14e8554 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -90,6 +90,10 @@ func main() { // Load config cfg, err := LoadConfig(configDir) if err != nil { + var invalidPolicy *invalidEstimatedPositionsConfigError + if errors.As(err, &invalidPolicy) { + log.Fatalf("[config] fatal: %v", err) + } log.Printf("[config] warning: %v (using defaults)", err) } diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index efae40a1d..240ced49b 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -37,7 +37,7 @@ func routeDescriptions() map[string]routeMeta { return map[string]routeMeta{ // Config "GET /api/config/cache": {Summary: "Get cache configuration", Tag: "config"}, - "GET /api/config/client": {Summary: "Get client configuration", Tag: "config"}, + "GET /api/config/client": {Summary: "Client-visible operator configuration", Description: "Includes estimatedPositions: {enabled: boolean}, the effective server-startup policy for neighbor-derived position estimates. Missing server configuration defaults to enabled; explicit false suppresses these estimates in node detail, live and archived paths, and estimate-dependent analytics. Reported GPS and independent IATA/name-match observer positioning are unchanged. Restart the server to change the policy.", Tag: "config"}, "GET /api/config/regions": {Summary: "Get configured regions", Tag: "config"}, "GET /api/config/theme": {Summary: "Get theme configuration", Description: "Returns color maps, CSS variables, and theme defaults.", Tag: "config"}, "GET /api/config/map": {Summary: "Get map configuration", Tag: "config"}, @@ -193,7 +193,7 @@ func routeDescriptions() map[string]routeMeta { Response: schemaRef("PacketPathResponse")}, "GET /api/iata-coords": {Summary: "Get IATA airport coordinates", Description: "Returns lat/lon for known airport codes (used for observer positioning).", Tag: "config"}, "GET /api/audio-lab/buckets": {Summary: "Audio lab frequency buckets", Description: "Returns frequency bucket data for audio analysis.", Tag: "analytics"}, - "GET /api/ping-scores": {Summary: "Ping-score highscore board", Description: "Global (not scoped by region/area) records and leaderboards derived from every ping-bot-triggering channel message ever seen: farthest reach, most hops, widest simultaneous spread, fastest full spread, and most airtime-efficient ping, plus which relay nodes and which observers appear most often. Computed from the same GetPacketPath + LoRa-airtime-estimate logic behind /api/packets/{hash}/path and refreshed on a background interval, so it may lag the very latest ping by a few minutes. Fields are omitted (not zero) until at least one qualifying ping has been recorded.", Tag: "packets", + "GET /api/ping-scores": {Summary: "Ping-score highscore board", Description: "Global (not scoped by region/area) records and leaderboards derived from every ping-bot-triggering channel message ever seen: farthest reach, most hops, widest simultaneous spread, fastest full spread, and most airtime-efficient ping, plus which relay nodes and which observers appear most often. Computed from the same getPacketPath + LoRa-airtime-estimate logic behind /api/packets/{hash}/path and refreshed on a background interval, so it may lag the very latest ping by a few minutes. Fields are omitted (not zero) until at least one qualifying ping has been recorded.", Tag: "packets", Response: schemaRef("PingScoresResponse")}, "GET /api/ping-scores/{hash}/path": {Summary: "Get a displayed ping record's saved path", Description: "Returns coherent live or archived path evidence for the current record slot. Archived capture time describes saved geometry, not necessarily the transmission time. Old expired observations cannot be reconstructed. Superseded slot/hash pairs return 404; invalid slots return 400. Current identity privacy rules apply to both sources; unavailable and initializing responses omit path.", Tag: "packets", QueryParams: []paramMeta{{Name: "record", Description: "allTime. or thisWeek.; kind is farthestPing, mostHopsPing, widestSpreadPing, fastestSpreadPing or mostEfficientPing", Type: "string", Required: true}}, @@ -508,7 +508,7 @@ func componentSchemas() map[string]interface{} { "type": "object", "description": "The station that produced a given branch's observation of a packet path, positioned from its own self-advertised GPS when known (same source as /api/observers), else its configured IATA code, else a weighted centroid of its positioned neighbors (see approx).", "properties": map[string]interface{}{ - "publicKey": str("Observer's mesh pubkey, when it has one (some bridge-type observers publish under a device name instead -- see the name-match fallback in GetPacketPath). Empty otherwise."), + "publicKey": str("Observer's mesh pubkey, when it has one (some bridge-type observers publish under a device name instead -- see the name-match fallback in getPacketPath). Empty otherwise."), "name": str("Observer display name."), "iata": str("Observer's configured IATA airport code, when set."), "role": str("Observer's own node role (e.g. repeater, room), when it's known as a mesh node itself -- not just an MQTT/API listener."), @@ -562,7 +562,7 @@ func componentSchemas() map[string]interface{} { }}, "PingScore": map[string]interface{}{ "type": "object", - "description": "One ping's computed highscore-relevant stats, derived from the same GetPacketPath + airtime-annotation logic behind /api/packets/{hash}/path.", + "description": "One ping's computed highscore-relevant stats, derived from the same getPacketPath + airtime-annotation logic behind /api/packets/{hash}/path.", "properties": map[string]interface{}{ "hash": str("The winning ping's hash; use /api/ping-scores/{hash}/path with its record slot for saved View Path evidence."), "sender": str("Display name of whoever sent the ping, when resolvable from the channel message."), @@ -669,8 +669,9 @@ func componentSchemas() map[string]interface{} { }, "AreaAnalyticsResponse": map[string]interface{}{ "type": "object", - "description": "Node density/health, cross-area bridge nodes, and position-fix coverage per configured Area (the drawn-polygon regions from the meshguide.dk sync, distinct from hashRegion scope adoption). Empty when no Areas are configured.", + "description": "Node density/health, cross-area bridge nodes, and position-fix coverage per configured Area. When estimatedPositions.enabled is false, returns estimatedPositionsEnabled:false and real density/bridgeNodes/unpositionedTotal only; positionGaps, estimatedNodes, and unpositionedNoNeighborFix are omitted because they were not evaluated.", "properties": map[string]interface{}{ + "estimatedPositionsEnabled": &openAPISchema{Type: "boolean", Description: "Present as false only when neighbor-derived position estimation is disabled by the operator."}, "density": map[string]interface{}{"type": "array", "items": schemaRef("AreaDensity")}, "bridgeNodes": map[string]interface{}{"type": "array", "items": schemaRef("AreaBridgeNode"), "description": "Top cross-area bridge nodes, ranked by how many other areas they reach."}, "positionGaps": map[string]interface{}{"type": "array", "items": schemaRef("AreaPositionGap")}, @@ -696,11 +697,12 @@ func componentSchemas() map[string]interface{} { }, "GPSSanityResponse": map[string]interface{}{ "type": "object", - "description": "Nodes whose self-reported GPS disagrees with a trusted cluster of their own RF neighbors.", + "description": "Nodes whose self-reported GPS disagrees with a trusted cluster of their own RF neighbors. When estimatedPositions.enabled is false, returns only estimatedPositionsEnabled:false; nodes, totalRealGps and evaluated are omitted, not reported as zero.", "properties": map[string]interface{}{ - "nodes": map[string]interface{}{"type": "array", "items": schemaRef("SuspiciousGPSNode"), "description": "Flagged nodes, sorted worst (largest distanceKm) first."}, - "totalRealGps": map[string]interface{}{"type": "integer", "description": "Every node with a real (non-zero) GPS fix -- the population this check ran over."}, - "evaluated": map[string]interface{}{"type": "integer", "description": "The subset of totalRealGps that had a trustworthy neighbor cluster to compare against."}, + "estimatedPositionsEnabled": &openAPISchema{Type: "boolean", Description: "Present as false only when neighbor-derived position estimation is disabled by the operator."}, + "nodes": map[string]interface{}{"type": "array", "items": schemaRef("SuspiciousGPSNode"), "description": "Flagged nodes, sorted worst (largest distanceKm) first."}, + "totalRealGps": map[string]interface{}{"type": "integer", "description": "Every node with a real (non-zero) GPS fix -- the population this check ran over."}, + "evaluated": map[string]interface{}{"type": "integer", "description": "The subset of totalRealGps that had a trustworthy neighbor cluster to compare against."}, }, }, "AllObserverNeighborsEntry": map[string]interface{}{ diff --git a/cmd/server/packet_path_airtime_test.go b/cmd/server/packet_path_airtime_test.go index 1b03e2b11..724b09e26 100644 --- a/cmd/server/packet_path_airtime_test.go +++ b/cmd/server/packet_path_airtime_test.go @@ -13,7 +13,7 @@ import ( // View Path's estimated LoRa Time-on-Air x distinct-relay-count for the // packet's whole flood, sourced from the in-memory PacketStore (same // formula as the Relay Airtime Share analytics metric, issue #1768) via -// the transmission ID GetPacketPath captures. Two observations record +// the transmission ID getPacketPath captures. Two observations record // PARTIALLY overlapping resolved_path relay sets -- the union (3 distinct // pubkeys) is what should feed the estimate, not either path alone. func TestHandlePacketPath_Airtime(t *testing.T) { diff --git a/cmd/server/packet_path_policy_test_helpers_test.go b/cmd/server/packet_path_policy_test_helpers_test.go new file mode 100644 index 000000000..62e3aa423 --- /dev/null +++ b/cmd/server/packet_path_policy_test_helpers_test.go @@ -0,0 +1,18 @@ +package main + +// The estimated-positions operator policy lives on *Server, never on the +// shared *DB handle, so every production path lookup passes the effective +// policy explicitly (see getPacketPath and getPacketPathsBulk). +// +// These always-estimating wrappers exist only so the pre-#315 path +// fixtures keep reading the way they did. They live in a _test.go file on +// purpose: a future production caller cannot reach them -- the server +// binary would not compile -- so no caller can hard-wire "estimates on" +// and silently bypass the operator's policy. +func (db *DB) testPacketPath(hash string, maxEdgeKm float64) (*PacketPathResponse, error) { + return db.getPacketPath(hash, maxEdgeKm, true) +} + +func (db *DB) testPacketPathsBulk(hashes []string, maxEdgeKm float64) (map[string]*PacketPathResponse, error) { + return db.getPacketPathsBulk(hashes, maxEdgeKm, true) +} diff --git a/cmd/server/ping_score_bulk.go b/cmd/server/ping_score_bulk.go index 514220b1a..53c12636a 100644 --- a/cmd/server/ping_score_bulk.go +++ b/cmd/server/ping_score_bulk.go @@ -7,7 +7,7 @@ import ( ) // This file holds additive, read-only bulk-query helpers for Ping Scores -// Phase 4B: batched siblings of GetPacketPath and nearestPositionedNeighbor +// Phase 4B: batched siblings of getPacketPath and nearestPositionedNeighbor // that answer the same question for many inputs in O(chunks) queries // instead of O(N). None of these are wired into main.go, the recomputer, or // the public API yet -- that's Phase 4C+. See buildPacketPathResponseFromReduction @@ -330,7 +330,7 @@ func (db *DB) nearestPositionedNeighborsChunk(targets []string, maxEdgeKm float6 return nil } -// GetPacketPathsBulk computes the same PacketPathResponse GetPacketPath +// getPacketPathsBulk computes the same PacketPathResponse getPacketPath // would for each hash in hashes. All branch-assembly logic is the single // shared buildPacketPathResponseFromReduction (db.go), so the two can never // silently diverge in output shape or field values. @@ -357,9 +357,12 @@ func (db *DB) nearestPositionedNeighborsChunk(targets []string, maxEdgeKm float6 // // A hash with no observation rows at all (never observed, or unknown to // this DB) is simply absent from the returned map -- not an error, -// matching GetPacketPath's own contract of returning an empty-Branches +// matching getPacketPath's own contract of returning an empty-Branches // response rather than erroring for an unknown hash. -func (db *DB) GetPacketPathsBulk(hashes []string, maxEdgeKm float64) (map[string]*PacketPathResponse, error) { +// getPacketPathsBulk takes the caller's immutable operator policy +// explicitly, for the same reason getPacketPath does: there is no +// always-estimating exported wrapper to bypass the #315 setting with. +func (db *DB) getPacketPathsBulk(hashes []string, maxEdgeKm float64, estimatesEnabled bool) (map[string]*PacketPathResponse, error) { // result is created and the empty-input check runs BEFORE the // hasResolvedPath schema check on purpose: an empty request should // short-circuit to an empty, error-free result without touching the @@ -489,18 +492,23 @@ func (db *DB) GetPacketPathsBulk(hashes []string, maxEdgeKm float64) (map[string } fallbackSet := make(map[string]bool) - for _, red := range reductions { - for pk := range collectPacketPathFallbackCandidates(red.first, red.best, nodeByPK, nodeByName) { - fallbackSet[pk] = true + if estimatesEnabled { + for _, red := range reductions { + for pk := range collectPacketPathFallbackCandidates(red.first, red.best, nodeByPK, nodeByName) { + fallbackSet[pk] = true + } } } fallbackList := make([]string, 0, len(fallbackSet)) for pk := range fallbackSet { fallbackList = append(fallbackList, pk) } - estimates, err := db.nearestPositionedNeighborsBulk(fallbackList, maxEdgeKm) - if err != nil { - return nil, fmt.Errorf("packet path bulk neighbor estimate: %w", err) + var estimates map[string]neighborEstimate + if estimatesEnabled { + estimates, err = db.nearestPositionedNeighborsBulk(fallbackList, maxEdgeKm) + if err != nil { + return nil, fmt.Errorf("packet path bulk neighbor estimate: %w", err) + } } neighborLookup := func(pk string) (neighborEstimate, bool) { e, ok := estimates[pk] diff --git a/cmd/server/ping_score_bulk_test.go b/cmd/server/ping_score_bulk_test.go index cea572bb7..ceb5ff152 100644 --- a/cmd/server/ping_score_bulk_test.go +++ b/cmd/server/ping_score_bulk_test.go @@ -16,7 +16,7 @@ import ( // --- query-count instrumentation (test-only) ------------------------------- // -// Proves GetPacketPathsBulk/nearestPositionedNeighborsBulk genuinely batch +// Proves getPacketPathsBulk/nearestPositionedNeighborsBulk genuinely batch // (query count scales with chunk count, not with hash/pubkey count) and // proves the VALUES-CTE parameter budget is exactly N per chunk of N // targets, not 2N -- by wrapping modernc.org/sqlite's real driver.Conn and @@ -114,7 +114,7 @@ func registerCountingDriver() { var countingDBNameCounter int64 // setupPacketPathCountingDB builds an isolated, query-counting v3-schema DB -// covering only what GetPacketPath/GetPacketPathsBulk/ +// covering only what getPacketPath/getPacketPathsBulk/ // nearestPositionedNeighbor/observationFingerprintsBulk touch. Deliberately // smaller than setupTestDB's schema and on its own driver registration, // since query-count instrumentation needs to intercept the actual @@ -160,23 +160,23 @@ func setupPacketPathCountingDB(t *testing.T) *DB { return db } -// --- GetPacketPathsBulk: golden equivalence against GetPacketPath ---------- +// --- getPacketPathsBulk: golden equivalence against getPacketPath ---------- -// TestGetPacketPathsBulk_MatchesGetPacketPath_RichFixture is the core +// TestGetPacketPathsBulk_MatchesgetPacketPath_RichFixture is the core // golden test: one DB seeded with five hashes covering First-vs-deepest- // branch divergence, weighted-neighbor-centroid fallback for both a hop // point and an observer, (0,0) null-island exclusion, ambiguous // name-match skipping, and observer-position source ordering (own GPS > // name match > IATA) -- plus one hash never inserted at all. All five are -// requested from GetPacketPathsBulk in a single call (forcing the shared +// requested from getPacketPathsBulk in a single call (forcing the shared // nodeByPK/nodeByName/neighbor-estimate resolution to run jointly across // all of them, exactly the risky part of this refactor) and each is -// compared field-for-field against an independent GetPacketPath call for +// compared field-for-field against an independent getPacketPath call for // that same hash. Every fixture keeps branch hop-counts distinct within a // hash -- resp.Branches is built from iterating a Go map, so if two // branches tied on Hops their relative order would be nondeterministic // between the single and bulk paths' independently-populated maps; that's -// pre-existing GetPacketPath behavior, not something this phase changes, +// pre-existing getPacketPath behavior, not something this phase changes, // so the fixtures simply avoid exercising it. func TestGetPacketPathsBulk_MatchesGetPacketPath_RichFixture(t *testing.T) { db := setupTestDB(t) @@ -250,9 +250,9 @@ func TestGetPacketPathsBulk_MatchesGetPacketPath_RichFixture(t *testing.T) { "bulkfirst0000001", "bulkapprox000001", "bulknull00000001", "bulkambig0000001", "bulkownpos000001", "bulkunknown0000x", } - bulk, err := db.GetPacketPathsBulk(hashes, 500) + bulk, err := db.testPacketPathsBulk(hashes, 500) if err != nil { - t.Fatalf("GetPacketPathsBulk: %v", err) + t.Fatalf("testPacketPathsBulk: %v", err) } if _, ok := bulk["bulkunknown0000x"]; ok { @@ -260,16 +260,16 @@ func TestGetPacketPathsBulk_MatchesGetPacketPath_RichFixture(t *testing.T) { } for _, hash := range hashes[:5] { - single, err := db.GetPacketPath(hash, 500) + single, err := db.testPacketPath(hash, 500) if err != nil { - t.Fatalf("GetPacketPath(%s): %v", hash, err) + t.Fatalf("testPacketPath(%s): %v", hash, err) } gotBulk, ok := bulk[hash] if !ok { t.Fatalf("bulk map missing %s", hash) } if !reflect.DeepEqual(gotBulk, single) { - t.Errorf("GetPacketPathsBulk(%s) != GetPacketPath(%s):\n bulk: %+v\n single: %+v", hash, hash, dumpPacketPathResponse(gotBulk), dumpPacketPathResponse(single)) + t.Errorf("testPacketPathsBulk(%s) != testPacketPath(%s):\n bulk: %+v\n single: %+v", hash, hash, dumpPacketPathResponse(gotBulk), dumpPacketPathResponse(single)) } } } @@ -315,9 +315,9 @@ func derefF(f *float64) interface{} { func TestGetPacketPathsBulk_EmptyInput(t *testing.T) { db := setupTestDB(t) defer db.Close() - result, err := db.GetPacketPathsBulk(nil, 0) + result, err := db.testPacketPathsBulk(nil, 0) if err != nil { - t.Fatalf("GetPacketPathsBulk(nil): %v", err) + t.Fatalf("testPacketPathsBulk(nil): %v", err) } if len(result) != 0 { t.Errorf("result = %+v, want empty map", result) @@ -329,9 +329,9 @@ func TestGetPacketPathsBulk_NoResolvedPath(t *testing.T) { defer db.Close() db.hasResolvedPathFlag.v.Store(false) - _, err := db.GetPacketPathsBulk([]string{"whatever"}, 0) + _, err := db.testPacketPathsBulk([]string{"whatever"}, 0) if err == nil { - t.Fatal("GetPacketPathsBulk with hasResolvedPath=false: want error, got nil") + t.Fatal("testPacketPathsBulk with hasResolvedPath=false: want error, got nil") } } @@ -346,7 +346,7 @@ func TestGetPacketPathsBulk_DuplicateHashesCollapse(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 1, 9.0, -88, '[]', 100)`) - result, err := db.GetPacketPathsBulk([]string{"bulkdup00000001", "BULKDUP00000001", "bulkdup00000001"}, 0) + result, err := db.testPacketPathsBulk([]string{"bulkdup00000001", "BULKDUP00000001", "bulkdup00000001"}, 0) if err != nil { t.Fatal(err) } @@ -355,7 +355,7 @@ func TestGetPacketPathsBulk_DuplicateHashesCollapse(t *testing.T) { } } -// --- GetPacketPathsBulk: query-count instrumentation ----------------------- +// --- getPacketPathsBulk: query-count instrumentation ----------------------- // TestGetPacketPathsBulk_QueryCountIndependentOfHashCount proves the whole // point of Phase 4B: total query count for a batch that fits in one chunk @@ -403,15 +403,15 @@ func TestGetPacketPathsBulk_QueryCountIndependentOfHashCount(t *testing.T) { smallHashes := seedPacketPathFixture("small", 3) resetBulkTestQueryLog() - if _, err := db.GetPacketPathsBulk(smallHashes, 0); err != nil { - t.Fatalf("GetPacketPathsBulk(small): %v", err) + if _, err := db.testPacketPathsBulk(smallHashes, 0); err != nil { + t.Fatalf("testPacketPathsBulk(small): %v", err) } smallCount := len(bulkTestQueryLog()) largeHashes := seedPacketPathFixture("large", 30) resetBulkTestQueryLog() - if _, err := db.GetPacketPathsBulk(largeHashes, 0); err != nil { - t.Fatalf("GetPacketPathsBulk(large): %v", err) + if _, err := db.testPacketPathsBulk(largeHashes, 0); err != nil { + t.Fatalf("testPacketPathsBulk(large): %v", err) } largeCount := len(bulkTestQueryLog()) @@ -426,8 +426,8 @@ func TestGetPacketPathsBulk_QueryCountIndependentOfHashCount(t *testing.T) { // concrete rather than just "not proportional". resetBulkTestQueryLog() for _, h := range largeHashes { - if _, err := db.GetPacketPath(h, 0); err != nil { - t.Fatalf("GetPacketPath(%s): %v", h, err) + if _, err := db.testPacketPath(h, 0); err != nil { + t.Fatalf("testPacketPath(%s): %v", h, err) } } naiveCount := len(bulkTestQueryLog()) @@ -708,15 +708,15 @@ func TestNearestPositionedNeighborsBulk_ChunksAcrossBoundary(t *testing.T) { // // setupTestDB/setupTestDBV2 don't cover this: setupTestDB always forces // isV3Flag true, and setupTestDBV2's observations table has no -// resolved_path column at all (it predates GetPacketPath entirely), so -// GetPacketPath's own legacy-schema SQL branch has never actually been +// resolved_path column at all (it predates getPacketPath entirely), so +// getPacketPath's own legacy-schema SQL branch has never actually been // exercised by a Go test in this codebase -- a pre-existing gap, not -// something Phase 4B changes. GetPacketPathsBulk adds a second copy of +// something Phase 4B changes. getPacketPathsBulk adds a second copy of // that branching (isV3() ? v3 query : legacy query), so this test builds // a minimal legacy-shaped schema (observer_id/observer_name columns // instead of an observers table join, but WITH resolved_path present) -// to prove the legacy query text GetPacketPathsBulk issues is at least -// syntactically correct and produces the same result GetPacketPath would. +// to prove the legacy query text getPacketPathsBulk issues is at least +// syntactically correct and produces the same result getPacketPath would. func setupPacketPathLegacyTestDB(t *testing.T) *DB { t.Helper() conn, err := sql.Open("sqlite", ":memory:") @@ -764,31 +764,31 @@ func TestGetPacketPathsBulk_LegacySchemaMatchesGetPacketPath(t *testing.T) { db.conn.Exec(`INSERT INTO observations (transmission_id, observer_id, observer_name, snr, rssi, path_json, resolved_path, timestamp) VALUES (1, 'legacyobs2', 'Legacy Observer Two', 4.0, -95, '["aa","bb"]', '["pklegacy1","pklegacy2"]', 1736935260)`) - single, err := db.GetPacketPath("legacyhash000001", 0) + single, err := db.testPacketPath("legacyhash000001", 0) if err != nil { - t.Fatalf("GetPacketPath: %v", err) + t.Fatalf("testPacketPath: %v", err) } if len(single.Branches) != 2 { t.Fatalf("sanity check failed: single.Branches = %+v, want 2 (fixture problem, not the code under test)", single.Branches) } - bulk, err := db.GetPacketPathsBulk([]string{"legacyhash000001"}, 0) + bulk, err := db.testPacketPathsBulk([]string{"legacyhash000001"}, 0) if err != nil { - t.Fatalf("GetPacketPathsBulk: %v", err) + t.Fatalf("testPacketPathsBulk: %v", err) } got, ok := bulk["legacyhash000001"] if !ok { t.Fatal("bulk map missing legacyhash000001") } if !reflect.DeepEqual(got, single) { - t.Errorf("GetPacketPathsBulk != GetPacketPath on the legacy schema branch:\n bulk: %+v\n single: %+v", dumpPacketPathResponse(got), dumpPacketPathResponse(single)) + t.Errorf("testPacketPathsBulk != testPacketPath on the legacy schema branch:\n bulk: %+v\n single: %+v", dumpPacketPathResponse(got), dumpPacketPathResponse(single)) } } // ============================================================================ // Fix round 2 (review of commit 511438d5): chunking correctness for // resolveNodesByPubkey/resolveNodesByName/nearestPositionedNeighborsChunk's -// candidate lookup, deterministic tie-breaks, and the GetPacketPathsBulk +// candidate lookup, deterministic tie-breaks, and the getPacketPathsBulk // empty-input fast path. // ============================================================================ @@ -1147,7 +1147,7 @@ func TestGetPacketPath_TieBreak_SameHopsDifferentPath(t *testing.T) { VALUES (1, 1, 9.0, -88, '["bb"]', '["pkTieB"]', 100)`) for i := 0; i < 5; i++ { - resp, err := db.GetPacketPath("tiehops0000001", 0) + resp, err := db.testPacketPath("tiehops0000001", 0) if err != nil { t.Fatal(err) } @@ -1182,7 +1182,7 @@ func TestGetPacketPath_TieBreak_SameEarliestTimestamp(t *testing.T) { VALUES (1, 2, 4.0, -95, '[]', 100)`) for i := 0; i < 5; i++ { - resp, err := db.GetPacketPath("tiefirst0000001", 0) + resp, err := db.testPacketPath("tiefirst0000001", 0) if err != nil { t.Fatal(err) } @@ -1229,7 +1229,7 @@ func TestGetPacketPath_BranchSortTieBreak_MultipleBranchesSameHops(t *testing.T) var firstOrder []string for i := 0; i < 5; i++ { - resp, err := db.GetPacketPath("tiebranch000001", 0) + resp, err := db.testPacketPath("tiebranch000001", 0) if err != nil { t.Fatal(err) } @@ -1261,11 +1261,11 @@ func TestGetPacketPath_BranchSortTieBreak_MultipleBranchesSameHops(t *testing.T) } } -// TestGetPacketPathsBulk_MatchesGetPacketPath_WithHopsAndTimestampTies is a +// TestGetPacketPathsBulk_MatchesgetPacketPath_WithHopsAndTimestampTies is a // golden-equivalence test specifically for the tie-break paths: three // branches tied on Hops (observer-key order) plus a First/earliest-timestamp -// tie. GetPacketPathsBulk must resolve to the exact same PacketPathResponse -// GetPacketPath does, byte-for-byte, proving the deterministic tie-breaks +// tie. getPacketPathsBulk must resolve to the exact same PacketPathResponse +// getPacketPath does, byte-for-byte, proving the deterministic tie-breaks // (obsID-based fold, observer-key-ascending build order, stable Hops sort) // are genuinely shared between the two paths and not just each // independently "consistent with itself". @@ -1286,11 +1286,11 @@ func TestGetPacketPathsBulk_MatchesGetPacketPath_WithHopsAndTimestampTies(t *tes db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES (1, 3, 9.0, -88, '["aa","bb"]', 200)`) - single, err := db.GetPacketPath("tiebulk00000001", 0) + single, err := db.testPacketPath("tiebulk00000001", 0) if err != nil { t.Fatal(err) } - bulk, err := db.GetPacketPathsBulk([]string{"tiebulk00000001"}, 0) + bulk, err := db.testPacketPathsBulk([]string{"tiebulk00000001"}, 0) if err != nil { t.Fatal(err) } @@ -1299,11 +1299,11 @@ func TestGetPacketPathsBulk_MatchesGetPacketPath_WithHopsAndTimestampTies(t *tes t.Fatal("bulk map missing tiebulk00000001") } if !reflect.DeepEqual(got, single) { - t.Errorf("GetPacketPathsBulk != GetPacketPath under ties:\n bulk: %+v\n single: %+v", dumpPacketPathResponse(got), dumpPacketPathResponse(single)) + t.Errorf("testPacketPathsBulk != testPacketPath under ties:\n bulk: %+v\n single: %+v", dumpPacketPathResponse(got), dumpPacketPathResponse(single)) } } -// --- GetPacketPathsBulk empty-input fast path ------------------------------- +// --- getPacketPathsBulk empty-input fast path ------------------------------- // TestGetPacketPathsBulk_EmptyInputSkipsSchemaCheck proves the fixed // ordering: an empty hashes slice returns an empty map with no error even @@ -1317,9 +1317,9 @@ func TestGetPacketPathsBulk_EmptyInputSkipsSchemaCheck(t *testing.T) { db.hasResolvedPathFlag.v.Store(false) // simulates a schema without resolved_path resetBulkTestQueryLog() - result, err := db.GetPacketPathsBulk(nil, 0) + result, err := db.testPacketPathsBulk(nil, 0) if err != nil { - t.Fatalf("GetPacketPathsBulk(nil) on a no-resolved_path schema: want no error, got %v", err) + t.Fatalf("testPacketPathsBulk(nil) on a no-resolved_path schema: want no error, got %v", err) } if len(result) != 0 { t.Errorf("result = %+v, want empty map", result) @@ -1331,8 +1331,8 @@ func TestGetPacketPathsBulk_EmptyInputSkipsSchemaCheck(t *testing.T) { // Sanity check: a NON-empty request against the same no-resolved_path // DB must still error -- the fast path is empty-input-specific, not a // blanket skip of the schema check. - _, err = db.GetPacketPathsBulk([]string{"whatever"}, 0) + _, err = db.testPacketPathsBulk([]string{"whatever"}, 0) if err == nil { - t.Fatal("GetPacketPathsBulk([]string{\"whatever\"}) on a no-resolved_path schema: want an error") + t.Fatal("testPacketPathsBulk([]string{\"whatever\"}) on a no-resolved_path schema: want an error") } } diff --git a/cmd/server/ping_score_history.go b/cmd/server/ping_score_history.go index 61467426f..08e5fa7ac 100644 --- a/cmd/server/ping_score_history.go +++ b/cmd/server/ping_score_history.go @@ -260,7 +260,7 @@ type PingScoreHistoryIntegrity struct { DetectedAt string // RFC3339 // Relevant to "initial-backfill-incomplete" (a later phase populates - // these once GetPacketPathsBulk exists; Phase 4A only provides the + // these once getPacketPathsBulk exists; Phase 4A only provides the // storage for them). TotalTriggers int ScoredCount int diff --git a/cmd/server/ping_score_history_convert.go b/cmd/server/ping_score_history_convert.go index f3d390fdd..6eb0ae1c0 100644 --- a/cmd/server/ping_score_history_convert.go +++ b/cmd/server/ping_score_history_convert.go @@ -99,7 +99,7 @@ type PingScoreHistoryEntryState struct { } // pingScoreHistoryEntryFromScore converts a trigger + freshly computed -// score (nil when GetPacketPath/GetPacketPathsBulk produced no usable path +// score (nil when getPacketPath/getPacketPathsBulk produced no usable path // this cycle -- see buildPingScoreFromPath's own nil contract) + observation // fingerprint + carried-through state into a brand-new PingScoreHistoryEntry // ready to persist. Used for a tx_id that has no existing entry yet; see diff --git a/cmd/server/ping_score_history_engine.go b/cmd/server/ping_score_history_engine.go index fd360c9c7..8daa355e1 100644 --- a/cmd/server/ping_score_history_engine.go +++ b/cmd/server/ping_score_history_engine.go @@ -67,8 +67,8 @@ type pingScoreHistoryEngineConfig struct { // meaning here. RetentionDuration time.Duration - // MaxEdgeKm is threaded through to GetPacketPathsBulk's geo-sanity - // filter, matching GetPacketPath's/nearestPositionedNeighbor's own + // MaxEdgeKm is threaded through to getPacketPathsBulk's geo-sanity + // filter, matching getPacketPath's/nearestPositionedNeighbor's own // existing parameter and its own documented convention: <=0 // deliberately DISABLES the geo-sanity filter (not an error) -- so // unlike the other three fields, a non-positive MaxEdgeKm is NOT @@ -233,7 +233,7 @@ func needsFingerprintCheck(e PingScoreHistoryEntry) bool { // 2. plan reconciliation (planPingScoreHistoryReconcile) // 3. select fingerprint- and deep-sweep candidates // 4. observationFingerprintsBulk -// 5. GetPacketPathsBulk +// 5. getPacketPathsBulk // 6. buildPingScoreFromPath // 7. merge into a cloned candidate index // 8. history snapshot built + names enriched (buildPingScoresSnapshotFromHistory) @@ -330,7 +330,7 @@ func (e *pingScoreHistoryEngine) Cycle() (*PingScoresSnapshot, error) { // unreconstructability") -- an Unscorable entry that has JUST // crossed retention, or crossed it long ago but was never // actually re-attempted, REMAINS eligible below and gets - // exactly one real GetPacketPathsBulk-backed attempt; only + // exactly one real getPacketPathsBulk-backed attempt; only // THAT attempt's outcome (via maybeMarkPermanentlyUnreconstructable, // called from the deep-sweep merge loop) can ever set this // flag. It still settles normally (this check runs strictly @@ -400,7 +400,7 @@ func (e *pingScoreHistoryEngine) Cycle() (*PingScoresSnapshot, error) { } } - // --- 5. GetPacketPathsBulk: the union of every hash needing a real + // --- 5. getPacketPathsBulk: the union of every hash needing a real // path recompute this cycle (reconciliation + fingerprint-changed + // deep-sweep) --- txIDsNeedingPath := map[int64]bool{} @@ -424,12 +424,12 @@ func (e *pingScoreHistoryEngine) Cycle() (*PingScoresSnapshot, error) { } var pathResults map[string]*PacketPathResponse if len(hashesList) > 0 { - pathResults, err = e.server.db.GetPacketPathsBulk(hashesList, e.config.MaxEdgeKm) + pathResults, err = e.server.db.getPacketPathsBulk(hashesList, e.config.MaxEdgeKm, e.server.estimatedPositionsEnabled()) if err != nil { return nil, fmt.Errorf("ping score history cycle: bulk path query: %w", err) } } - // pathResultFor normalizes the lookup key ONLY -- GetPacketPathsBulk's + // pathResultFor normalizes the lookup key ONLY -- getPacketPathsBulk's // result map is always keyed by the lowercased hash (see its own // implementation), but ping_triggers.hash (and therefore trigger.hash) // may not be. This must NEVER be used to change what gets PERSISTED: @@ -527,7 +527,7 @@ func (e *pingScoreHistoryEngine) Cycle() (*PingScoresSnapshot, error) { // Settled entry's fingerprint at all, so this is precisely where such // a change becomes visible again. LastDeepSweptAt is updated // unconditionally for every entry in this batch once the shared - // GetPacketPathsBulk call above has succeeded (Cycle is all-or-nothing, + // getPacketPathsBulk call above has succeeded (Cycle is all-or-nothing, // so reaching this loop at all means it did) -- an empty per-hash // result for one entry doesn't mean the SWEEP failed, only that this // particular attempt found nothing; the rotation must still advance so @@ -674,8 +674,8 @@ func (e *pingScoreHistoryEngine) Cycle() (*PingScoresSnapshot, error) { // Cycle's own snapshot-build step already does), so display names on // the returned records/leaderboards are resolved fresh, not frozen. // -// Explicitly NOT done here: no per-trigger GetPacketPath calls, no -// GetPacketPathsBulk/path-recompute of any kind, no reconciliation +// Explicitly NOT done here: no per-trigger getPacketPath calls, no +// getPacketPathsBulk/path-recompute of any kind, no reconciliation // (planPingScoreHistoryReconcile), no fingerprint or deep-sweep work, and // no persistence (no store write of any kind). This makes QuickSnapshot // CHEAPER than a full Cycle -- it skips every DB round-trip that scales @@ -736,7 +736,7 @@ func (e *pingScoreHistoryEngine) QuickSnapshot() (*PingScoresSnapshot, error) { // review of a1c3022d: "alder alene er ikke bevis for permanent // unreconstructability" -- age alone is not proof of permanent // unreconstructability). Requires ALL of: -// - score == nil: THIS cycle's real GetPacketPathsBulk-backed recompute +// - score == nil: THIS cycle's real getPacketPathsBulk-backed recompute // attempt (not a prediction) found nothing. // - existingEntry.Unscorable == true (the PRE-cycle state, before this // merge): this tx_id has never had a successful computation, ever -- diff --git a/cmd/server/ping_score_history_engine_test.go b/cmd/server/ping_score_history_engine_test.go index 69352be44..4ad8f836b 100644 --- a/cmd/server/ping_score_history_engine_test.go +++ b/cmd/server/ping_score_history_engine_test.go @@ -443,7 +443,7 @@ func TestCycle_DeepSweepPathChange(t *testing.T) { before := *entry1.FarthestKm // Move pingobsb far away -- doesn't touch observations/transmissions - // at all, so the fingerprint stays unchanged, but GetPacketPathsBulk's + // at all, so the fingerprint stays unchanged, but getPacketPathsBulk's // recomputed distance must differ. if _, err := fx.srv.db.conn.Exec(`UPDATE nodes SET lat = 10.0, lon = 10.0 WHERE public_key = 'pingobsb'`); err != nil { t.Fatal(err) @@ -467,7 +467,7 @@ func TestCycle_DeepSweepPathChange(t *testing.T) { // setupSettledPingWithGoneData settles one ping, then deletes its // underlying observations/transmission rows (simulating the ingestor // having pruned the raw packet past its retention window) -- so a -// subsequent deep-sweep's GetPacketPathsBulk call finds nothing for it, +// subsequent deep-sweep's getPacketPathsBulk call finds nothing for it, // while ping_triggers (and the history entry) still has the row. func setupSettledPingWithGoneData(t *testing.T, fx *engineFixture, hash, firstSeen string) int64 { t.Helper() @@ -1030,7 +1030,7 @@ func TestCycle_BulkPathQueryFailure_LeavesEverythingUnchanged(t *testing.T) { // fingerprint check (ToCompute path) -- query #1 = fetchPingTriggers, // query #2 = observationFingerprintsBulk for this new tx_id's // fingerprint (needed so the fresh entry's persisted fingerprint is - // accurate -- see Cycle's doc comment), query #3 = GetPacketPathsBulk. + // accurate -- see Cycle's doc comment), query #3 = getPacketPathsBulk. seedFaultTrigger(t, fx, 1, "faultbulk00001") before := captureEngineState(t, fx) @@ -1241,7 +1241,7 @@ func TestCycle_YoungUnscorableEntry_StillDeepSweptAfterSettle(t *testing.T) { // the central regression test for the fix-round-3 review: age alone must // NEVER exclude an entry from deep-sweep. It proves an Unscorable entry // keeps getting REAL sweeps (LastDeepSweptAt advancing, a real -// GetPacketPathsBulk-backed attempt) for as long as it takes to cross +// getPacketPathsBulk-backed attempt) for as long as it takes to cross // retention, that crossing retention with an empty result sets // PermanentlyUnreconstructable ATOMICALLY on that same cycle, and that the // cycle immediately after does NOT get yet another real sweep. @@ -1308,7 +1308,7 @@ func TestCycle_EntryCrossingRetention_GetsExactlyOneRealSweepThenFlagged(t *test // TestCycle_AfterFlagSet_NoFurtherPathQueryIssued proves the exclusion at // the query level: once evidence has actually been gathered (not merely -// once retention has passed), a later cycle issues NO GetPacketPathsBulk +// once retention has passed), a later cycle issues NO getPacketPathsBulk // call carrying this entry's hash, even though it's the ONLY entry that // would otherwise be deep-sweep eligible. func TestCycle_AfterFlagSet_NoFurtherPathQueryIssued(t *testing.T) { @@ -1430,7 +1430,7 @@ func TestCycle_DeepSweepQueryFailureDuringEvidenceGathering_NoFlagNoGapChange(t } // This cycle's query order: #1 fetchPingTriggers, #2 observationFingerprintsBulk - // (for the deep-sweep-eligible entry), #3 GetPacketPathsBulk -- fail + // (for the deep-sweep-eligible entry), #3 getPacketPathsBulk -- fail // exactly the bulk path fetch, matching TestCycle_BulkPathQueryFailure_LeavesEverythingUnchanged's // own established pattern. resetBulkTestQueryLog() @@ -2276,7 +2276,7 @@ func TestCycle_UnchangedPermanentPopulation_NoGapWriteAcrossCycles(t *testing.T) // --- Fix 3: hash normalization at bulk-result lookup ------------------------ // seedMixedCaseTrigger inserts a transmission with hash stored LOWERCASE -// (the only form GetPacketPath/GetPacketPathsBulk's `t.hash = LOWER(?)` +// (the only form getPacketPath/getPacketPathsBulk's `t.hash = LOWER(?)` // queries can ever match -- both callsites lowercase their query input, // so a mixed-case-STORED transmissions.hash could never be found at all, // mixed case or not) paired with a ping_triggers row whose hash is a @@ -2317,7 +2317,7 @@ func TestCycle_MixedCaseTriggerHashScoredCorrectly(t *testing.T) { t.Fatal("index missing entry for mixed-case hash trigger") } if entry.Unscorable { - t.Error("Unscorable = true, want false -- GetPacketPathsBulk's result must be found despite hash casing") + t.Error("Unscorable = true, want false -- testPacketPathsBulk's result must be found despite hash casing") } if entry.StationCount != 2 { t.Errorf("StationCount = %d, want 2", entry.StationCount) @@ -2582,7 +2582,7 @@ func TestCycle_ZeroRetentionDuration_DisablesDataPrunedAndBootstrapIntegrity(t * // ============================================================================ // Fase 5B (production-wiring design, approved v5): QuickSnapshot -- a // read-only, synchronous snapshot built from whatever is ALREADY -// persisted plus a fresh trigger fetch, with no GetPacketPathsBulk, +// persisted plus a fresh trigger fetch, with no getPacketPathsBulk, // reconciliation, or persistence. Isolated tests only -- no worker, // healthz, or main.go wiring in this phase. // ============================================================================ diff --git a/cmd/server/ping_score_history_index.go b/cmd/server/ping_score_history_index.go index 5b2788083..163f0e99b 100644 --- a/cmd/server/ping_score_history_index.go +++ b/cmd/server/ping_score_history_index.go @@ -116,7 +116,7 @@ func (idx *pingScoreHistoryIndex) Entries() []PingScoreHistoryEntry { // against a fresh ping_triggers read: what a recompute cycle needs to do, // with no I/O performed and nothing mutated. Phase 4C only builds this // plan (planPingScoreHistoryReconcile); Phase 4D+ is what actually acts on -// it (calling GetPacketPathsBulk for ToCompute, deleting ToDelete from the +// it (calling getPacketPathsBulk for ToCompute, deleting ToDelete from the // store, etc). type pingScoreHistoryReconcilePlan struct { // ToCompute is every trigger needing a fresh computation this cycle: diff --git a/cmd/server/ping_score_history_paths.go b/cmd/server/ping_score_history_paths.go index d15d43d3c..3d770b27c 100644 --- a/cmd/server/ping_score_history_paths.go +++ b/cmd/server/ping_score_history_paths.go @@ -212,6 +212,43 @@ func newPingScorePathArchive(key string, score *PingScore, path *PacketPathRespo return PingScorePathArchive{RecordKey: key, Hash: score.Hash, Timestamp: score.Timestamp, CapturedAt: now.UTC().Format(time.RFC3339), Path: immutable}, true } +// pingScorePathArchiveFingerprint is the comparison form of a captured +// path: the serialization writePingScorePathArchives would store, viewed +// through whatever filter the operator policy applies to responses. +// +// While estimated positions are DISABLED, freshly captured paths carry no +// neighbor-derived geometry at all, so a byte comparison against an +// archive captured while they were enabled always differs -- and the +// engine would replace the stored evidence with an estimate-free copy and +// bump its CapturedAt, destroying the recorded approximation for good once +// the raw observations expire. The policy filters request-owned copies +// only (issue #315 point 5), so compare both sides through the same strip +// the response path uses: when the only difference is the estimate +// geometry, the old capture is still the right evidence and is kept +// untouched. Any real change to the route still differs and still +// replaces it. +func pingScorePathArchiveFingerprint(path *PacketPathResponse, estimatesEnabled bool) (string, bool) { + b, err := boundedPingPathJSON(path) + if err != nil { + return "", false + } + if estimatesEnabled { + return string(b), true + } + // Strip a clone, never the caller's path: old.Path is the live + // in-memory archive the Ping Scores snapshot still serves. + var clone PacketPathResponse + if err := json.Unmarshal(b, &clone); err != nil { + return "", false + } + stripEstimatedPositions(&clone) + stripped, err := boundedPingPathJSON(&clone) + if err != nil { + return "", false + } + return string(stripped), true +} + func (e *pingScoreHistoryEngine) pathsForRecords(snap *PingScoresSnapshot, paths map[string]*PacketPathResponse, attempted map[string]bool, now time.Time) (map[string]PingScorePathArchive, bool, error) { slots := pingScoreRecordSlots(snap) missing, seen := []string{}, map[string]bool{} @@ -224,7 +261,7 @@ func (e *pingScoreHistoryEngine) pathsForRecords(snap *PingScoresSnapshot, paths } if len(missing) > 0 { // At most ten distinct displayed hashes, in one existing bulk helper. - extra, err := e.server.db.GetPacketPathsBulk(missing, e.config.MaxEdgeKm) + extra, err := e.server.db.getPacketPathsBulk(missing, e.config.MaxEdgeKm, e.server.estimatedPositionsEnabled()) if err != nil { return nil, false, fmt.Errorf("record path capture: %w", err) } @@ -238,6 +275,7 @@ func (e *pingScoreHistoryEngine) pathsForRecords(snap *PingScoresSnapshot, paths } } } + estimatesEnabled := e.server.estimatedPositionsEnabled() out := make(map[string]PingScorePathArchive, len(slots)) for key, score := range slots { old, oldOK := e.pathArchives[key] @@ -245,9 +283,9 @@ func (e *pingScoreHistoryEngine) pathsForRecords(snap *PingScoresSnapshot, paths if next, ok := newPingScorePathArchive(key, score, paths[strings.ToLower(score.Hash)], now); ok { // Identical evidence doesn't create a new capture time or DB write. if oldOK { - a, _ := boundedPingPathJSON(&old.Path) - b, _ := boundedPingPathJSON(&next.Path) - if string(a) == string(b) { + a, aOK := pingScorePathArchiveFingerprint(&old.Path, estimatesEnabled) + b, bOK := pingScorePathArchiveFingerprint(&next.Path, estimatesEnabled) + if aOK && bOK && a == b { next = old } } diff --git a/cmd/server/ping_score_history_record_paths_test.go b/cmd/server/ping_score_history_record_paths_test.go index 5ae678776..5c8729869 100644 --- a/cmd/server/ping_score_history_record_paths_test.go +++ b/cmd/server/ping_score_history_record_paths_test.go @@ -78,7 +78,7 @@ func TestPingRecordDistanceSurvivesKnownIATAFallback(t *testing.T) { if _, err := fx.srv.db.conn.Exec(statement); err != nil { t.Fatal(err) } - path, err := fx.srv.db.GetPacketPath(before.FarthestPing.Hash, EstimateMaxEdgeKm) + path, err := fx.srv.db.testPacketPath(before.FarthestPing.Hash, EstimateMaxEdgeKm) if err != nil { t.Fatal(err) } @@ -153,7 +153,7 @@ func TestPingRecordArchivesSurviveRetentionAndRestart(t *testing.T) { if !reflect.DeepEqual(old, restarted.pathArchives["allTime.farthestPing"]) { t.Fatal("restart lost archived positions") } - live, err := fx.srv.db.GetPacketPath(old.Hash, EstimateMaxEdgeKm) + live, err := fx.srv.db.testPacketPath(old.Hash, EstimateMaxEdgeKm) if err != nil || len(live.Branches) != 0 { t.Fatal("fixture did not lose live path") } diff --git a/cmd/server/ping_score_history_snapshot.go b/cmd/server/ping_score_history_snapshot.go index a9de2ce37..251662701 100644 --- a/cmd/server/ping_score_history_snapshot.go +++ b/cmd/server/ping_score_history_snapshot.go @@ -3,7 +3,7 @@ // // buildPingScoresSnapshotFromHistory produces the same PingScoresSnapshot // shape computeAllPingScores does, but from already-persisted history -// entries instead of a live GetPacketPath call per trigger. It is NOT yet +// entries instead of a live getPacketPath call per trigger. It is NOT yet // wired into the production recomputer (Phase 4D+ cuts over) -- it exists // now so it can be validated against computeAllPingScores' own output via // equivalence tests (ping_score_history_snapshot_test.go) before any @@ -40,9 +40,9 @@ func (e *PingScoreHistoryMismatchError) Error() string { // pubkeys -- v3 schema only. // // IMPORTANT, investigated for this phase: this mirrors the name SOURCE -// GetPacketPath's live Observer.Name field actually uses (b.observerName, +// getPacketPath's live Observer.Name field actually uses (b.observerName, // read directly off the observers table via the observer_idx join in -// GetPacketPath's own query) -- it is observers.name, NOT nodes.name. +// getPacketPath's own query) -- it is observers.name, NOT nodes.name. // These are two independently maintained fields (an observer's own // self-reported name at packet-hearing time vs. a node's self-reported // name at ADVERT time) that usually agree for the same physical device but @@ -50,7 +50,7 @@ func (e *PingScoreHistoryMismatchError) Error() string { // for RelayLeaderboard names) queries nodes.name instead, matching // computeAllPingScores' own existing relay-name lookup exactly. Neither // path consults inactive_nodes anywhere -- grep confirms nothing in -// GetPacketPath, resolveNodesByPubkey/resolveNodesByName, or +// getPacketPath, resolveNodesByPubkey/resolveNodesByName, or // namesAndRolesForPubkeys ever references that table, so this function // doesn't either; today's ping-score name resolution has never fallen back // to it, and this preserves that. @@ -65,7 +65,7 @@ func (e *PingScoreHistoryMismatchError) Error() string { // correctness here. // // Legacy (non-v3) schema has no separate observers table at all -- -// GetPacketPath instead reads observer_name directly off each OBSERVATION +// getPacketPath instead reads observer_name directly off each OBSERVATION // row for that schema, a value that can vary per observation and isn't // otherwise indexed for a bulk historical lookup like this one. Returns an // empty map for that case (a known limitation, not silently pretended @@ -136,7 +136,7 @@ func (db *DB) observerNamesByPubkey(pubkeys []string) map[string]string { // job, not this function's). A trigger with no corresponding entry yet is // likewise just excluded (nothing to show for it this cycle), matching // computeAllPingScores' own "score == nil -> continue" behavior for a -// trigger GetPacketPath couldn't resolve. TotalPings is len(triggers) -- +// trigger getPacketPath couldn't resolve. TotalPings is len(triggers) -- // the fresh, live count -- never len(entries) or an index's Len(). // // A trigger whose hash or timestamp doesn't match its history entry's own diff --git a/cmd/server/ping_score_history_snapshot_test.go b/cmd/server/ping_score_history_snapshot_test.go index 32036b7c3..5a7293801 100644 --- a/cmd/server/ping_score_history_snapshot_test.go +++ b/cmd/server/ping_score_history_snapshot_test.go @@ -10,7 +10,7 @@ import ( // --- shared equivalence helpers ------------------------------------------- // historyEntriesFromLiveScores computes each trigger's score via the SAME -// live path (computePingScore -> GetPacketPath) computeAllPingScores uses, +// live path (computePingScore -> getPacketPath) computeAllPingScores uses, // then converts each into a PingScoreHistoryEntry via // pingScoreHistoryEntryFromScore -- simulating "persist today's live // computation, unchanged, as history" for the equivalence tests below. @@ -224,7 +224,7 @@ func TestPingScoresHistoryEquivalence_LoneStationHasNilAirtime(t *testing.T) { } // TestBuildPingScoresSnapshotFromHistory_DerivedKmPerSecondAirtime -// constructs an entry directly (bypassing the live GetPacketPath path, to +// constructs an entry directly (bypassing the live getPacketPath path, to // isolate this specific derivation) with both FarthestKm and a positive // AirtimeMs persisted, and confirms the resulting snapshot record carries // the correctly-derived KmPerSecondAirtime -- not merely that @@ -257,7 +257,7 @@ func TestBuildPingScoresSnapshotFromHistory_DerivedKmPerSecondAirtime(t *testing // TestBuildPingScoresSnapshotFromHistory_UnscorableEntrySkippedButCounted // covers a trigger whose persisted entry is Unscorable=true (this cycle's -// -- or every cycle's -- GetPacketPath produced nothing usable): it must +// -- or every cycle's -- getPacketPath produced nothing usable): it must // be excluded from every record/leaderboard, yet still counted in // TotalPings (which reflects the live trigger table, not "how many were // actually scored"). diff --git a/cmd/server/ping_score_path.go b/cmd/server/ping_score_path.go index 636682fdb..61dc443c2 100644 --- a/cmd/server/ping_score_path.go +++ b/cmd/server/ping_score_path.go @@ -47,7 +47,7 @@ func (s *Server) handlePingScorePath(w http.ResponseWriter, r *http.Request) { result.Status, result.CapturedAt = "archived", a.CapturedAt } else if s.db != nil { var err error - path, err = s.db.GetPacketPath(score.Hash, EstimateMaxEdgeKm) + path, err = s.db.getPacketPath(score.Hash, EstimateMaxEdgeKm, s.estimatedPositionsEnabled()) if err != nil { writeError(w, http.StatusInternalServerError, "could not load ping record path") return @@ -74,6 +74,9 @@ func (s *Server) handlePingScorePath(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, "could not read ping record path") return } + if !s.estimatedPositionsEnabled() { + stripEstimatedPositions(&visible) + } filtered, err := s.filterPingScorePath(r.Context(), &visible) if err != nil { // A failed live name lookup cannot turn cached private names public. diff --git a/cmd/server/ping_scores.go b/cmd/server/ping_scores.go index 1b893af5c..cbe9930a9 100644 --- a/cmd/server/ping_scores.go +++ b/cmd/server/ping_scores.go @@ -4,7 +4,7 @@ // message to ping_triggers (tx_id, hash, channel_hash, sender, first_seen // -- see cmd/ingestor/ping_triggers.go and internal/dbschema's // ensurePingTriggersTable). This file periodically joins that detection -// index with the SAME GetPacketPath + airtime-annotation logic View Path +// index with the SAME getPacketPath + airtime-annotation logic View Path // already uses, deriving records (farthest, most hops, widest spread, // fastest full spread, most airtime-efficient) and leaderboards (which // relay appears most often, which observer hears pings first most often), @@ -26,7 +26,7 @@ import ( // pingScoresRecomputeInterval: pings are rare relative to general channel // traffic, so this doesn't need the 60s cadence of the hotter recomputers // (neighbor graph, analytics) -- a couple of minutes keeps the highscore -// board fresh without adding needless periodic GetPacketPath-per-ping load. +// board fresh without adding needless periodic getPacketPath-per-ping load. const pingScoresRecomputeInterval = 2 * time.Minute // PingScore is one ping's computed highscore-relevant stats. @@ -180,11 +180,11 @@ func (db *DB) fetchPingTriggers() ([]pingTriggerRow, error) { return out, nil } -// computePingScore builds one ping's full stats via the same GetPacketPath +// computePingScore builds one ping's full stats via the same getPacketPath // + airtime-annotation path View Path uses, so the numbers on the // highscore board always match what "View path" shows for that packet. func (s *Server) computePingScore(trigger pingTriggerRow) *PingScore { - resp, err := s.db.GetPacketPath(trigger.hash, EstimateMaxEdgeKm) + resp, err := s.db.getPacketPath(trigger.hash, EstimateMaxEdgeKm, s.estimatedPositionsEnabled()) if err != nil { return nil } @@ -192,17 +192,17 @@ func (s *Server) computePingScore(trigger pingTriggerRow) *PingScore { } // buildPingScoreFromPath is the shared scoring core computePingScore uses -// (with resp sourced from a live GetPacketPath call, as before this +// (with resp sourced from a live getPacketPath call, as before this // extraction -- behavior and API output are unchanged) and that a future // bulk recomputer (Phase 4D+) will reuse with resp sourced from -// GetPacketPathsBulk instead, without duplicating this logic. resp is not +// getPacketPathsBulk instead, without duplicating this logic. resp is not // yet airtime-annotated when passed in -- annotatePacketPathAirtime is // applied exactly once, here, so neither caller needs to remember to call // it separately (and a caller that DOES call it first would double-annotate, // which this function's callers must not do). // -// nil (or a response with zero branches) means GetPacketPath/ -// GetPacketPathsBulk couldn't build a usable path for this trigger this +// nil (or a response with zero branches) means getPacketPath/ +// getPacketPathsBulk couldn't build a usable path for this trigger this // cycle -- explicitly returns nil rather than a zero-value *PingScore, so // callers can distinguish "no score, don't record anything" from "score, // but every field happens to be zero". @@ -220,7 +220,7 @@ func (s *Server) buildPingScoreFromPath(trigger pingTriggerRow, resp *PacketPath StationCount: len(resp.Branches), } - // Branches are sorted deepest-first by GetPacketPath. + // Branches are sorted deepest-first by getPacketPath. deepest := resp.Branches[0] score.DeepestHops = deepest.Hops if deepest.Observer != nil { diff --git a/cmd/server/ping_scores_test.go b/cmd/server/ping_scores_test.go index ed2fda5b8..72f582a72 100644 --- a/cmd/server/ping_scores_test.go +++ b/cmd/server/ping_scores_test.go @@ -42,7 +42,7 @@ func seedPingObservation(t *testing.T, srv *Server, txID int64, observerID strin } // setupPingScoresFixture seeds observers with known positions (via nodes -// row for lat/lon lookups the same way GetPacketPath resolves them) and +// row for lat/lon lookups the same way getPacketPath resolves them) and // returns the server (+ router, for handler-level tests) ready for // computePingScore/computeAllPingScores. func setupPingScoresFixture(t *testing.T) (*Server, *mux.Router) { diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 8294cc146..186362789 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -161,6 +161,9 @@ type Server struct { gpsSanityMu sync.Mutex gpsSanityCache *GPSSanityResponse gpsSanityCachedAt time.Time + + // Copied once by NewServer; zero value preserves default-on behavior. + estimatedPositionsDisabled bool } // PerfStats tracks request performance. @@ -201,6 +204,8 @@ func NewServer(db *DB, cfg *Config, hub *Hub) *Server { version: resolveVersion(), commit: resolveCommit(), buildTime: resolveBuildTime(), + + estimatedPositionsDisabled: !cfg.estimatedPositionsEnabled(), } } @@ -588,6 +593,7 @@ func (s *Server) handleConfigClient(w http.ResponseWriter, r *http.Request) { ClientRxCoverage: s.cfg.ClientRxCoverageEnabled(), GeoFilter: s.getGeoFilter(), Privacy: privacy, + EstimatedPositions: EstimatedPositionsClientConfig{Enabled: s.estimatedPositionsEnabled()}, }) } @@ -659,13 +665,13 @@ func (s *Server) handleAreaAnalytics(w http.ResponseWriter, r *http.Request) { if s.areaAnalyticsCache != nil && time.Since(s.areaAnalyticsCachedAt) < areaAnalyticsTTL { cached := s.areaAnalyticsCache s.areaAnalyticsMu.Unlock() - writeJSON(w, cached) + s.writeAreaAnalytics(w, cached) return } s.areaAnalyticsMu.Unlock() if s.cfg == nil || len(s.cfg.Areas) == 0 { - writeJSON(w, &AreaAnalyticsResponse{}) + s.writeAreaAnalytics(w, &AreaAnalyticsResponse{}) return } @@ -680,15 +686,13 @@ func (s *Server) handleAreaAnalytics(w http.ResponseWriter, r *http.Request) { graph = s.store.graph.Load() } - positionGaps, noNeighborFix, estimatedNodes := computeAreaPositionGaps(s.db, positioned, unpositioned, s.cfg.Areas, EstimateMaxEdgeKm) - resp := &AreaAnalyticsResponse{ - Density: computeAreaDensity(positioned, s.cfg.Areas, s.cfg.GetHealthThresholds()), - BridgeNodes: computeAreaBridgeNodes(positioned, s.cfg.Areas, graph), - PositionGaps: positionGaps, - UnpositionedTotal: len(unpositioned), - UnpositionedNoNeighborFix: noNeighborFix, - EstimatedNodes: estimatedNodes, + Density: computeAreaDensity(positioned, s.cfg.Areas, s.cfg.GetHealthThresholds()), + BridgeNodes: computeAreaBridgeNodes(positioned, s.cfg.Areas, graph), + UnpositionedTotal: len(unpositioned), + } + if s.estimatedPositionsEnabled() { + resp.PositionGaps, resp.UnpositionedNoNeighborFix, resp.EstimatedNodes = computeAreaPositionGaps(s.db, positioned, unpositioned, s.cfg.Areas, EstimateMaxEdgeKm) } s.areaAnalyticsMu.Lock() @@ -696,7 +700,7 @@ func (s *Server) handleAreaAnalytics(w http.ResponseWriter, r *http.Request) { s.areaAnalyticsCachedAt = time.Now() s.areaAnalyticsMu.Unlock() - writeJSON(w, resp) + s.writeAreaAnalytics(w, resp) } // handleGPSSanity serves computeSuspiciousGPSPositions' cross-check of @@ -707,6 +711,10 @@ func (s *Server) handleAreaAnalytics(w http.ResponseWriter, r *http.Request) { // whole positioned population on every request otherwise. func (s *Server) handleGPSSanity(w http.ResponseWriter, r *http.Request) { const gpsSanityTTL = 30 * time.Second + if !s.estimatedPositionsEnabled() { + writeJSON(w, EstimatedPositionsDisabledResponse{}) + return + } s.gpsSanityMu.Lock() if s.gpsSanityCache != nil && time.Since(s.gpsSanityCachedAt) < gpsSanityTTL { @@ -2071,7 +2079,7 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) { nodeLat, hasLat := node["lat"].(float64) nodeLon, hasLon := node["lon"].(float64) hasRealFix := hasLat && hasLon && !(nodeLat == 0 && nodeLon == 0) - if _, lat, lon, contributorCount, _, ok := s.db.nearestPositionedNeighbor(pubkey, EstimateMaxEdgeKm); ok { + if _, lat, lon, contributorCount, _, ok := s.estimateNodePosition(pubkey); ok { node["estimated_lat"] = lat node["estimated_lon"] = lon node["estimated_contributor_count"] = contributorCount @@ -3720,7 +3728,7 @@ func (s *Server) handlePacketPath(w http.ResponseWriter, r *http.Request) { writeJSON(w, PacketPathResponse{Hash: hash, Branches: []PacketPathBranch{}}) return } - resp, err := s.db.GetPacketPath(hash, EstimateMaxEdgeKm) + resp, err := s.db.getPacketPath(hash, EstimateMaxEdgeKm, s.estimatedPositionsEnabled()) if err != nil { writeError(w, 500, err.Error()) return @@ -3734,7 +3742,7 @@ func (s *Server) handlePacketPath(w http.ResponseWriter, r *http.Request) { // AirtimeRelayCount: the LoRa Time-on-Air x distinct-relay-count estimate // for this packet's whole flood (same formula as the Relay Airtime Share // analytics metric, issue #1768), looked up from the in-memory -// PacketStore via the transmission ID GetPacketPath captured. Left +// PacketStore via the transmission ID getPacketPath captured. Left // unset -- not a guessed zero -- when the store is unavailable (DB-only // mode) or this transmission has been evicted from memory. func (s *Server) annotatePacketPathAirtime(resp *PacketPathResponse) { @@ -3759,7 +3767,7 @@ func (s *Server) annotatePacketPathAirtime(resp *PacketPathResponse) { // configured area any point or observer on the path falls in, deduped and // alphabetized, uncapped (unlike annotateBotReplyTouchedAreas's capped // pong-reply list -- the map view has room to show the full set). Unlike -// that function, no DB round-trip is needed: GetPacketPath already +// that function, no DB round-trip is needed: getPacketPath already // resolved every position (including the neighbor-centroid approximation // fallback), so this just reads the lat/lon already on the response. func (s *Server) annotatePacketPathTouchedAreas(resp *PacketPathResponse) { diff --git a/cmd/server/types.go b/cmd/server/types.go index f7f72b006..caec59ab3 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -1634,6 +1634,8 @@ type ClientConfigResponse struct { // {"enabled":true}: no operator-configured text is ever sent to the // frontend. See PrivacyClientConfig below and PrivacyConfig (config.go). Privacy *PrivacyClientConfig `json:"privacy,omitempty"` + + EstimatedPositions EstimatedPositionsClientConfig `json:"estimatedPositions"` } // PrivacyClientConfig is the privacy block of /api/config/client. It carries diff --git a/config.example.json b/config.example.json index 63ac78db0..a89aff342 100644 --- a/config.example.json +++ b/config.example.json @@ -368,6 +368,10 @@ "cacheRecomputeIntervalSeconds": 300, "_comment": "maxAgeDays: neighbor edges older than this many days are pruned on startup and daily. Default 5. maxEdgeKm: geo-implausibility filter — when both endpoints have GPS, edges with haversine distance > maxEdgeKm are rejected at build time to prevent disambiguator self-reinforcement on wide-geo MQTT deployments. Default 500 km (well above any plausible terrestrial LoRa hop). 0 ⇒ use default; set negative to disable the filter. Rejected count surfaces in /api/analytics/neighbor-graph stats. Issue #1228. cacheRecomputeIntervalSeconds: how often the background recomputer rebuilds the default-shape /api/analytics/neighbor-graph response (#1481 P0-1). Default 300 (5 min). Lower = fresher data, more CPU per minute. Issue #1483." }, + "estimatedPositions": { + "enabled": true, + "_comment": "Operator policy for neighbor-derived position estimates. Omit this section or enabled to preserve the default (true). Set false to disable estimates in node details, packet/Ping Scores paths, maps, and estimate-dependent analytics. Reported GPS, ordinary neighbor graphs, and independent IATA/name-based fallbacks are unchanged. Requires a Go server restart and browser refresh; not reloaded by SIGHUP." + }, "observersCache": { "ttlSeconds": 30, "_comment": "TTL for the default-shape /api/observers response cache (#1481 P0-3). Default 30s. Lower = fresher data, more SQL pressure on the 1.9M-row observations table. TTL-boundary refills are collapsed via singleflight so concurrent requests cause exactly one SQL fill. Issue #1483." diff --git a/docs/api-spec.md b/docs/api-spec.md index 54e86d785..56e9d5d6e 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -11,6 +11,7 @@ ## Table of Contents - [Conventions](#conventions) +- [Estimated-position policy](#estimated-position-policy) - [GET /api/stats](#get-apistats) - [GET /api/health](#get-apihealth) - [GET /api/perf](#get-apiperf) @@ -66,6 +67,38 @@ --- +## Estimated-position policy + +`config.json` accepts `estimatedPositions: { "enabled": false }`. Missing +section or field defaults to `true`. This is a server-startup policy, not a +request parameter or a browser preference. Restart the server to change it. +`GET /api/config/client` always publishes the effective value as +`estimatedPositions: { "enabled": }`. + +When disabled: + +- Node detail omits `estimated_lat`, `estimated_lon`, + `estimated_contributor_count`, and `estimated_distance_km`. Reported + `lat`/`lon` are unchanged. +- Packet paths and saved Ping Scores path responses retain route identities + and reported coordinates but omit neighbor-derived coordinates and their + approximation metadata. Endpoint distances depending on removed estimates + are omitted; distances between reported endpoints remain valid. Saved + source archives are not modified -- neither by a request nor by the + background Ping Scores history refresh -- so re-enabling the policy serves + the original archived geometry again. +- `/api/analytics/areas` returns `estimatedPositionsEnabled: false` alongside + `density`, `bridgeNodes`, and `unpositionedTotal`. It omits uncomputed + `positionGaps`, `estimatedNodes`, and `unpositionedNoNeighborFix` rather than + claiming zero gaps or no neighbor evidence. +- `/api/analytics/gps-sanity` returns only + `{ "estimatedPositionsEnabled": false }`, without running the estimator. + +Enabled analytics retain their existing response shapes. API clients must +distinguish disabled computation from an enabled, empty result. The policy +does not disable ordinary neighbor graphs or independent IATA/name-based +position fallbacks. No query parameter can override the server setting. + ## GET /api/ping-scores/:hash/path Returns path evidence for a currently displayed Ping Scores record. Requires @@ -2338,7 +2371,8 @@ Client-side configuration values. "wsReconnectMs": number | null, "cacheInvalidateMs": number | null, "externalUrls": object | null, - "propagationBufferMs": number // default: 5000 + "propagationBufferMs": number, // default: 5000 + "estimatedPositions": { "enabled": boolean } // default: true; operator policy } ``` diff --git a/docs/deployment.md b/docs/deployment.md index 471023cfa..0022ef5fa 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -184,6 +184,44 @@ See `config.example.json` in the repository for all available options including: - Geo-filtering - Map tile providers (OSM, Stamen, Carto, etc.) +### Disable estimated node positions + +Neighbor-derived position estimates are enabled by default. To disable them +for the entire instance, add this to `config.json`: + +```json +{ + "estimatedPositions": { + "enabled": false + } +} +``` + +Omitting the section or its `enabled` field preserves existing behavior +(`true`). Use a JSON boolean, not the string `"false"`; invalid policy types +are rejected at startup. + +Restart the **Go server** after changing this setting, then refresh open +browser tabs. The policy is fixed for the lifetime of each server instance; +SIGHUP does not reload it. No ingestor change or database migration is needed. + +When disabled, the server skips neighbor-position estimation and omits its +results from node details and packet paths, including saved Ping Scores +paths. Estimate-dependent tools and map views display an operator-disabled +notice instead of suggesting that no neighbor evidence exists. Areas retains +reported-position density, bridge nodes, and the unpositioned-node count. +Reported GPS, ordinary neighbor graphs, and independent IATA/name-based +fallbacks are unchanged. No stored data is deleted or rewritten: Ping Scores +path archives captured while estimates were enabled keep their recorded +approximate geometry and capture time, and the background history refresh +does not rewrite them while the policy is off, so the original evidence +reappears as soon as the setting is turned back on. + +The effective setting is published through `/api/config/client`. Browser +preferences and deep links cannot enable it against the server policy. A +Customizer display preference is deferred to a later milestone and must +remain subordinate to this operator setting. + ### Reloading config changes without a restart (SIGHUP) Most `config.json` changes require a container restart to take effect. **`hashChannels`** and **`hashRegions`** are the exception — the ingestor can reload just these two settings live: diff --git a/internal/dbschema/dbschema.go b/internal/dbschema/dbschema.go index 191bdf34e..a4b44874f 100644 --- a/internal/dbschema/dbschema.go +++ b/internal/dbschema/dbschema.go @@ -1065,7 +1065,7 @@ func ensureTransmissionsLastSeenColumn(rw *sql.DB, logf Logger) error { // Deliberately just a detection index, not the computed stats themselves: // tx_id/hash/channel_hash/sender/first_seen are cheap to write once at // ingest time, while farthest/deepest/spread/airtime are derived from the -// SAME GetPacketPath + airtime-annotation logic View Path already uses, +// SAME getPacketPath + airtime-annotation logic View Path already uses, // recomputed periodically by the server's ping-scores recomputer // (cmd/server/ping_scores.go) rather than persisted here -- so a later // observation of an old ping (e.g. a station that only just relayed it diff --git a/public/analytics.js b/public/analytics.js index 4e1a20a3e..d28b3e003 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -7157,13 +7157,16 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf async function _renderAreasTabBody(el) { try { var d = await api('/analytics/areas', { ttl: 30000 }); + if (window.MeshConfigReady) await window.MeshConfigReady; + var estimatesEnabled = window.EstimatedPositions?.enabled(d) !== false; var density = (d && d.density) || []; var bridgeNodes = (d && d.bridgeNodes) || []; - var positionGaps = (d && d.positionGaps) || []; - var estimatedNodes = (d && d.estimatedNodes) || []; + var positionGaps = estimatesEnabled ? ((d && d.positionGaps) || []) : []; + var estimatedNodes = estimatesEnabled ? ((d && d.estimatedNodes) || []) : []; if (!density.length && !bridgeNodes.length && !positionGaps.length) { - el.innerHTML = '
No Areas are configured — this tab needs at least one drawn-polygon Area (meshguide.dk sync) to report on.
'; + el.innerHTML = '
No Areas are configured — this tab needs at least one drawn-polygon Area (meshguide.dk sync) to report on.
' + + (estimatesEnabled ? '' : window.EstimatedPositions.disabledNoticeHTML); return; } @@ -7385,7 +7388,7 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf var unpositionedNote = '

' + (d.unpositionedTotal || 0).toLocaleString() + ' node' + (d.unpositionedTotal === 1 ? '' : 's') + ' network-wide have no real GPS fix' + - (d.unpositionedNoNeighborFix ? ', of which ' + d.unpositionedNoNeighborFix.toLocaleString() + ' also have no positioned neighbor to estimate from — those can\'t be placed anywhere, not even approximately, so they\'re absent from the table above entirely.' : '.') + + (estimatesEnabled && d.unpositionedNoNeighborFix ? ', of which ' + d.unpositionedNoNeighborFix.toLocaleString() + ' also have no positioned neighbor to estimate from — those can\'t be placed anywhere, not even approximately, so they\'re absent from the table above entirely.' : '.') + '

'; el.innerHTML = @@ -7404,14 +7407,14 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _stopForeignTraf 'Position-Fix Coverage Gaps by Area' + (estimatedNodes.length ? 'View Estimated Nodes on Map (' + estimatedNodes.length.toLocaleString() + ')' : '') + '' + - '

How many of each area\'s nodes have an actual reported GPS position vs. how many were only placeable via a neighbor-based estimate (same technique used for View Path\'s approximate markers). Click a column header to sort by it.

' + - '
' + gapsSection.tableHtml() + '
' + + (estimatesEnabled ? '

How many of each area\'s nodes have an actual reported GPS position vs. how many were only placeable via a neighbor-based estimate (same technique used for View Path\'s approximate markers). Click a column header to sort by it.

' : '') + + '
' + (estimatesEnabled ? gapsSection.tableHtml() : window.EstimatedPositions.disabledNoticeHTML) + '
' + unpositionedNote + ''; densitySection.attach(); bridgeSection.attach(); - gapsSection.attach(); + if (estimatesEnabled) gapsSection.attach(); } catch (e) { el.innerHTML = '
Failed to load area analytics: ' + esc(String(e)) + '
'; } diff --git a/public/area-nodes-map.js b/public/area-nodes-map.js index 214407b8e..669ccef86 100644 --- a/public/area-nodes-map.js +++ b/public/area-nodes-map.js @@ -41,6 +41,9 @@ function open(label, points) { close(); // in case one's already open + // Only opened from the estimate-specific tool, whose load awaits config. + if (window.EstimatedPositions?.enabled() === false) return; + var pts = (points || []).filter(function (p) { return p && typeof p.lat === 'number' && typeof p.lon === 'number'; }); var overlay = document.createElement('div'); diff --git a/public/gps-sanity.js b/public/gps-sanity.js index c25538fb1..f852ba017 100644 --- a/public/gps-sanity.js +++ b/public/gps-sanity.js @@ -64,7 +64,17 @@ if (contentEl) contentEl.innerHTML = '

Loading…

'; if (statusEl) statusEl.textContent = ''; api('/analytics/gps-sanity', { ttl: 30000 }) - .then(function (data) { + .then(async function (data) { + if (window.MeshConfigReady) await window.MeshConfigReady; + if (!container) return; + if (window.EstimatedPositions?.enabled(data) === false) { + rows = []; + if (statusEl) statusEl.textContent = ''; + if (contentEl) contentEl.innerHTML = window.EstimatedPositions.disabledNoticeHTML; + var filterInput = document.getElementById('gps-sanity-filter'); + if (filterInput) filterInput.disabled = true; + return; + } rows = (data && Array.isArray(data.nodes)) ? data.nodes : []; totalRealGPS = (data && data.totalRealGps) || 0; evaluated = (data && data.evaluated) || 0; diff --git a/public/map.js b/public/map.js index 6807bca32..b94c2e415 100644 --- a/public/map.js +++ b/public/map.js @@ -1203,6 +1203,10 @@ return Math.min(0.75, 0.35 + contributorCount * 0.08); } function drawEstimatedNodes(points) { + if (window.EstimatedPositions?.enabled() === false) { + showEstimatedPositionsDisabled(); + return; + } if (markerLayer) map.removeLayer(markerLayer); if (clusterGroup) map.removeLayer(clusterGroup); if (heatLayer) map.removeLayer(heatLayer); @@ -1704,6 +1708,12 @@ async function loadEstimatedNodesFromDeepLink() { const alive = mapToken(); try { + if (window.MeshConfigReady) await window.MeshConfigReady; + if (!alive()) return; + if (window.EstimatedPositions?.enabled() === false) { + showEstimatedPositionsDisabled(); + return; + } const resp = await fetch('/api/analytics/areas'); if (!alive()) return; // #123 if (!resp.ok) { @@ -1712,6 +1722,10 @@ } const data = await resp.json(); if (!alive()) return; + if (window.EstimatedPositions?.enabled(data) === false) { + showEstimatedPositionsDisabled(); + return; + } const points = (data && Array.isArray(data.estimatedNodes)) ? data.estimatedNodes : []; drawEstimatedNodes(points); } catch (e) { @@ -1719,6 +1733,20 @@ } } + // Keep the ordinary, reported-GPS markers visible for a disabled deep link. + // This is not the empty-estimates state, and must not replace the base map. + function showEstimatedPositionsDisabled() { + const container = map.getContainer(); + let label = container.querySelector('.mc-estimated-nodes-label'); + if (!label) { + label = document.createElement('div'); + label.className = 'mc-estimated-nodes-label'; + label.style.cssText = 'position:absolute;top:10px;left:50px;right:10px;z-index:1000;background:var(--input-bg);color:var(--text);padding:4px 10px;border-radius:4px;font-size:12px'; + container.appendChild(label); + } + label.textContent = window.EstimatedPositions.disabledMessage; + } + // #123: every call is a new request generation; only the newest request // of the current mount renders. A superseded call resolves when the newest // one does, so callers chaining on loadNodes() still see loaded nodes. diff --git a/public/nodes.js b/public/nodes.js index 654b80de8..ef8baf3e4 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -758,7 +758,8 @@ async function fetchNodeDetail(pubkey) { const [nodeData, healthData] = await Promise.all([ api('/nodes/' + encodeURIComponent(pubkey) + '?include=advertRoutes', { ttl: CLIENT_TTL.nodeDetail }), - api('/nodes/' + encodeURIComponent(pubkey) + '/health', { ttl: CLIENT_TTL.nodeDetail }).catch(() => null) + api('/nodes/' + encodeURIComponent(pubkey) + '/health', { ttl: CLIENT_TTL.nodeDetail }).catch(() => null), + window.MeshConfigReady ]); nodeData.healthData = healthData; return nodeData; @@ -834,7 +835,8 @@ // use). Shown alongside a real fix too, not just as a fallback when // one's missing -- lets a node flagged by Suspicious GPS Positions // be visually cross-checked against its own claimed position. - const hasEstLoc = n.estimated_lat != null && n.estimated_lon != null; + const estimatesEnabled = window.EstimatedPositions?.enabled() !== false; + const hasEstLoc = estimatesEnabled && n.estimated_lat != null && n.estimated_lon != null; // Health stats const h = healthData || {}; @@ -946,6 +948,7 @@ ${stats.avgHops ? `Avg Hops${stats.avgHops}` : ''} ${hasLoc ? `Location${Number(n.lat).toFixed(5)}, ${Number(n.lon).toFixed(5)}` : ''} ${hasEstLoc ? `${hasLoc ? 'Neighbor Estimate' : 'Location'} (estimated)~${Number(n.estimated_lat).toFixed(5)}, ~${Number(n.estimated_lon).toFixed(5)} (from ${n.estimated_contributor_count} neighbor${n.estimated_contributor_count === 1 ? '' : 's'}${hasLoc ? ', ' + Number(n.estimated_distance_km).toFixed(1) + ' km from reported position' : ', no real GPS fix'})` : ''} + ${!estimatesEnabled ? `Position estimates${window.EstimatedPositions.disabledNoticeHTML}` : ''} Hash Prefix${n.hash_size ? '' + n.public_key.slice(0, n.hash_size * 2).toUpperCase() + ' (' + n.hash_size + '-byte)' : 'Unknown'}${n.hash_size_inconsistent ? ' varies' : ''} @@ -1936,7 +1939,8 @@ // Same "real fix" convention and estimate-alongside-real-fix behavior // as loadFullNode above -- see its comments. const hasLoc = n.lat != null && n.lon != null && !(n.lat === 0 && n.lon === 0); - const hasEstLoc = n.estimated_lat != null && n.estimated_lon != null; + const estimatesEnabled = window.EstimatedPositions?.enabled() !== false; + const hasEstLoc = estimatesEnabled && n.estimated_lat != null && n.estimated_lon != null; const nodeUrl = location.origin + '/#/nodes/' + encodeURIComponent(n.public_key); // Status calculation via shared helper @@ -1980,6 +1984,7 @@ ${stats.avgHops ? `
Avg Hops
${stats.avgHops}
` : ''} ${hasLoc ? `
Location
${Number(n.lat).toFixed(5)}, ${Number(n.lon).toFixed(5)}
` : ''} ${hasEstLoc ? `
${hasLoc ? 'Neighbor Estimate' : 'Location'} (estimated)
~${Number(n.estimated_lat).toFixed(5)}, ~${Number(n.estimated_lon).toFixed(5)} (from ${n.estimated_contributor_count} neighbor${n.estimated_contributor_count === 1 ? '' : 's'}${hasLoc ? ', ' + Number(n.estimated_distance_km).toFixed(1) + ' km from reported position' : ', no real GPS fix'})
` : ''} + ${!estimatesEnabled ? `
Position estimates
${window.EstimatedPositions.disabledNoticeHTML}
` : ''} diff --git a/public/packet-path-map.js b/public/packet-path-map.js index 06c47c5e7..66f276067 100644 --- a/public/packet-path-map.js +++ b/public/packet-path-map.js @@ -237,7 +237,8 @@ // renders as a hollow, dashed marker instead of a solid one, never // mistaken for a real fix. function chainForBranch(b) { - var located = (b.points || []).filter(function (p) { return p.lat != null && p.lon != null; }); + var estimatesEnabled = window.EstimatedPositions?.enabled() !== false; + var located = (b.points || []).filter(function (p) { return p.lat != null && p.lon != null && (estimatesEnabled || !p.approx); }); var chain = located.map(function (p, hi) { return { lat: p.lat, lon: p.lon, name: p.name, label: 'hop ' + (hi + 1) + ' of ' + b.hops, approx: !!p.approx, @@ -245,7 +246,7 @@ publicKey: p.publicKey, }; }); - if (b.observer && b.observer.lat != null && b.observer.lon != null) { + if (b.observer && b.observer.lat != null && b.observer.lon != null && (estimatesEnabled || !b.observer.approx)) { var observerLabel = b.hops + ' hop' + (b.hops === 1 ? '' : 's'); if (typeof b.secondsAfterFirst === 'number') observerLabel += ', ' + formatElapsed(b.secondsAfterFirst); if (typeof b.distanceFromFirstKm === 'number' && b.distanceFromFirstKm > 0) observerLabel += ', ' + b.distanceFromFirstKm.toFixed(1) + ' km away'; @@ -287,11 +288,12 @@ '

Relay Path

' + '

How far and how wide this packet spread. Click a marker to open that node\'s detail page.

' + '' + + '' + '
' + 'farthest-traveled route' + '' + 'other station' + - 'approximate position' + + 'approximate position' + 'first to hear it' + '
' + '
' + @@ -350,6 +352,7 @@ var data; try { data = historical ? await options.loadPath(hash) : await api('/packets/' + encodeURIComponent(hash) + '/path'); + if (window.MeshConfigReady) await window.MeshConfigReady; } catch (e) { if (!isCurrent()) return; if (historical) withoutMap('Failed to load path: ' + e.message, true); @@ -358,6 +361,16 @@ } if (!isCurrent()) return; + if (window.EstimatedPositions?.enabled() === false) { + var approxLegend = document.getElementById('packetPathApproxLegend'); + if (approxLegend) approxLegend.style.display = 'none'; + var estimatePolicy = document.getElementById('packetPathEstimatePolicy'); + if (estimatePolicy) { + estimatePolicy.style.display = 'block'; + estimatePolicy.textContent = window.EstimatedPositions.disabledMessage; + } + } + if (historical) { if (data && data.status === 'initializing') { withoutMap('Ping history is initializing. Try again shortly.', true); diff --git a/public/position-gaps.js b/public/position-gaps.js index 8f6bdad98..b9e716cb1 100644 --- a/public/position-gaps.js +++ b/public/position-gaps.js @@ -106,7 +106,16 @@ if (contentEl) contentEl.innerHTML = '

Loading…

'; if (statusEl) statusEl.textContent = ''; api('/analytics/areas', { ttl: 30000 }) - .then(function (data) { + .then(async function (data) { + if (window.MeshConfigReady) await window.MeshConfigReady; + if (!container) return; + if (window.EstimatedPositions?.enabled(data) === false) { + areaRows = []; + estimatedRows = []; + if (statusEl) statusEl.textContent = ''; + if (contentEl) contentEl.innerHTML = window.EstimatedPositions.disabledNoticeHTML; + return; + } areaRows = (data && Array.isArray(data.positionGaps)) ? data.positionGaps : []; estimatedRows = (data && Array.isArray(data.estimatedNodes)) ? data.estimatedNodes : []; unpositionedTotal = (data && data.unpositionedTotal) || 0; diff --git a/public/roles.js b/public/roles.js index 95bd336d6..c7253e407 100644 --- a/public/roles.js +++ b/public/roles.js @@ -673,8 +673,20 @@ flasher: 'https://flasher.meshcore.io/' }; + // One operator-owned policy for every estimated-position surface. Local + // preferences and deep links cannot enable it; older servers default on. + var estimatedPositionsEnabled = true; + window.EstimatedPositions = { + enabled: function (response) { + return estimatedPositionsEnabled && (!response || response.estimatedPositionsEnabled !== false); + }, + disabledMessage: 'Estimated positions are disabled by the instance operator.', + disabledNoticeHTML: '

Estimated positions are disabled by the instance operator.

' + }; + // ─── Fetch server overrides ─── window.MeshConfigReady = fetch('/api/config/client').then(function (r) { return r.json(); }).then(function (cfg) { + estimatedPositionsEnabled = !cfg.estimatedPositions || cfg.estimatedPositions.enabled !== false; window.MC_CLIENT_RX_COVERAGE = cfg.clientRxCoverage === true; // Coverage is opt-in: the nav link is NOT in static HTML (so the default-off // nav matches upstream and the nav-overflow tests). Inject it after Analytics diff --git a/public/style.css b/public/style.css index 0ff8dd601..27475bbd0 100644 --- a/public/style.css +++ b/public/style.css @@ -1790,6 +1790,10 @@ fieldset.mc-section legend.mc-label { padding: 0; } /* === Misc === */ .text-muted { color: var(--text-muted); } +/* Explanatory note beside a tool's own content (e.g. the operator-disabled + notice in Areas -> Position-Fix Coverage Gaps): same size as the muted + notes it sits next to, so it does not read as body copy. */ +.estimated-positions-note { font-size: 0.85em; margin: 0 0 8px; } .text-center { text-align: center; } .mt-8 { margin-top: 8px; } .mb-8 { margin-bottom: 8px; } diff --git a/test-all.sh b/test-all.sh index cf8d1eeae..2f6d4a597 100755 --- a/test-all.sh +++ b/test-all.sh @@ -146,6 +146,7 @@ run test-node-changes-tool.js run test-network-digest-tool.js run test-position-gaps-tool.js run test-gps-sanity-tool.js +run test-estimated-positions-config.js run test-map-scope-filter.js run test-issue-117-ws-watchdog.js run test-issue-111-drawer-version.js diff --git a/test-analytics-areas-tab.js b/test-analytics-areas-tab.js index 4b3c56797..24318f6ae 100644 --- a/test-analytics-areas-tab.js +++ b/test-analytics-areas-tab.js @@ -138,6 +138,26 @@ function makeApiStub(resp) { (async () => { console.log('\n=== analytics.js: renderAreasTab ==='); + await testAsync('disabled estimates preserve real density/bridges without zero-value estimate tables or links', async () => { + // Deliberately retain stale estimate fields: the capability flag wins. + const ctx = makeAnalyticsSandbox(makeApiStub(makeAreasResponse({ estimatedPositionsEnabled: false }))); + const el = fakeEl(); + await ctx.window._analyticsRenderAreasTab(el); + assert.ok(el.innerHTML.includes('disabled by the instance operator')); + assert.ok(el.innerHTML.includes('Odense by') && el.innerHTML.includes('BridgeNode')); + assert.ok(el.innerHTML.includes('3 nodes network-wide have no real GPS fix')); + assert.ok(!el.innerHTML.includes('View Estimated Nodes') && !el.innerHTML.includes('Estimated (via Neighbors)')); + assert.ok(!el.innerHTML.includes('no positioned neighbor')); + }); + + await testAsync('disabled estimates are explained even when no areas exist', async () => { + const ctx = makeAnalyticsSandbox(makeApiStub({ estimatedPositionsEnabled: false })); + const el = fakeEl(); + await ctx.window._analyticsRenderAreasTab(el); + assert.ok(el.innerHTML.includes('disabled by the instance operator')); + assert.ok(el.innerHTML.includes('No Areas are configured')); + }); + await testAsync('renders the Node Density & Health table from the API response', async () => { const ctx = makeAnalyticsSandbox(makeApiStub(makeAreasResponse())); const el = fakeEl(); diff --git a/test-estimated-positions-config.js b/test-estimated-positions-config.js new file mode 100644 index 000000000..7d9670f84 --- /dev/null +++ b/test-estimated-positions-config.js @@ -0,0 +1,84 @@ +'use strict'; + +// Exercise the shared runtime policy and the actual tool renderers, not copies. +const assert = require('assert'); +const fs = require('fs'); +const vm = require('vm'); + +function sandbox(config, response, configFetch) { + const elements = new Map(); + const element = id => { + if (!elements.has(id)) elements.set(id, { + innerHTML: '', textContent: '', style: {}, value: '', + addEventListener() {}, querySelectorAll() { return []; }, querySelector() { return null; } + }); + return elements.get(id); + }; + const ctx = vm.createContext({ + window: {}, console, Promise, URLSearchParams, + fetch: configFetch || (() => Promise.resolve({ json: () => Promise.resolve(config) })), + api: () => Promise.resolve(response), + document: { getElementById: element, querySelector: () => null }, + registerPage() {}, location: { hash: '' } + }); + vm.runInContext(fs.readFileSync('public/roles.js', 'utf8'), ctx); + return { ctx, element, load: file => vm.runInContext(fs.readFileSync('public/' + file, 'utf8'), ctx) }; +} + +async function run() { + for (const config of [{}, { estimatedPositions: {} }, { estimatedPositions: { enabled: true } }]) { + const { ctx } = sandbox(config); + await ctx.window.MeshConfigReady; + assert.strictEqual(ctx.window.EstimatedPositions.enabled(), true, 'omitted or enabled config preserves existing behavior'); + assert.strictEqual(ctx.window.EstimatedPositions.enabled({ estimatedPositionsEnabled: false }), false, 'endpoint disabled flag takes precedence'); + } + const off = sandbox({ estimatedPositions: { enabled: false } }); + await off.ctx.window.MeshConfigReady; + assert.strictEqual(off.ctx.window.EstimatedPositions.enabled(), false); + assert.strictEqual(off.ctx.window.EstimatedPositions.enabled({ estimatedPositionsEnabled: true }), false, 'response cannot override operator disable'); + const unavailable = sandbox({}, {}, () => Promise.reject(new Error('offline'))); + await unavailable.ctx.window.MeshConfigReady; + assert.strictEqual(unavailable.ctx.window.EstimatedPositions.enabled(), true, 'failed config retains legacy default; backend remains authoritative'); + + // The disabled notice must read as the muted explanatory note it sits + // beside (Areas -> Position-Fix Coverage Gaps), not as body copy. The + // size comes from a class in public/style.css, never an inline literal. + const notice = off.ctx.window.EstimatedPositions.disabledNoticeHTML; + assert.ok(/class="[^"]*\bestimated-positions-note\b/.test(notice), 'disabled notice carries the muted-note class'); + assert.ok(!/font-size/.test(notice), 'disabled notice must not hardcode a font size inline'); + const noticeRule = fs.readFileSync('public/style.css', 'utf8').match(/\.estimated-positions-note\s*\{[^}]*\}/); + assert.ok(noticeRule, '.estimated-positions-note is defined in public/style.css'); + assert.ok(/font-size:\s*0\.85em/.test(noticeRule[0]), 'disabled notice matches the 0.85em muted notes beside it'); + + for (const [file, tool, prefix] of [ + ['position-gaps.js', 'PositionGapsTool', 'position-gaps'], + ['gps-sanity.js', 'GPSSanityTool', 'gps-sanity'] + ]) { + for (const [config, response] of [ + [{}, { estimatedPositionsEnabled: false }], + [{ estimatedPositions: { enabled: false } }, { nodes: [], positionGaps: [], estimatedNodes: [] }] + ]) { + const s = sandbox(config, response); + await s.ctx.window.MeshConfigReady; + s.load(file); + s.ctx.window[tool].init({ innerHTML: '' }); + await new Promise(resolve => setTimeout(resolve, 0)); + const html = s.element(prefix + '-content').innerHTML; + assert.ok(html.includes('disabled by the instance operator'), file + ' explains disabled state'); + assert.ok(!html.includes('No suspicious GPS') && !html.includes('No Areas'), file + ' must not claim no evidence'); + assert.strictEqual(s.element(prefix + '-status').textContent, '', file + ' must not report manufactured zero counts'); + } + let finishConfig; + const pending = sandbox(null, {}, () => new Promise(resolve => { finishConfig = resolve; })); + pending.load(file); + pending.ctx.window[tool].init({ innerHTML: '' }); + await new Promise(resolve => setTimeout(resolve, 0)); + assert.ok(pending.element(prefix + '-content').innerHTML.includes('Loading'), file + ' waits for operator policy'); + finishConfig({ json: () => Promise.resolve({ estimatedPositions: { enabled: false } }) }); + await new Promise(resolve => setTimeout(resolve, 0)); + assert.ok(pending.element(prefix + '-content').innerHTML.includes('disabled by the instance operator'), file + ' delayed false policy wins'); + } + console.log('Estimated-position policy and disabled tool states passed.'); +} + +run().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/test-fixtures/seed-315-estimated-positions.sql b/test-fixtures/seed-315-estimated-positions.sql new file mode 100644 index 000000000..70a5f8943 --- /dev/null +++ b/test-fixtures/seed-315-estimated-positions.sql @@ -0,0 +1,30 @@ +-- Synthetic, private-network-independent fixture for issue #315. Apply only +-- to an empty, migrated test database (the E2E harness creates that database). +INSERT INTO nodes (public_key, name, role, lat, lon, first_seen, last_seen, advert_count) VALUES + ('3151000000000000000000000000000000000000000000000000000000000000', 'Position Policy Reported GPS', 'repeater', 56.5, 12, + datetime('now', '-1 day'), datetime('now'), 1), + ('3152000000000000000000000000000000000000000000000000000000000000', 'Position Policy Missing GPS', 'repeater', NULL, NULL, + datetime('now', '-1 day'), datetime('now'), 1), + ('3153000000000000000000000000000000000000000000000000000000000000', 'Position Policy Anchor One', 'repeater', 55, 12, + datetime('now', '-1 day'), datetime('now'), 1), + ('3154000000000000000000000000000000000000000000000000000000000000', 'Position Policy Anchor Two', 'repeater', 55.02, 12.02, + datetime('now', '-1 day'), datetime('now'), 1); + +INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES + ('3151000000000000000000000000000000000000000000000000000000000000', '3153000000000000000000000000000000000000000000000000000000000000', 20, datetime('now')), + ('3151000000000000000000000000000000000000000000000000000000000000', '3154000000000000000000000000000000000000000000000000000000000000', 10, datetime('now')), + ('3152000000000000000000000000000000000000000000000000000000000000', '3153000000000000000000000000000000000000000000000000000000000000', 20, datetime('now')), + ('3152000000000000000000000000000000000000000000000000000000000000', '3154000000000000000000000000000000000000000000000000000000000000', 10, datetime('now')); + +INSERT INTO observers (rowid, id, name, first_seen, last_seen, packet_count) VALUES + (1, '3153000000000000000000000000000000000000000000000000000000000000', 'Position Policy Anchor One', datetime('now', '-1 day'), datetime('now'), 1); + +INSERT INTO transmissions (id, raw_hex, hash, first_seen, last_seen, route_type, payload_type, payload_version, decoded_json, from_pubkey, route_mask) VALUES + (315001, '1100315001', 'e2e3150000000001', strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), CAST(strftime('%s', 'now') AS INTEGER), 1, 4, 0, + '{"type":"ADVERT","name":"Position Policy Reported GPS","pubKey":"3151000000000000000000000000000000000000000000000000000000000000"}', + '3151000000000000000000000000000000000000000000000000000000000000', 2); + +INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, resolved_path, timestamp) VALUES + (315001, 1, 'rx', 10, -80, 0, '["3151","3152","3155"]', + '["3151000000000000000000000000000000000000000000000000000000000000","3152000000000000000000000000000000000000000000000000000000000000","3155000000000000000000000000000000000000000000000000000000000000"]', + CAST(strftime('%s', 'now') AS INTEGER)); diff --git a/test-issue-315-estimated-positions-e2e.js b/test-issue-315-estimated-positions-e2e.js new file mode 100644 index 000000000..d98102141 --- /dev/null +++ b/test-issue-315-estimated-positions-e2e.js @@ -0,0 +1,286 @@ +#!/usr/bin/env node +/** + * #315: real Go server + real Leaflet, isolated synthetic SQLite data. + * No production endpoint, MQTT broker, API mock, or external browser request. + * + * Build cmd/server and cmd/migrate first, then run with: + * CORESCOPE_SERVER_BIN=/absolute/server CORESCOPE_MIGRATE_BIN=/absolute/migrate + * CORESCOPE_LEAFLET_DIR=/absolute/leaflet-1.9.4/dist + * node test-issue-315-estimated-positions-e2e.js + * + * Leaflet assets must already exist locally; this test never downloads them. + * SCREENSHOT_DIR optionally selects an output directory. CHROMIUM_PATH can + * select a preinstalled browser; Playwright's installed Chromium is default. + */ +'use strict'; + +const assert = require('assert/strict'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const net = require('net'); +const { spawn, execFileSync } = require('child_process'); +const { chromium } = require('playwright'); + +const ROOT = __dirname; +const REPORTED = '3151' + '0'.repeat(60); +const MISSING = '3152' + '0'.repeat(60); +const UNKNOWN = '3155' + '0'.repeat(60); +const HASH = 'e2e3150000000001'; +const DISABLED = 'Estimated positions are disabled by the instance operator.'; +const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)); + +function requiredFile(variable, suffix = '') { + assert(process.env[variable], variable + ' is required; see the test header'); + const file = path.resolve(process.env[variable], suffix); + assert(fs.statSync(file).isFile(), file + ' must be a file'); + return file; +} + +async function freePort() { + const socket = net.createServer(); + await new Promise((resolve, reject) => { socket.once('error', reject); socket.listen(0, '127.0.0.1', resolve); }); + const port = socket.address().port; + await new Promise(resolve => socket.close(resolve)); + return port; +} + +async function stop(child) { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + const ended = new Promise(resolve => child.once('exit', resolve)); + child.kill('SIGTERM'); + const timer = setTimeout(() => child.kill('SIGKILL'), 5000); + await ended; + clearTimeout(timer); +} + +async function json(base, endpoint) { + const response = await fetch(base + endpoint, { signal: AbortSignal.timeout(5000) }); + assert.equal(response.status, 200, endpoint + ': ' + response.status); + return response.json(); +} + +async function waitReady(base, child, getLog) { + for (let n = 0; n < 100; n++) { + assert(child.exitCode === null && child.signalCode === null, 'server exited: ' + getLog()); + try { await json(base, '/api/config/client'); return; } catch (_) { await delay(100); } + } + throw new Error('server startup timed out: ' + getLog()); +} + +async function apiChecks(base, enabled) { + const config = await json(base, '/api/config/client'); + assert.equal(config.estimatedPositions.enabled, enabled); + for (const key of [REPORTED, MISSING]) { + const { node } = await json(base, '/api/nodes/' + key + '?estimatedNodes=1&estimatedPositions=true'); + assert.equal(node.public_key, key); + if (key === REPORTED) { assert.equal(node.lat, 56.5); assert.equal(node.lon, 12); } + else { assert(node.lat == null && node.lon == null, 'reported GPS must not be invented'); } + if (enabled) { + assert(Number.isFinite(node.estimated_lat) && Number.isFinite(node.estimated_lon)); + assert.equal(node.estimated_contributor_count, 2); + } else { + assert(!Object.keys(node).some(k => k.startsWith('estimated_')), 'disabled node leaked an estimate'); + } + } + const packet = await json(base, '/api/packets/' + HASH + '/path?estimatedNodes=1'); + assert(packet.branches.length > 0, 'fixture must exercise a real resolved path'); + for (const branch of [packet.first, ...packet.branches]) { + assert.deepEqual(branch.points.map(p => p.publicKey), [REPORTED, MISSING, UNKNOWN]); + const [real, missing, unknown] = branch.points; + assert.equal(real.lat, 56.5); assert.equal(real.lon, 12); assert(!real.approx); + assert.equal(branch.observer.lat, 55); assert.equal(branch.observer.lon, 12); + assert.equal(Boolean(missing.approx), enabled); + if (enabled) assert(Number.isFinite(missing.lat)); + else assert(missing.lat == null && missing.lon == null); + assert(unknown.lat == null && unknown.lon == null && !unknown.approx); + } + const areas = await json(base, '/api/analytics/areas?estimatedNodes=1'); + assert(areas.density.some(a => a.total > 0), 'real-GPS area density must survive'); + assert(Array.isArray(areas.bridgeNodes), 'ordinary bridge analytics must survive'); + const sanity = await json(base, '/api/analytics/gps-sanity'); + if (enabled) { + assert(areas.estimatedNodes.some(n => n.publicKey === MISSING)); + assert(areas.positionGaps.some(a => a.approximated > 0)); + assert(sanity.nodes.some(n => n.publicKey === REPORTED), 'fixture must produce a genuine sanity result'); + } else { + assert.equal(areas.estimatedPositionsEnabled, false); + for (const key of ['estimatedNodes', 'positionGaps', 'unpositionedNoNeighborFix']) assert(!(key in areas)); + assert.deepEqual(sanity, { estimatedPositionsEnabled: false }); + } +} + +async function browserChecks(browser, base, enabled, mode, assets, shots) { + const context = await browser.newContext({ viewport: { width: 1360, height: 980 }, serviceWorkers: 'block' }); + // Serve exactly the already-used Leaflet version, preserving the index's + // integrity checks. Other external resources (including map tiles) cannot + // leave the browser; no API endpoint is intercepted or fulfilled here. + await context.route('**/*', async route => { + const url = new URL(route.request().url()); + if (url.origin === base) return route.continue(); + const asset = assets[url.href]; + if (asset) return route.fulfill({ path: asset, headers: { 'access-control-allow-origin': '*' } }); + return route.abort(); + }); + await context.addInitScript(() => { + localStorage.setItem('meshcore-theme', 'light'); + localStorage.setItem('map-estimated-nodes', 'true'); + localStorage.setItem('estimatedPositions', JSON.stringify({ enabled: true })); + localStorage.setItem('estimatedPositions.enabled', 'true'); + }); + const page = await context.newPage(); + page.setDefaultTimeout(10000); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + async function go(hash) { + await page.goto(base + '/' + hash, { waitUntil: 'load' }); + await page.waitForFunction(() => window.L && window.L.version === '1.9.4'); + await page.evaluate(() => window.MeshConfigReady); + } + async function screenshot(name) { + if (shots) await page.screenshot({ path: path.join(shots, '315-' + mode + '-' + name + '.png'), fullPage: false, animations: 'disabled' }); + } + async function nodeState(selector, hasGPS) { + await page.waitForSelector(selector); + if (enabled) { + await page.waitForFunction(sel => document.querySelector(sel).textContent.includes('(estimated)'), selector); + } else { + await page.waitForSelector(selector + ' [data-estimated-positions-disabled]'); + assert((await page.textContent(selector)).includes(DISABLED)); + assert(!(await page.textContent(selector)).includes('Neighbor Estimate')); + } + const mapSelector = selector === '#nodeFullBody' ? '#nodeFullMap' : '#nodeMap'; + if (hasGPS || enabled) { + await page.waitForSelector(mapSelector + '.leaflet-container'); + if (hasGPS) await page.waitForSelector(mapSelector + ' .leaflet-marker-icon'); + if (enabled) await page.waitForSelector(mapSelector + ' path[stroke-dasharray]'); + assert.equal(await page.locator(mapSelector + ' path[stroke-dasharray]').count(), enabled ? (hasGPS ? 2 : 1) : 0); + } else assert.equal(await page.locator(mapSelector).count(), 0); + if (hasGPS) assert((await page.textContent(selector)).includes('56.50000')); + } + try { + await go('#/nodes/' + REPORTED); + await nodeState('#nodeFullBody', true); + await screenshot('reported-full'); + await go('#/nodes/' + MISSING); + await nodeState('#nodeFullBody', false); + await screenshot('missing-full'); + await go('#/nodes?search=Position%20Policy'); + await page.click('tr[data-key="' + REPORTED + '"]'); + await nodeState('.node-detail', true); + await screenshot('reported-pane'); + await go('#/nodes?search=Position%20Policy'); + await page.click('tr[data-key="' + MISSING + '"]'); + await nodeState('.node-detail', false); + await screenshot('missing-pane'); + await go('#/packets/' + HASH + '?viewPath=1'); + await page.waitForSelector('#packetPathModal .leaflet-container'); + await page.waitForSelector('#packetPathModal path.leaflet-interactive'); + if (!enabled) { + assert((await page.textContent('#packetPathEstimatePolicy')).includes(DISABLED)); + assert.equal(await page.locator('#packetPathApproxLegend').isVisible(), false); + assert.equal(await page.locator('#packetPathModal path[stroke-dasharray]').count(), 0); + } else { + assert.equal(await page.locator('#packetPathApproxLegend').isVisible(), true); + assert(await page.locator('#packetPathModal path[stroke-dasharray]').count() > 0); + } + await screenshot('packet-path'); + await go('#/map?estimatedNodes=1'); + await page.waitForSelector('.mc-estimated-nodes-label'); + const label = await page.textContent('.mc-estimated-nodes-label'); + if (!enabled) assert(label.includes(DISABLED)); + else assert(!label.includes(DISABLED)); + const markers = await page.evaluate(() => { + const out = []; window.__mc_map.eachLayer(l => { + if (l instanceof L.CircleMarker || l instanceof L.Marker) out.push({ lat: l.getLatLng().lat, approx: Boolean(l.options.dashArray) }); + }); return out; + }); + if (!enabled) assert(markers.some(m => m.lat === 56.5 && !m.approx), 'reported GPS map marker must remain'); + assert.equal(markers.some(m => m.approx), enabled, 'query/localStorage must not override server policy'); + await screenshot('map'); + await go('#/tools/position-gaps'); + await page.waitForFunction(() => !document.querySelector('#position-gaps-content').textContent.includes('Loading')); + if (!enabled) { + assert((await page.textContent('#position-gaps-content')).includes(DISABLED)); + assert.equal(await page.locator('#position-gaps-est-table').count(), 0); + } else await page.waitForSelector('#position-gaps-est-table'); + await screenshot('position-gaps'); + await go('#/tools/gps-sanity'); + await page.waitForFunction(() => !document.querySelector('#gps-sanity-content').textContent.includes('Loading')); + if (!enabled) { + assert((await page.textContent('#gps-sanity-content')).includes(DISABLED)); + assert.equal(await page.locator('#gps-sanity-table').count(), 0); + } else await page.waitForSelector('#gps-sanity-table'); + await screenshot('gps-sanity'); + await go('#/analytics?tab=areas'); + await page.waitForSelector('#areasDensity table'); + if (!enabled) { + assert((await page.textContent('#areasPositionGaps')).includes(DISABLED)); + assert.equal(await page.locator('#areasViewEstimatedNodes').count(), 0); + } else await page.waitForSelector('#areasViewEstimatedNodes'); + await screenshot('areas'); + if (!enabled) { + await page.setViewportSize({ width: 390, height: 844 }); + await go('#/nodes/' + REPORTED); + await nodeState('#nodeFullBody', true); + await page.locator('[data-estimated-positions-disabled]').scrollIntoViewIfNeeded(); + await screenshot('reported-mobile'); + } + assert.deepEqual(errors, [], 'browser runtime errors'); + } catch (error) { + await screenshot('failure'); + throw error; + } finally { await context.close(); } +} + +(async () => { + const serverBin = requiredFile('CORESCOPE_SERVER_BIN'); + const migrateBin = requiredFile('CORESCOPE_MIGRATE_BIN'); + const assets = { + 'https://unpkg.com/leaflet@1.9.4/dist/leaflet.js': requiredFile('CORESCOPE_LEAFLET_DIR', 'leaflet.js'), + 'https://unpkg.com/leaflet@1.9.4/dist/leaflet.css': requiredFile('CORESCOPE_LEAFLET_DIR', 'leaflet.css'), + }; + for (const image of ['marker-icon.png', 'marker-icon-2x.png', 'marker-shadow.png']) { + const local = path.join(process.env.CORESCOPE_LEAFLET_DIR, 'images', image); + if (fs.existsSync(local)) assets['https://unpkg.com/leaflet@1.9.4/dist/images/' + image] = local; + } + const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-315-e2e-')); + const shots = process.env.SCREENSHOT_DIR && path.resolve(process.env.SCREENSHOT_DIR); + if (shots) fs.mkdirSync(shots, { recursive: true }); + let browser, child; + try { + const database = path.join(scratch, 'synthetic.db'); + // Copy only the tracked fixture's schema: none of its real mesh records + // enter this suite. Migrations remain owned by the dedicated Go binary. + const schema = execFileSync('sqlite3', [path.join(ROOT, 'test-fixtures/e2e-fixture.db'), + "SELECT sql || ';' FROM sqlite_master WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' ORDER BY CASE type WHEN 'table' THEN 0 ELSE 1 END, rowid"]); + execFileSync('sqlite3', [database], { input: schema }); + execFileSync(migrateBin, ['-db', database], { stdio: 'pipe' }); + execFileSync('sqlite3', [database], { input: fs.readFileSync(path.join(ROOT, 'test-fixtures/seed-315-estimated-positions.sql')) }); + browser = await chromium.launch({ headless: true, executablePath: process.env.CHROMIUM_PATH || undefined }); + for (const mode of ['default', 'enabled', 'disabled']) { + const enabled = mode !== 'disabled'; + const config = { + areas: { synthetic: { label: 'Synthetic Test Area', latMin: 54, latMax: 58, lonMin: 11, lonMax: 16 } }, + mapDefaults: { center: [55.7, 12], zoom: 7 }, mqtt: { brokers: [] }, + }; + if (mode !== 'default') config.estimatedPositions = { enabled }; + fs.writeFileSync(path.join(scratch, 'config.json'), JSON.stringify(config)); + const port = await freePort(); + const base = 'http://127.0.0.1:' + port; + let log = ''; + child = spawn(serverBin, ['-config-dir', scratch, '-db', database, '-public', path.join(ROOT, 'public'), '-port', String(port)], { cwd: ROOT, stdio: ['ignore', 'pipe', 'pipe'] }); + child.stdout.on('data', chunk => { log = (log + chunk).slice(-30000); }); + child.stderr.on('data', chunk => { log = (log + chunk).slice(-30000); }); + await waitReady(base, child, () => log); + await apiChecks(base, enabled); + await browserChecks(browser, base, enabled, mode, assets, shots); + console.log('PASS #315 ' + mode + ': real API, node detail/pane, map, tools, areas'); + await stop(child); child = null; + } + } finally { + await stop(child); + if (browser) await browser.close(); + fs.rmSync(scratch, { recursive: true, force: true }); + } +})().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/test-packet-path-map.js b/test-packet-path-map.js index f374b1fac..a6c0d4b7f 100644 --- a/test-packet-path-map.js +++ b/test-packet-path-map.js @@ -207,6 +207,32 @@ function makeSandbox(apiImpl) { const archivedPath = { hash: 'historic', branches: [{ hops: 0, points: [], observer: { name: 'Saved station', lat: 56, lon: 10 } }] }; const pingOptions = (loadPath) => ({ loadPath, routePrefix: '#/ping-scores/', routeQueryKey: 'record', shareURL: 'https://stg.meshview.dk/#/ping-scores/historic?viewPath=1&record=allTime.farthestPing' }); + await historyCase('operator policy waits for config, hides approximate points/controls, and keeps real GPS', async () => { + let finishConfig; + const ctx = makeSandbox(async () => ({ branches: [{ hops: 2, points: [ + { name: 'Estimated relay', lat: 56.1, lon: 10.1, approx: true }, + { name: 'Reported relay', lat: 56.2, lon: 10.2 } + ], observer: { name: 'Estimated observer', lat: 56.3, lon: 10.3, approx: true } }] })); + ctx.fetch = () => new Promise(resolve => { finishConfig = resolve; }); + ctx.document.querySelector = () => null; + ctx.document.head = { appendChild() {} }; + vm.runInContext(fs.readFileSync('public/roles.js', 'utf8'), ctx); + const counts = installHistoryLeaflet(ctx); + const coordinates = []; + ctx.L.circleMarker = point => { coordinates.push(point); return { addTo() { return this; }, bindTooltip() { return this; }, on() { return this; } }; }; + const opened = ctx.window.PacketPathMap.open('policy-test'); + await Promise.resolve(); + assert.strictEqual(counts.maps, 0, 'no pre-config estimate map'); + finishConfig({ json: async () => ({ estimatedPositions: { enabled: false } }) }); + await opened; + assert.strictEqual(coordinates.length, 1); + assert.strictEqual(coordinates[0][0], 56.2, 'only reported GPS is plotted'); + assert.strictEqual(ctx.document.getElementById('packetPathApproxLegend').style.display, 'none'); + assert.ok(ctx.document.getElementById('packetPathEstimatePolicy').textContent.includes('disabled by the instance operator')); + assert.strictEqual(ctx.document.getElementById('packetPathApproxOnly'), null); + ctx.window.PacketPathMap.close(); + }); + await historyCase('ping archive loader renders a saved-time disclaimer and copies a ping-specific URL', async () => { const ctx = makeSandbox(() => { throw new Error('ordinary packet API must not be used'); }); const counts = installHistoryLeaflet(ctx);