Preserve Ping Scores records and archived paths

cmd/server/db.go

@@ -2190,6 +2190,10 @@ type PacketPathObserver struct {
 	Approx              bool     `json:"approx,omitempty"`
 	ApproxNeighborCount int      `json:"approxNeighborCount,omitempty"`
 	ApproxSpreadKm      *float64 `json:"approxSpreadKm,omitempty"`
+	// Internal scoring evidence only: airport coordinates are a display
+	// fallback, not proof that an observer's own GPS still exists. Kept out
+	// of the wire/archive shape so existing packet-path clients are unchanged.
+	iataFallback bool
 }
 
 // PacketPathBranch is one station's route to a packet: how far it
@@ -2740,6 +2744,7 @@ func buildPacketPathResponseFromReduction(
 				if coord, ok := iataCoords[obs.IATA]; ok {
 					lat, lon := coord.Lat, coord.Lon
 					obs.Lat, obs.Lon = &lat, &lon
+					obs.iataFallback = true
 				}
 			}
 			if obs.Lat == nil && b.observerPubkey != "" {

cmd/server/openapi.go

@@ -195,6 +195,9 @@ func routeDescriptions() map[string]routeMeta {
 		"GET /api/audio-lab/buckets": {Summary: "Audio lab frequency buckets", Description: "Returns frequency bucket data for audio analysis.", Tag: "analytics"},
 		"GET /api/ping-scores": {Summary: "Ping-score highscore board", Description: "Global (not scoped by region/area) records and leaderboards derived from every ping-bot-triggering channel message ever seen: farthest reach, most hops, widest simultaneous spread, fastest full spread, and most airtime-efficient ping, plus which relay nodes and which observers appear most often. Computed from the same GetPacketPath + LoRa-airtime-estimate logic behind /api/packets/{hash}/path and refreshed on a background interval, so it may lag the very latest ping by a few minutes. Fields are omitted (not zero) until at least one qualifying ping has been recorded.", Tag: "packets",
 			Response: schemaRef("PingScoresResponse")},
+		"GET /api/ping-scores/{hash}/path": {Summary: "Get a displayed ping record's saved path", Description: "Returns coherent live or archived path evidence for the current record slot. Archived capture time describes saved geometry, not necessarily the transmission time. Old expired observations cannot be reconstructed. Superseded slot/hash pairs return 404; invalid slots return 400. Current identity privacy rules apply to both sources; unavailable and initializing responses omit path.", Tag: "packets",
+			QueryParams: []paramMeta{{Name: "record", Description: "allTime.<kind> or thisWeek.<kind>; kind is farthestPing, mostHopsPing, widestSpreadPing, fastestSpreadPing or mostEfficientPing", Type: "string", Required: true}},
+			Response:    schemaRef("PingScorePathResponse")},
 		"GET /api/analytics/areas": {Summary: "Per-configured-Area node density, cross-area bridge nodes, and position-fix coverage", Description: "Three breakdowns over the drawn-polygon Areas configured via the meshguide.dk sync, distinct from hashRegion scope adoption (see /api/analytics/scope-stats): (1) density, node count/active-degraded-silent health/role mix per area (multi-membership via AreaKeysForPoint, so a node in a sub-area also counts toward its parent region), (2) bridgeNodes, nodes whose packet-derived neighbor_edges reach into at least one OTHER area (single most-specific area via AreaKeyForPoint), ranked by how many other areas they reach -- distinct from the network-wide, area-unaware bridge_score betweenness centrality, (3) positionGaps, per area how many nodes have a real GPS fix vs. how many were only placeable via the same neighbor-centroid estimate View Path's approx markers use (nearestPositionedNeighbor, geo-sanity-filtered by Config.NeighborMaxEdgeKm so a stray MQTT-bridge observer↔last-hop edge hundreds of km away can't skew the estimate or inflate its spreadKm). estimatedNodes is the flat, network-wide list backing positionGaps' approximated counts, with actual estimated coordinates -- used by the Areas tab's \"View Estimated Nodes\" map view and Tools > Position-Fix Coverage Gaps. Returns an empty response if no Areas are configured. Cached 30s.", Tag: "analytics",
 			Response: schemaRef("AreaAnalyticsResponse")},
 		"GET /api/analytics/gps-sanity": {Summary: "Nodes whose self-reported GPS disagrees with their own RF neighbors", Description: "The neighbor-centroid technique nearestPositionedNeighbor uses to ESTIMATE a position for a node with no GPS, flipped around to sanity-check a node that DOES report one. For each node with a real (non-zero) GPS fix, takes its strongest neighbor_edges neighbor as an anchor, keeps whichever other positioned neighbors agree with the anchor within GPSSanityClusterTightKm (50km), and -- only if at least GPSSanityMinClusterSize (2) survive that filter -- compares the node's own position against their weighted centroid. Flags it when the distance exceeds GPSSanitySuspectKm (100km). Most nodes are skipped, not evaluated (no neighbor_edges, no positioned neighbor, or too scattered a neighbor set to trust), so evaluated is always well under totalRealGps. v1: doesn't weight by neighbor_edges' hash-prefix ambiguity mode (the confidence indicator public/nodes.js's Neighbors panel shows) since that breakdown only lives in the in-memory NeighborGraph, not the persisted table this reads. Not area-scoped -- works regardless of whether Areas are configured. Cached 30s.", Tag: "analytics",
@@ -551,11 +554,17 @@ func componentSchemas() map[string]interface{} {
 				"airtimeRelayCount":  map[string]interface{}{"type": "integer", "description": "Distinct relay count behind estimatedAirtimeMs. Present only alongside it."},
 			},
 		},
+		"PingScorePathResponse": &openAPISchema{Type: "object", Properties: map[string]*openAPISchema{
+			"status":     {Type: "string", Enum: []string{"live", "archived", "unavailable", "initializing"}},
+			"capturedAt": {Type: "string", Description: "UTC archive capture time, present for archived geometry."},
+			"reason":     {Type: "string", Enum: []string{"raw_data_expired_before_capture", "no_coordinates", "privacy_filtered", "archive_too_large", "record_evidence_unavailable"}},
+			"path":       openAPIRef("PacketPathResponse"),
+		}},
 		"PingScore": map[string]interface{}{
 			"type":        "object",
 			"description": "One ping's computed highscore-relevant stats, derived from the same GetPacketPath + airtime-annotation logic behind /api/packets/{hash}/path.",
 			"properties": map[string]interface{}{
-				"hash":               str("The winning ping's packet hash -- pass to /api/packets/{hash}/path for the full View Path map."),
+				"hash":               str("The winning ping's hash; use /api/ping-scores/{hash}/path with its record slot for saved View Path evidence."),
 				"sender":             str("Display name of whoever sent the ping, when resolvable from the channel message."),
 				"channelHash":        str("Which channel the ping was sent on."),
 				"timestamp":          str("RFC3339 timestamp the ping was first seen."),

cmd/server/ping_score_history.go

@@ -52,7 +52,9 @@ import (
 // PingScoreHistoryEntry.PermanentlyUnreconstructable) and the new
 // ping_score_history_gaps table (see PingScoreHistoryGap) -- both purely
 // additive, see applyPingScoreHistoryV2.
-const pingScoreHistorySchemaVersion = 2
+// v3 adds bounded displayed-record path archives and a nullable distance
+// origin fact in this sidecar. It does not alter or scan the main DB.
+const pingScoreHistorySchemaVersion = 3
 
 // PingScoreHistoryStore owns the separate, server-only SQLite connection to
 // ping_scores_history.db. Not safe for concurrent use from multiple
@@ -91,12 +93,16 @@ type PingScoreHistoryEntry struct {
 	RelayCount       int
 	RelayPubkeysJSON string
 	FirstPubkey      string
-	Unscorable       bool
-	FingerprintCount int64
-	FingerprintMaxID int64
-	StableSince      string
-	Settled          bool
-	DataPruned       bool
+	// DistanceFirstPubkey is the landmark behind FarthestKm, independent
+	// of FirstPubkey's actual earliest-hearer credit. Empty means legacy
+	// metadata: materialization falls back to FirstPubkey.
+	DistanceFirstPubkey string
+	Unscorable          bool
+	FingerprintCount    int64
+	FingerprintMaxID    int64
+	StableSince         string
+	Settled             bool
+	DataPruned          bool
 
 	// PermanentlyUnreconstructable is true only once a REAL deep-sweep
 	// attempt (not an age estimate) has actually proven, this tx_id had
@@ -550,6 +556,7 @@ type pingScoreHistoryMigration struct {
 var pingScoreHistoryMigrations = []pingScoreHistoryMigration{
 	{toVersion: 1, apply: applyPingScoreHistoryV1},
 	{toVersion: 2, apply: applyPingScoreHistoryV2},
+	{toVersion: 3, apply: applyPingScoreHistoryV3},
 }
 
 func (s *PingScoreHistoryStore) migrateFrom(fromVersion int) error {
@@ -710,18 +717,18 @@ const pingScoreHistoryUpsertSQL = `
 INSERT INTO ping_score_history_entries (
 	tx_id, hash, sender, channel_hash, timestamp, station_count, deepest_hops,
 	deepest_pubkey, farthest_km, farthest_pubkey, spread_seconds, airtime_ms,
-	relay_count, relay_pubkeys_json, first_pubkey, unscorable,
+	relay_count, relay_pubkeys_json, first_pubkey, distance_first_pubkey, unscorable,
 	fingerprint_count, fingerprint_max_id, stable_since, settled, data_pruned,
 	permanently_unreconstructable, last_deep_swept_at, computed_at
-) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
 ON CONFLICT(tx_id) DO UPDATE SET
 	hash=excluded.hash, sender=excluded.sender, channel_hash=excluded.channel_hash,
 	timestamp=excluded.timestamp, station_count=excluded.station_count,
 	deepest_hops=excluded.deepest_hops, deepest_pubkey=excluded.deepest_pubkey,
 	farthest_km=excluded.farthest_km, farthest_pubkey=excluded.farthest_pubkey,
 	spread_seconds=excluded.spread_seconds, airtime_ms=excluded.airtime_ms,
 	relay_count=excluded.relay_count, relay_pubkeys_json=excluded.relay_pubkeys_json,
-	first_pubkey=excluded.first_pubkey, unscorable=excluded.unscorable,
+	first_pubkey=excluded.first_pubkey, distance_first_pubkey=excluded.distance_first_pubkey, unscorable=excluded.unscorable,
 	fingerprint_count=excluded.fingerprint_count, fingerprint_max_id=excluded.fingerprint_max_id,
 	stable_since=excluded.stable_since, settled=excluded.settled, data_pruned=excluded.data_pruned,
 	permanently_unreconstructable=excluded.permanently_unreconstructable,
@@ -759,8 +766,9 @@ func (s *PingScoreHistoryStore) UpsertDeleteAndIntegrity(upserts []PingScoreHist
 // delete, and up to three optional metadata writes -- an integrity record,
 // a gap record, and the one-time history-initialized marker -- all as ONE
 // SQL transaction: either everything lands, or (on any error) NONE of it
-// does -- the deferred Rollback is a no-op after a successful Commit, and
-// fires on every error path before this function returns.
+// does. Statement failures use deferred Rollback; a final Commit failure
+// additionally clears SQLite's still-open transaction on the sole pooled
+// connection before this function returns.
 //
 // This exists (Phase 4D, extended in the fix-round-2 review of a1c3022d)
 // because none of these metadata writes are safe to persist as a SEPARATE
@@ -784,10 +792,17 @@ func (s *PingScoreHistoryStore) UpsertDeleteAndIntegrity(upserts []PingScoreHist
 //     value is an RFC3339 timestamp to record under the `_meta` key
 //     'history_initialized_at'.
 func (s *PingScoreHistoryStore) UpsertDeleteAndMetadata(upserts []PingScoreHistoryEntry, deleteTxIDs []int64, integrity *PingScoreHistoryIntegrity, gap *PingScoreHistoryGap, historyInitializedAt *string) error {
+	return s.upsertDeleteMetadataAndArchives(upserts, deleteTxIDs, integrity, gap, historyInitializedAt, nil)
+}
+
+// A non-nil archives map replaces the complete bounded displayed-slot set
+// in the very same transaction as this cycle's scores. A nil map preserves
+// the set (legacy callers, or an unchanged path archive set).
+func (s *PingScoreHistoryStore) upsertDeleteMetadataAndArchives(upserts []PingScoreHistoryEntry, deleteTxIDs []int64, integrity *PingScoreHistoryIntegrity, gap *PingScoreHistoryGap, historyInitializedAt *string, archives map[string]PingScorePathArchive) error {
 	if s.readOnly {
 		return fmt.Errorf("ping score history store: read-only (on-disk schema is newer than this code understands)")
 	}
-	if len(upserts) == 0 && len(deleteTxIDs) == 0 && integrity == nil && gap == nil && historyInitializedAt == nil {
+	if len(upserts) == 0 && len(deleteTxIDs) == 0 && integrity == nil && gap == nil && historyInitializedAt == nil && archives == nil {
 		return nil
 	}
 
@@ -809,7 +824,7 @@ func (s *PingScoreHistoryStore) UpsertDeleteAndMetadata(upserts []PingScoreHisto
 				e.StationCount, e.DeepestHops, nullableString(e.DeepestPubkey),
 				nullableFloat(e.FarthestKm), nullableString(e.FarthestPubkey), nullableFloat(e.SpreadSeconds),
 				nullableFloat(e.AirtimeMs), e.RelayCount, nullableString(e.RelayPubkeysJSON),
-				nullableString(e.FirstPubkey), boolToInt(e.Unscorable),
+				nullableString(e.FirstPubkey), nullableString(e.DistanceFirstPubkey), boolToInt(e.Unscorable),
 				e.FingerprintCount, e.FingerprintMaxID, nullableString(e.StableSince),
 				boolToInt(e.Settled), boolToInt(e.DataPruned), boolToInt(e.PermanentlyUnreconstructable),
 				nullableString(e.LastDeepSweptAt), e.ComputedAt,
@@ -892,7 +907,24 @@ func (s *PingScoreHistoryStore) UpsertDeleteAndMetadata(upserts []PingScoreHisto
 		}
 	}
 
+	if err := writePingScorePathArchives(tx, archives); err != nil {
+		return fmt.Errorf("ping score history store: record path archives: %w", err)
+	}
 	if err := tx.Commit(); err != nil {
+		// database/sql marks a Tx done before asking SQLite to COMMIT.
+		// SQLite can leave the transaction open when COMMIT fails (for
+		// example on a deferred constraint), so tx.Rollback's deferred
+		// call would then be an ErrTxDone no-op. This single-owner pool
+		// has exactly one connection: explicitly clear that transaction
+		// before any later read/write can see its uncommitted candidate.
+		if _, rollbackErr := s.conn.Exec(`ROLLBACK`); rollbackErr != nil {
+			// A lost/automatically rolled-back connection may report that
+			// there is no active transaction. Retiring the pool is safe in
+			// either case and prevents a genuinely poisoned connection
+			// from being used for a subsequent publication.
+			s.conn.Close()
+			return fmt.Errorf("ping score history store: commit: %w; rollback cleanup: %v", err, rollbackErr)
+		}
 		return fmt.Errorf("ping score history store: commit: %w", err)
 	}
 	return nil
@@ -903,14 +935,27 @@ func (s *PingScoreHistoryStore) UpsertDeleteAndMetadata(upserts []PingScoreHisto
 // future column from a newer schema version) never changes what this
 // specific code version reads.
 func (s *PingScoreHistoryStore) LoadAll() ([]PingScoreHistoryEntry, error) {
-	rows, err := s.conn.Query(`
+	distanceOriginColumn := "distance_first_pubkey"
+	if s.readOnly {
+		// A future-version file may have the old-compatible score table
+		// without this optional column. Stay physically read-only and
+		// retain its legacy origin semantics rather than migrating it.
+		var present int
+		if err := s.conn.QueryRow(`SELECT COUNT(*) FROM pragma_table_info('ping_score_history_entries') WHERE name='distance_first_pubkey'`).Scan(&present); err != nil {
+			return nil, fmt.Errorf("ping score history store: inspect distance origin column: %w", err)
+		}
+		if present == 0 {
+			distanceOriginColumn = "NULL"
+		}
+	}
+	rows, err := s.conn.Query(fmt.Sprintf(`
 		SELECT tx_id, hash, sender, channel_hash, timestamp, station_count, deepest_hops,
 			deepest_pubkey, farthest_km, farthest_pubkey, spread_seconds, airtime_ms,
-			relay_count, relay_pubkeys_json, first_pubkey, unscorable,
+			relay_count, relay_pubkeys_json, first_pubkey, %s, unscorable,
 			fingerprint_count, fingerprint_max_id, stable_since, settled, data_pruned,
 			permanently_unreconstructable, last_deep_swept_at, computed_at
 		FROM ping_score_history_entries
-		ORDER BY tx_id`)
+		ORDER BY tx_id`, distanceOriginColumn))
 	if err != nil {
 		return nil, fmt.Errorf("ping score history store: load all: %w", err)
 	}
@@ -919,15 +964,15 @@ func (s *PingScoreHistoryStore) LoadAll() ([]PingScoreHistoryEntry, error) {
 	var out []PingScoreHistoryEntry
 	for rows.Next() {
 		var e PingScoreHistoryEntry
-		var sender, channelHash, deepestPubkey, farthestPubkey, relayPubkeysJSON, firstPubkey sql.NullString
+		var sender, channelHash, deepestPubkey, farthestPubkey, relayPubkeysJSON, firstPubkey, distanceFirstPubkey sql.NullString
 		var stableSince, lastDeepSweptAt sql.NullString
 		var farthestKm, spreadSeconds, airtimeMs sql.NullFloat64
 		var relayCount sql.NullInt64
 		var unscorable, settled, dataPruned, permanentlyUnreconstructable int
 		if err := rows.Scan(
 			&e.TxID, &e.Hash, &sender, &channelHash, &e.Timestamp, &e.StationCount, &e.DeepestHops,
 			&deepestPubkey, &farthestKm, &farthestPubkey, &spreadSeconds, &airtimeMs,
-			&relayCount, &relayPubkeysJSON, &firstPubkey, &unscorable,
+			&relayCount, &relayPubkeysJSON, &firstPubkey, &distanceFirstPubkey, &unscorable,
 			&e.FingerprintCount, &e.FingerprintMaxID, &stableSince, &settled, &dataPruned,
 			&permanentlyUnreconstructable, &lastDeepSweptAt, &e.ComputedAt,
 		); err != nil {
@@ -939,6 +984,7 @@ func (s *PingScoreHistoryStore) LoadAll() ([]PingScoreHistoryEntry, error) {
 		e.FarthestPubkey = farthestPubkey.String
 		e.RelayPubkeysJSON = relayPubkeysJSON.String
 		e.FirstPubkey = firstPubkey.String
+		e.DistanceFirstPubkey = distanceFirstPubkey.String
 		e.StableSince = stableSince.String
 		e.LastDeepSweptAt = lastDeepSweptAt.String
 		if farthestKm.Valid {

cmd/server/ping_score_history_convert.go

@@ -146,6 +146,7 @@ func pingScoreHistoryEntryFromScore(
 	e.RelayCount = score.RelayCount
 	e.RelayPubkeysJSON = marshalRelayPubkeysJSON(score.relayPubkeys)
 	e.FirstPubkey = score.firstPubkey
+	e.DistanceFirstPubkey = score.distanceFirstPubkey
 	return e
 }
 
@@ -164,7 +165,8 @@ func pingScoreHistoryEntryFromScore(
 // for the full rationale):
 //   - score != nil ("a successful computation") replaces every path fact
 //     (StationCount, DeepestHops, DeepestPubkey, FarthestKm, FarthestPubkey,
-//     SpreadSeconds, RelayPubkeysJSON, FirstPubkey) with this cycle's
+//     SpreadSeconds, RelayPubkeysJSON, FirstPubkey, DistanceFirstPubkey)
+//     with this cycle's
 //     values, and clears Unscorable AND PermanentlyUnreconstructable (see
 //     the score != nil branch below for why the latter is a defensive
 //     clear, not an expected-to-fire code path).
@@ -218,6 +220,7 @@ func mergePingScoreHistoryEntry(
 		merged.FarthestPubkey = score.FarthestPubkey
 		merged.SpreadSeconds = score.SpreadSeconds
 		merged.FirstPubkey = score.firstPubkey
+		merged.DistanceFirstPubkey = score.distanceFirstPubkey
 		merged.RelayPubkeysJSON = marshalRelayPubkeysJSON(score.relayPubkeys)
 
 		airtimeAlreadyLocked := existing.AirtimeMs != nil && *existing.AirtimeMs > 0
@@ -276,20 +279,26 @@ func materializePingScoreFromHistoryEntry(e PingScoreHistoryEntry) (*PingScore,
 		return nil, fmt.Errorf("materialize tx_id=%d: %w", e.TxID, err)
 	}
 	score := &PingScore{
-		Hash:           e.Hash,
-		Sender:         e.Sender,
-		ChannelHash:    e.ChannelHash,
-		Timestamp:      e.Timestamp,
-		StationCount:   e.StationCount,
-		DeepestHops:    e.DeepestHops,
-		DeepestPubkey:  e.DeepestPubkey,
-		FarthestKm:     e.FarthestKm,
-		FarthestPubkey: e.FarthestPubkey,
-		SpreadSeconds:  e.SpreadSeconds,
-		AirtimeMs:      e.AirtimeMs,
-		RelayCount:     e.RelayCount,
-		relayPubkeys:   relayPubkeys,
-		firstPubkey:    e.FirstPubkey,
+		Hash:                e.Hash,
+		Sender:              e.Sender,
+		ChannelHash:         e.ChannelHash,
+		Timestamp:           e.Timestamp,
+		StationCount:        e.StationCount,
+		DeepestHops:         e.DeepestHops,
+		DeepestPubkey:       e.DeepestPubkey,
+		FarthestKm:          e.FarthestKm,
+		FarthestPubkey:      e.FarthestPubkey,
+		SpreadSeconds:       e.SpreadSeconds,
+		AirtimeMs:           e.AirtimeMs,
+		RelayCount:          e.RelayCount,
+		relayPubkeys:        relayPubkeys,
+		firstPubkey:         e.FirstPubkey,
+		distanceFirstPubkey: e.DistanceFirstPubkey,
+	}
+	if score.distanceFirstPubkey == "" {
+		// A v1/v2 history row had only one origin fact, so its stored
+		// first station was also the origin used for its distance.
+		score.distanceFirstPubkey = e.FirstPubkey
 	}
 	if e.FarthestKm != nil && e.AirtimeMs != nil && *e.AirtimeMs > 0 {
 		kmPerSec := *e.FarthestKm / (*e.AirtimeMs / 1000.0)