Fix cross-aircraft cluster contamination in dark solver candidates

backend/.env.example

@@ -129,9 +129,28 @@ MENDER_PAT=
 # from its association guess before it is rejected; the default is 6 km = 2 x
 # the 3 km association grid step, since a dark guess is a quantised grid point
 # rather than an ADS-B fix. ADS-B-anchored solves keep the fixed 2 km cap.
+#
+# SOLVER_ALT_MODE is how an n>=3 solve gets its altitude. sweep (default)
+# solves once per fixed altitude layer and keeps the lowest rms_delay — six
+# process-pool round trips per candidate, and an altitude quantised to a
+# ladder whose 2 km spacing puts up to 1 km of error straight into the
+# residual the reject gate reads. free makes one pool call to the geolocator's
+# multi-start helper, which solves altitude as a sixth unknown. Both modes
+# stamp altitude_mode on the solve-history record, so /api/test/mlat-history
+# can compare them across a deploy of each. See docs/solverflow.md.
+#
+# SOLVER_FREE_ALT_STARTS is how many start altitudes free mode gives that
+# helper, clamped to the number of layers. 1 (the default) starts at the layer
+# nearest the association guess, or at the guess altitude itself when it comes
+# from ADS-B; more is a window around it, at one LM run each. Measured on test
+# over 1019 free solves, three starts changed rms_delay by more than 0.1 us in
+# 13 of them, so the extra runs are off by default and worth turning on only
+# where the geometry sends a single start to the wrong side of an ellipse.
 # SOLVER_WORKERS=2
 # SOLVER_RESOLVE_INTERVAL_S=12
 # SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0
+# SOLVER_ALT_MODE=sweep
+# SOLVER_FREE_ALT_STARTS=1
 
 # Detection mirror. Production only. Every accepted v1 detection frame is
 # forwarded to another environment's /api/radar/detections/bulk, batched once a

backend/core/state.py

@@ -74,6 +74,45 @@
 if KNOWN_LANE_MODE not in ("off", "shadow", "binding"):
     KNOWN_LANE_MODE = "shadow"
 
+# How the n>=3 solve gets its altitude (see services/tasks/solver.py's
+# _solve_best_altitude).  sweep/free, read here rather than in that module so
+# it sits with its sibling mode flags and a test can monkeypatch it without
+# reimporting the solver.
+#   sweep (default) — solve once per fixed altitude layer, keep the lowest
+#           rms_delay.  Six pool round trips per candidate, and an altitude
+#           quantised to the ladder: layers are 2 km apart, so the pin is
+#           systematically up to 1 km wrong and that error lands in the
+#           residual the reject gate reads.
+#   free  — one pool call to retina_geolocator's multi-start helper, which
+#           solves altitude as a sixth unknown, started from
+#           SOLVER_FREE_ALT_STARTS of those layers.
+# Not off/shadow/active: there is no shadow here, because the two modes
+# produce the same shape of result and the history record carries
+# altitude_mode either way — running both would double the solver's cost to
+# learn what one deploy of each already says.  An unrecognised value falls
+# back to "sweep", the same degrade-to-inert rule the sibling flags use.
+SOLVER_ALT_MODE = os.getenv("SOLVER_ALT_MODE", "sweep").lower()
+if SOLVER_ALT_MODE not in ("sweep", "free"):
+    SOLVER_ALT_MODE = "sweep"
+
+# How many start altitudes the free mode hands that helper.  Read here beside
+# the mode it qualifies; _free_alt_starts in services/tasks/solver.py clamps it
+# into [1, len(layers)] against the ladder that module owns.  1 starts at the
+# layer nearest the association guess — where the sweep would have pinned;
+# more is a window around it.
+#
+# The default is 1 because three starts did not pay for themselves: over 1019
+# free-mode solves on test, the three starts' rms_delay differed by more than
+# 0.1 us in 13 of them, and the nearest-layer start was more than 0.5 us worse
+# than the best start in 2.  That is ~0.2% of solves helped for 3x the solver
+# CPU, and the pool — not the altitude ladder — is what this deployment is
+# short of (~1.7 attempts/s against a 2.0 s average latency on two workers).
+# The knob stays because the reason for several starts is the LM's locality,
+# which is a property of the geometry rather than of this fleet: nodes lying
+# nearer a bistatic ellipse than these can send a single start to the wrong
+# side of it, and finding that out should not need a code change.
+SOLVER_FREE_ALT_STARTS = max(1, int(os.getenv("SOLVER_FREE_ALT_STARTS", "1")))
+
 node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE)
 
 
@@ -475,6 +514,15 @@ def _adsb_for_seeding() -> dict[str, dict]:
 
 # Monotonic counter for dropped frames (useful for monitoring)
 frames_dropped: int = 0
+# Frames the per-node rate limiter refused before they ever reached
+# frame_queue (tcp_handler's NODE_FRAME_MIN_INTERVAL_S gate).  A different
+# event from frames_dropped, which is queue saturation: this one is the
+# pipeline deliberately sampling a node down to ~1 Hz, and a node streaming at
+# 22 fps therefore reports a large number here while dropping nothing.  It was
+# uncounted, so "how much of a node's evidence does the tracker actually see"
+# had no answer at all — the frames_dropped that IS published
+# (/api/admin/metrics) says zero throughout.
+node_frames_rate_limited: int = 0
 frames_processed: int = 0
 solver_successes: int = 0
 solver_failures: int = 0
@@ -535,14 +583,36 @@ def _adsb_for_seeding() -> dict[str, dict]:
 solver_stale_drops: int = 0
 
 # Candidates dequeued and skipped because every single-node track they carry
-# was already solved within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes (see
-# solver.py's _claim_resolve_slot).  Association is per-node and rate-limited
-# per node, so one aircraft arrives as one candidate per node that can see it;
-# this counts the copies that were never worth solving.  High against
-# solver_successes is normal and is the mechanism working — it is
-# solver_stale_drops that means work was lost.
+# was already PUBLISHED within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes
+# (see solver.py's _resolve_slot_covered).  Association is per-node and
+# rate-limited per node, so one aircraft arrives as one candidate per node
+# that can see it; this counts the copies that were never worth solving.  High
+# against solver_successes is normal and is the mechanism working — it is
+# solver_stale_drops that means work was lost.  Read it against
+# solver_successes, not against attempts: while the claim was taken on
+# ADMISSION rather than on publication, a rejected candidate blacked out every
+# later one sharing a track id and this counter ran at ~2.4x attempts.
 solver_resolve_skips: int = 0
 
+# The dark-lane share of the counter above, split out because the two lanes
+# read completely differently: an ADS-B-anchored duplicate that is skipped
+# costs nothing (the transponder keeps the track alive anyway), while a
+# skipped dark candidate may be the only chance that aircraft had of reaching
+# the map this window.  Lane is decided by solver._is_dark_solver_input, the
+# same predicate routes.test._record_lane falls back to for a record that
+# never got a key — and a skip never gets one.
+solver_resolve_skips_dark: int = 0
+
+# The last few hundred resolve-slot skips, with the claims that blocked them.
+# Deliberately NOT the solve-history deque: a skip is not a solve outcome, and
+# writing one record per skip into mlat_solve_history would evict the real
+# records at roughly twice their rate (live: ~1 537 skips per 646 dark
+# attempts per 30 min).  Small and separate, read by
+# /api/test/solver-stats' resolve_skips block and dumped by
+# /api/test/mlat-history?kind=resolve_skips.  ~250 B/entry.
+SOLVER_RESOLVE_SKIPS_RECENT_MAX = 500
+solver_resolve_skips_recent: deque = deque(maxlen=SOLVER_RESOLVE_SKIPS_RECENT_MAX)
+
 # Multinode entries removed because a later solve shared a source single-node
 # track with them AND the spatial/identical-inputs guard in solver.py's
 # _supersession_match agreed they are the same aircraft — the age-scaled
@@ -746,12 +816,14 @@ def _reset_for_tests() -> None:
     global latest_mlat_accuracy_bytes, latest_mlat_verification_bytes
     global latest_storage_bytes, simulation_config
     global frames_dropped, frames_processed, solver_successes, solver_failures
+    global node_frames_rate_limited
     global adsb_seed_frames_autotagged, adsb_capture_ts_fallback
     global known_claims_made, known_claim_contentions, known_claims_bound
     global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects
     global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes
     global coverage_rebuild_backlog
     global solver_queue_drops, solver_stale_drops, solver_resolve_skips
+    global solver_resolve_skips_dark
     global mn_superseded, mn_superseded_blocked, solver_trimmed
     global solver_consensus_selected, solver_consensus_filtered
     global solver_consensus_fallback, solver_consensus_shadow
@@ -801,6 +873,7 @@ def _reset_for_tests() -> None:
     track_archive_buffer.clear()
     mlat_solve_history.clear()
     mlat_solve_history_known.clear()
+    solver_resolve_skips_recent.clear()
     accuracy_samples.clear()
     mlat_samples.clear()
     for q in (frame_queue, solver_queue):
@@ -832,7 +905,7 @@ def _reset_for_tests() -> None:
     simulation_config = dict(_SIMULATION_CONFIG_DEFAULTS)
 
     with counters_lock:
-        frames_dropped = frames_processed = 0
+        frames_dropped = frames_processed = node_frames_rate_limited = 0
         solver_successes = solver_failures = n2_unconfirmed = 0
         adsb_seed_frames_autotagged = adsb_capture_ts_fallback = 0
         known_claims_made = known_claim_contentions = known_claims_bound = 0
@@ -841,7 +914,7 @@ def _reset_for_tests() -> None:
         coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0
         coverage_rebuild_backlog = 0
         solver_stale_drops = 0
-        solver_resolve_skips = 0
+        solver_resolve_skips = solver_resolve_skips_dark = 0
         mn_superseded = mn_superseded_blocked = 0
         solver_trimmed = 0
         solver_consensus_selected = solver_consensus_filtered = 0

backend/routes/analytics.py

@@ -125,26 +125,42 @@ async def association_status():
     return {
         "registered_nodes": len(_a.node_geometries),
         "overlap_zones": len(_a.overlap_zones),
+        # Node pairs that got no grid because the two nodes are in different
+        # worlds (node_world above).  Read next to overlap_zones: on a fleet
+        # of 50 synthetic nodes over the same city as 8 receivers it is the
+        # 400 sim/real pairs whose grids could only ever have paired a
+        # simulated echo with a real one.  Counted per pair considered, so it
+        # keeps rising as nodes re-register — zero means the fleet is single-
+        # world (or untagged), not that the gate is off.
+        "assoc_world_skipped_pairs": getattr(_a, "assoc_world_skipped_pairs", 0),
         # Confirmed single-node tracks each node last submitted; these are what
         # pairings are drawn from.
         "pending_tracks": {nid: len(tracks) for nid, tracks in list(_a._pending_tracks.items())},
         # Track-pairing outcomes since boot.  gated is everything past the
         # coarse delay grid; unfitted counts the pairings handed to the solver
         # worker (which runs the fit and the n=2 gate); deferred counts rounds
-        # a budget cut short.  Those three are the live production surface.
+        # a budget cut short; superseded counts pairings dropped because a
+        # better-ranked one claiming the same track implied a velocity theirs
+        # contradicts; cluster_splits counts position clusters that held two
+        # tracks of one node and were emitted as one solver input each.  All
+        # five are the live production surface.
         "track_pairs": {
             "gated": getattr(_a, "track_pairs_gated", 0),
             "unfitted": getattr(_a, "track_pairs_unfitted", 0),
             "deferred": getattr(_a, "track_pairs_deferred", 0),
+            "superseded": getattr(_a, "track_pairs_superseded", 0),
+            "cluster_splits": getattr(_a, "cluster_splits", 0),
         },
         # Inline-fit counters — permanently zero in production BY DESIGN
         # (state.py builds the associator with cv_fit=None; only the offline
-        # bench's inline mode exercises stage-2 selection).  Split out so
+        # bench's inline mode exercises the chi2 threshold).  Split out so
         # nobody reads a structural zero as "no rejections happening".
+        # superseded used to live here too, and no longer can: the deferred
+        # path now has an exclusivity stage of its own, so the counter moves
+        # in production.
         "track_pairs_inline_only": {
             "accepted": getattr(_a, "track_pairs_accepted", 0),
             "rejected": getattr(_a, "track_pairs_rejected", 0),
-            "superseded": getattr(_a, "track_pairs_superseded", 0),
         },
         # Top-down claiming (ASSOC_CLAIM_MODE) since boot.  rounds/matched/
         # conflicts/anchored_inputs are all live in shadow too — _claim_round