diff --git a/backend/.env.example b/backend/.env.example
index 20a107b4..2f645c62 100644
--- a/backend/.env.example
+++ b/backend/.env.example
@@ -121,7 +121,8 @@ MENDER_PAT=
# SOLVER_RESOLVE_INTERVAL_S is how long an aircraft is not re-solved for.
# Association is rate-limited per node, so every node that sees an aircraft
# emits its own candidate for it and most arrivals are duplicates; this is the
-# window that suppresses them. Do not exceed the 60 s multinode_tracks expiry —
+# window that suppresses them. Do not exceed the multinode_tracks expiry
+# (MN_DARK_EXPIRY_S below for dark tracks, 60 s for ADS-B-assisted ones) —
# tracks would lapse between solves. 0 turns the suppression off. See
# services/tasks/solver.py and docs/runbook.md's solver_latency_high entry.
#
@@ -129,9 +130,60 @@ MENDER_PAT=
# from its association guess before it is rejected; the default is 6 km = 2 x
# the 3 km association grid step, since a dark guess is a quantised grid point
# rather than an ADS-B fix. ADS-B-anchored solves keep the fixed 2 km cap.
+#
+# MN_DR_CAP_S is how far a multinode entry is dead-reckoned past its last
+# solve before it holds position, and MN_DARK_EXPIRY_S is when a DARK
+# (mn-dark-*) entry is dropped from the feed entirely. Both are position-error
+# budgets: measured against ground truth, dark entries run 1.5 km median error
+# at 8-15 s of solve age but 2.0 km at 15-30 s and 4.0 km at 30-60 s. ADS-B-
+# assisted (mn-adsb-*) entries keep the 60 s expiry — a transponder hex anchors
+# them. Raise both to trade position accuracy for track continuity.
+#
+# SOLVER_ALT_MODE is how an n>=3 solve gets its altitude. sweep (default)
+# solves once per fixed altitude layer and keeps the lowest rms_delay — six
+# process-pool round trips per candidate, and an altitude quantised to a
+# ladder whose 2 km spacing puts up to 1 km of error straight into the
+# residual the reject gate reads. free makes one pool call to the geolocator's
+# multi-start helper, which solves altitude as a sixth unknown. Both modes
+# stamp altitude_mode on the solve-history record, so /api/test/mlat-history
+# can compare them across a deploy of each. See docs/solverflow.md.
+#
+# SOLVER_FREE_ALT_STARTS is how many start altitudes free mode gives that
+# helper, clamped to the number of layers. 1 (the default) starts at the layer
+# nearest the association guess, or at the guess altitude itself when it comes
+# from ADS-B; more is a window around it, at one LM run each. Measured on test
+# over 1019 free solves, three starts changed rms_delay by more than 0.1 us in
+# 13 of them, so the extra runs are off by default and worth turning on only
+# where the geometry sends a single start to the wrong side of an ellipse.
# SOLVER_WORKERS=2
# SOLVER_RESOLVE_INTERVAL_S=12
# SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0
+# MN_DR_CAP_S=15.0
+# MN_DARK_EXPIRY_S=30.0
+# SOLVER_ALT_MODE=sweep
+# SOLVER_FREE_ALT_STARTS=1
+
+# Dark track following (services/dark_follow.py, docs/solverflow.md 3d). The
+# known lane's top-down claim applied to aircraft with no transponder: an
+# established mn-dark-* track's Kalman state predicts delay/Doppler at each
+# node, matching detections are claimed against it, and the resulting solve
+# carries the followed key as its anchor so it lands back on the SAME track.
+#
+# DARK_FOLLOW_MODE: off | shadow | binding. shadow (the default) claims,
+# solves and records but leaves the detections in the dark pool and publishes
+# nothing; binding strips them and publishes through the normal dark gates.
+#
+# The rest are the eligibility and guard thresholds. A key is followable while
+# its last solve is under MAX_AGE_S old with at least MIN_SOLVES solves and a
+# filter velocity sigma under MAX_VEL_SIGMA_MS; it is dropped for COOLDOWN_S
+# after two consecutive rejected follow-solves, which is what stops a ghost
+# locking itself onto the map. INTERVAL_S rate-limits follow-solves per key.
+# DARK_FOLLOW_MODE=shadow
+# DARK_FOLLOW_MAX_AGE_S=20
+# DARK_FOLLOW_MIN_SOLVES=3
+# DARK_FOLLOW_MAX_VEL_SIGMA_MS=60
+# DARK_FOLLOW_INTERVAL_S=2.0
+# DARK_FOLLOW_COOLDOWN_S=30
# Detection mirror. Production only. Every accepted v1 detection frame is
# forwarded to another environment's /api/radar/detections/bulk, batched once a
diff --git a/backend/config/constants.py b/backend/config/constants.py
index ea788abf..3f136df3 100644
--- a/backend/config/constants.py
+++ b/backend/config/constants.py
@@ -84,11 +84,25 @@ def as_num(v) -> float:
N2_CONFIRM_MIN_EPOCHS = 4 # Floor on samples; span is the real gate
N2_TRACK_HISTORY_MAX = 20 # Per-node track samples fed to the fit
+# How old a track's newest REAL detection may be before the track stops being
+# offered to association (see services/frame_processor.confirmed_track_views).
+# A COASTING track is kept alive for N_DELETE=10 frames after its last
+# association, and at the fleet's 0.74-1 Hz per-node cadence that is up to ~13 s
+# of dead reckoning. confirmed_track_views hands association the track's last
+# real sample, and association hands the solver that sample as if it were
+# current — so an aircraft that has flown out of a node's beam keeps
+# contributing a seconds-old delay to n>=3 solves. Measured on the test
+# droplet: 230 out of-cone nodes survived into published dark solves in 20 min,
+# median 4 deg outside the beam edge (p90 22 deg) and 1.8 km beyond max range,
+# and they are the nodes the rms trim then throws away. 0 disables the filter.
+TRACK_MAX_STALE_S = float(os.getenv("TRACK_MAX_STALE_S", "3.0"))
+
# A 2-node track needs this many solves before it renders a plane; 1
# disables the gate. One-shot n=2 solves were the dominant ghost source.
MN_N2_MIN_SOLVES = int(os.getenv("MN_N2_MIN_SOLVES", "2"))
# One-shot display lifetime for n>=3 solves, seconds. A track confirmed by
-# a second solve gets the normal 60 s entry expiry / 30 s DR cap.
+# a second solve gets the normal MN_DARK_EXPIRY_S/60 s entry expiry and the
+# MN_DR_CAP_S dead-reckoning cap below.
#
# The window has to outlive the wait for the confirmation it is waiting for,
# or it is not a preview window — it is a guaranteed disappearance. A second
@@ -97,11 +111,36 @@ def as_num(v) -> float:
# cadence is a 9 s median and a 25 s p90. At 5 s the great majority of
# genuine n>=3 one-shots blinked out before the round that would have
# confirmed them ever ran, which reads on the map as flicker, not as caution.
-# 15 s covers the median and most of the p90 while still being well short of
-# the 60 s entry expiry, so an unconfirmed one-shot is still withdrawn long
-# before a confirmed track would be.
+# 15 s covers the median and most of the p90 while still being short of the
+# entry expiry (MN_DARK_EXPIRY_S 30 s dark, 60 s assisted), so an unconfirmed
+# one-shot is still withdrawn before a confirmed track would be.
MN_ONESHOT_TTL_S = float(os.getenv("MN_ONESHOT_TTL_S", "15.0"))
+# How far a multinode entry may be dead-reckoned past its last solve, seconds.
+# Beyond this it holds its last dead-reckoned point until the entry expires.
+#
+# The cap is a position-error budget, not a cadence allowance. Measured on
+# the test droplet over 20 minutes (dark multinode feed entries vs ground
+# truth): median error 1.05 km under 3 s of solve age, 1.21 km at 3–8 s,
+# 1.50 km at 8–15 s, then 2.02 km at 15–30 s (7% of entries more than 5 km
+# off). The knee is at 15 s, which is where the KF's learned velocity error
+# starts to dominate the solve error it is extrapolating. The old 30 s cap
+# was set when a dark aircraft was re-solved every 12 s and the extra window
+# bought coverage; dark solves now land every 1–3 s, so a 15 s gap is a lost
+# track rather than a cadence gap, and extrapolating it only invents motion.
+MN_DR_CAP_S = float(os.getenv("MN_DR_CAP_S", "15.0"))
+
+# Entry expiry for DARK multinode tracks (mn-dark-*), seconds. ADS-B-assisted
+# entries (mn-adsb-*) keep the 60 s expiry: they are anchored to a transponder
+# fix, so a gap there is the ADS-B feed breathing rather than a lost target.
+#
+# Same 20-minute capture: the 30–60 s age band was 12% of all displayed dark
+# entries, with a 3.99 km median error and 32% more than 5 km off — an icon
+# that reads as a live target while sitting kilometres from any aircraft. At
+# the current 1–3 s dark solve cadence an entry that has not re-solved in 30 s
+# is a lost track, and withdrawing it is more honest than holding it.
+MN_DARK_EXPIRY_S = float(os.getenv("MN_DARK_EXPIRY_S", "30.0"))
+
# Quality gate for adopting the constant-velocity fit's velocity into a
# published solve, in place of the single-epoch Doppler solution (see
# solver.py's _resolve_cv_fit / velocity adoption in _process_solver_item).
diff --git a/backend/core/state.py b/backend/core/state.py
index 634aa212..c5aeb35c 100644
--- a/backend/core/state.py
+++ b/backend/core/state.py
@@ -74,6 +74,69 @@
if KNOWN_LANE_MODE not in ("off", "shadow", "binding"):
KNOWN_LANE_MODE = "shadow"
+# Dark track following (see services/dark_follow.py) — the same top-down claim
+# as the known lane, against established mn-dark-* tracks instead of ADS-B
+# identities. Same three-way vocabulary, and the same fallback discipline: an
+# unrecognised value degrades to the inert-for-the-feed mode. Default
+# "shadow", unlike KNOWN_LANE_MODE's "binding", because this lane decides which
+# aircraft the map believes in without a transponder to check itself against —
+# it earns its soak before it binds. Claiming rides the known lane's per-frame
+# stage, so this is inert whenever KNOWN_LANE_MODE is off (an ADS-B aircraft
+# must never lose a detection to a dark pseudo-state, and that precedence needs
+# the ADS-B claims to have been made first).
+DARK_FOLLOW_MODE = os.getenv("DARK_FOLLOW_MODE", "shadow").lower()
+if DARK_FOLLOW_MODE not in ("off", "shadow", "binding"):
+ DARK_FOLLOW_MODE = "shadow"
+# Measurement epoch alignment (see services/tasks/solver.align_measurement_epochs).
+# on/off rather than the off/shadow/active triple its neighbours use: there is
+# nothing to shadow — the correction is a closed-form dead-reckoning of each
+# delay along its own measured Doppler, so a dry run would produce the same
+# number the acting run applies and observe nothing extra. Default "on",
+# because leaving it off is the bug: nodes sample at independent phases and the
+# solver treats their measurements as simultaneous, so a 250 m/s target charges
+# up to ~1 us of delay error per second of skew (measured ~0.3 us rms at 2 s
+# skew on the fleet) straight to the 3 us rms gate. The flag exists so the
+# alignment can be turned off live without a rollback if it ever misbehaves.
+SOLVER_EPOCH_ALIGN = os.getenv("SOLVER_EPOCH_ALIGN", "on").strip().lower() != "off"
+# How the n>=3 solve gets its altitude (see services/tasks/solver.py's
+# _solve_best_altitude). sweep/free, read here rather than in that module so
+# it sits with its sibling mode flags and a test can monkeypatch it without
+# reimporting the solver.
+# sweep (default) — solve once per fixed altitude layer, keep the lowest
+# rms_delay. Six pool round trips per candidate, and an altitude
+# quantised to the ladder: layers are 2 km apart, so the pin is
+# systematically up to 1 km wrong and that error lands in the
+# residual the reject gate reads.
+# free — one pool call to retina_geolocator's multi-start helper, which
+# solves altitude as a sixth unknown, started from
+# SOLVER_FREE_ALT_STARTS of those layers.
+# Not off/shadow/active: there is no shadow here, because the two modes
+# produce the same shape of result and the history record carries
+# altitude_mode either way — running both would double the solver's cost to
+# learn what one deploy of each already says. An unrecognised value falls
+# back to "sweep", the same degrade-to-inert rule the sibling flags use.
+SOLVER_ALT_MODE = os.getenv("SOLVER_ALT_MODE", "sweep").lower()
+if SOLVER_ALT_MODE not in ("sweep", "free"):
+ SOLVER_ALT_MODE = "sweep"
+
+# How many start altitudes the free mode hands that helper. Read here beside
+# the mode it qualifies; _free_alt_starts in services/tasks/solver.py clamps it
+# into [1, len(layers)] against the ladder that module owns. 1 starts at the
+# layer nearest the association guess — where the sweep would have pinned;
+# more is a window around it.
+#
+# The default is 1 because three starts did not pay for themselves: over 1019
+# free-mode solves on test, the three starts' rms_delay differed by more than
+# 0.1 us in 13 of them, and the nearest-layer start was more than 0.5 us worse
+# than the best start in 2. That is ~0.2% of solves helped for 3x the solver
+# CPU, and the pool — not the altitude ladder — is what this deployment is
+# short of (~1.7 attempts/s against a 2.0 s average latency on two workers).
+# The knob stays because the reason for several starts is the LM's locality,
+# which is a property of the geometry rather than of this fleet: nodes lying
+# nearer a bistatic ellipse than these can send a single start to the wrong
+# side of it, and finding that out should not need a code change.
+SOLVER_FREE_ALT_STARTS = max(1, int(os.getenv("SOLVER_FREE_ALT_STARTS", "1")))
+
node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE)
@@ -475,6 +538,15 @@ def _adsb_for_seeding() -> dict[str, dict]:
# Monotonic counter for dropped frames (useful for monitoring)
frames_dropped: int = 0
+# Frames the per-node rate limiter refused before they ever reached
+# frame_queue (tcp_handler's NODE_FRAME_MIN_INTERVAL_S gate). A different
+# event from frames_dropped, which is queue saturation: this one is the
+# pipeline deliberately sampling a node down to ~1 Hz, and a node streaming at
+# 22 fps therefore reports a large number here while dropping nothing. It was
+# uncounted, so "how much of a node's evidence does the tracker actually see"
+# had no answer at all — the frames_dropped that IS published
+# (/api/admin/metrics) says zero throughout.
+node_frames_rate_limited: int = 0
frames_processed: int = 0
solver_successes: int = 0
solver_failures: int = 0
@@ -508,6 +580,23 @@ def _adsb_for_seeding() -> dict[str, dict]:
# Claiming-stage exceptions absorbed by frame_processor's fail-open guard.
# Nonzero means the known lane is broken and silently contributing nothing.
known_claims_errors: int = 0
+# Dark track following (DARK_FOLLOW_MODE) — see services/dark_follow.py.
+# targets is a GAUGE (the size of the current pseudo-state list, assigned on
+# every rebuild), the other four are since-boot counters. The funnel reads
+# targets -> claims -> inputs -> published, with dropped the guard's own
+# tally: a lane with targets but no claims is predicting positions no node
+# agrees with, and one with rising dropped is the guard doing its job.
+dark_follow_targets: int = 0
+dark_follow_claims: int = 0
+dark_follow_inputs: int = 0
+dark_follow_published: int = 0
+dark_follow_dropped: int = 0
+# Bottom-up dark solves refused at keying because the follow lane owns the key
+# they landed on (solver.multinode_key_decision's "shadowed" verdict, binding
+# mode only). They passed every gate, so they are counted in solver_successes
+# too — this is the difference between a success and a publication on the dark
+# lane, and the matching rejects.by_reason entry is "shadowed_by_follow".
+dark_bottomup_shadowed: int = 0
# n=2 solves withheld from the map because their track pairing has not (yet)
# passed the constant-velocity fit. Counted separately from solver_failures:
# the solve succeeded, it simply has not earned publication, and a real target
@@ -526,6 +615,21 @@ def _adsb_for_seeding() -> dict[str, dict]:
# fleet's trigger rate — constraints are then converging slower than the
# coverage they follow, which no rebuild counter can show.
coverage_rebuild_backlog: int = 0
+
+# Confirmed tracks withheld from association because their newest REAL
+# detection was older than TRACK_MAX_STALE_S at the frame being processed —
+# see services/frame_processor.confirmed_track_views. These are aircraft that
+# have left a node's beam and whose track is dead-reckoning toward deletion;
+# their last real sample used to reach the solver as a current measurement.
+tracks_stale_skipped: int = 0
+
+# Solver inputs whose measurements could not be aligned to a common epoch
+# because at least one lacked t_s, doppler_hz, or a node config with fc_hz —
+# see services/tasks/solver.align_measurement_epochs. Counted only when
+# SOLVER_EPOCH_ALIGN is on; a nonzero value against solver_successes says how
+# much of the fleet is still emitting untimed measurements.
+solver_epoch_align_skipped: int = 0
+
solver_queue_drops: int = 0
# Queue items discarded unsolved because they aged past _SOLVER_MAX_QUEUE_AGE_S
# waiting for a worker. Was only a DEBUG log, which staging does not emit —
@@ -535,14 +639,36 @@ def _adsb_for_seeding() -> dict[str, dict]:
solver_stale_drops: int = 0
# Candidates dequeued and skipped because every single-node track they carry
-# was already solved within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes (see
-# solver.py's _claim_resolve_slot). Association is per-node and rate-limited
-# per node, so one aircraft arrives as one candidate per node that can see it;
-# this counts the copies that were never worth solving. High against
-# solver_successes is normal and is the mechanism working — it is
-# solver_stale_drops that means work was lost.
+# was already PUBLISHED within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes
+# (see solver.py's _resolve_slot_covered). Association is per-node and
+# rate-limited per node, so one aircraft arrives as one candidate per node
+# that can see it; this counts the copies that were never worth solving. High
+# against solver_successes is normal and is the mechanism working — it is
+# solver_stale_drops that means work was lost. Read it against
+# solver_successes, not against attempts: while the claim was taken on
+# ADMISSION rather than on publication, a rejected candidate blacked out every
+# later one sharing a track id and this counter ran at ~2.4x attempts.
solver_resolve_skips: int = 0
+# The dark-lane share of the counter above, split out because the two lanes
+# read completely differently: an ADS-B-anchored duplicate that is skipped
+# costs nothing (the transponder keeps the track alive anyway), while a
+# skipped dark candidate may be the only chance that aircraft had of reaching
+# the map this window. Lane is decided by solver._is_dark_solver_input, the
+# same predicate routes.test._record_lane falls back to for a record that
+# never got a key — and a skip never gets one.
+solver_resolve_skips_dark: int = 0
+
+# The last few hundred resolve-slot skips, with the claims that blocked them.
+# Deliberately NOT the solve-history deque: a skip is not a solve outcome, and
+# writing one record per skip into mlat_solve_history would evict the real
+# records at roughly twice their rate (live: ~1 537 skips per 646 dark
+# attempts per 30 min). Small and separate, read by
+# /api/test/solver-stats' resolve_skips block and dumped by
+# /api/test/mlat-history?kind=resolve_skips. ~250 B/entry.
+SOLVER_RESOLVE_SKIPS_RECENT_MAX = 500
+solver_resolve_skips_recent: deque = deque(maxlen=SOLVER_RESOLVE_SKIPS_RECENT_MAX)
+
# Multinode entries removed because a later solve shared a source single-node
# track with them AND the spatial/identical-inputs guard in solver.py's
# _supersession_match agreed they are the same aircraft — the age-scaled
@@ -746,12 +872,16 @@ def _reset_for_tests() -> None:
global latest_mlat_accuracy_bytes, latest_mlat_verification_bytes
global latest_storage_bytes, simulation_config
global frames_dropped, frames_processed, solver_successes, solver_failures
+ global node_frames_rate_limited
global adsb_seed_frames_autotagged, adsb_capture_ts_fallback
global known_claims_made, known_claim_contentions, known_claims_bound
global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects
+ global dark_follow_targets, dark_follow_claims, dark_follow_inputs
+ global dark_follow_published, dark_follow_dropped, dark_bottomup_shadowed
global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes
- global coverage_rebuild_backlog
+ global coverage_rebuild_backlog, tracks_stale_skipped, solver_epoch_align_skipped
global solver_queue_drops, solver_stale_drops, solver_resolve_skips
+ global solver_resolve_skips_dark
global mn_superseded, mn_superseded_blocked, solver_trimmed
global solver_consensus_selected, solver_consensus_filtered
global solver_consensus_fallback, solver_consensus_shadow
@@ -801,6 +931,7 @@ def _reset_for_tests() -> None:
track_archive_buffer.clear()
mlat_solve_history.clear()
mlat_solve_history_known.clear()
+ solver_resolve_skips_recent.clear()
accuracy_samples.clear()
mlat_samples.clear()
for q in (frame_queue, solver_queue):
@@ -832,16 +963,20 @@ def _reset_for_tests() -> None:
simulation_config = dict(_SIMULATION_CONFIG_DEFAULTS)
with counters_lock:
- frames_dropped = frames_processed = 0
+ frames_dropped = frames_processed = node_frames_rate_limited = 0
solver_successes = solver_failures = n2_unconfirmed = 0
adsb_seed_frames_autotagged = adsb_capture_ts_fallback = 0
known_claims_made = known_claim_contentions = known_claims_bound = 0
known_claims_errors = known_claims_visibility_rejects = 0
known_claims_world_rejects = 0
+ dark_follow_targets = dark_follow_claims = dark_follow_inputs = 0
+ dark_follow_published = dark_follow_dropped = 0
+ dark_bottomup_shadowed = 0
coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0
coverage_rebuild_backlog = 0
+ tracks_stale_skipped = solver_epoch_align_skipped = 0
solver_stale_drops = 0
- solver_resolve_skips = 0
+ solver_resolve_skips = solver_resolve_skips_dark = 0
mn_superseded = mn_superseded_blocked = 0
solver_trimmed = 0
solver_consensus_selected = solver_consensus_filtered = 0
diff --git a/backend/routes/admin.py b/backend/routes/admin.py
index 2d0eca6d..4d11fb0b 100644
--- a/backend/routes/admin.py
+++ b/backend/routes/admin.py
@@ -590,6 +590,8 @@ async def system_metrics(_user=Depends(require_admin)):
"solver_queue_drops": state.solver_queue_drops,
"solver_stale_drops": state.solver_stale_drops,
"solver_resolve_skips": state.solver_resolve_skips,
+ "tracks_stale_skipped": state.tracks_stale_skipped,
+ "solver_epoch_align_skipped": state.solver_epoch_align_skipped,
"mn_superseded": state.mn_superseded,
"solver_trimmed": state.solver_trimmed,
"solver_last_latency_s": round(state.solver_last_latency_s, 3),
diff --git a/backend/routes/analytics.py b/backend/routes/analytics.py
index 07d83a9c..93af0243 100644
--- a/backend/routes/analytics.py
+++ b/backend/routes/analytics.py
@@ -125,26 +125,42 @@ async def association_status():
return {
"registered_nodes": len(_a.node_geometries),
"overlap_zones": len(_a.overlap_zones),
+ # Node pairs that got no grid because the two nodes are in different
+ # worlds (node_world above). Read next to overlap_zones: on a fleet
+ # of 50 synthetic nodes over the same city as 8 receivers it is the
+ # 400 sim/real pairs whose grids could only ever have paired a
+ # simulated echo with a real one. Counted per pair considered, so it
+ # keeps rising as nodes re-register — zero means the fleet is single-
+ # world (or untagged), not that the gate is off.
+ "assoc_world_skipped_pairs": getattr(_a, "assoc_world_skipped_pairs", 0),
# Confirmed single-node tracks each node last submitted; these are what
# pairings are drawn from.
"pending_tracks": {nid: len(tracks) for nid, tracks in list(_a._pending_tracks.items())},
# Track-pairing outcomes since boot. gated is everything past the
# coarse delay grid; unfitted counts the pairings handed to the solver
# worker (which runs the fit and the n=2 gate); deferred counts rounds
- # a budget cut short. Those three are the live production surface.
+ # a budget cut short; superseded counts pairings dropped because a
+ # better-ranked one claiming the same track implied a velocity theirs
+ # contradicts; cluster_splits counts position clusters that held two
+ # tracks of one node and were emitted as one solver input each. All
+ # five are the live production surface.
"track_pairs": {
"gated": getattr(_a, "track_pairs_gated", 0),
"unfitted": getattr(_a, "track_pairs_unfitted", 0),
"deferred": getattr(_a, "track_pairs_deferred", 0),
+ "superseded": getattr(_a, "track_pairs_superseded", 0),
+ "cluster_splits": getattr(_a, "cluster_splits", 0),
},
# Inline-fit counters — permanently zero in production BY DESIGN
# (state.py builds the associator with cv_fit=None; only the offline
- # bench's inline mode exercises stage-2 selection). Split out so
+ # bench's inline mode exercises the chi2 threshold). Split out so
# nobody reads a structural zero as "no rejections happening".
+ # superseded used to live here too, and no longer can: the deferred
+ # path now has an exclusivity stage of its own, so the counter moves
+ # in production.
"track_pairs_inline_only": {
"accepted": getattr(_a, "track_pairs_accepted", 0),
"rejected": getattr(_a, "track_pairs_rejected", 0),
- "superseded": getattr(_a, "track_pairs_superseded", 0),
},
# Top-down claiming (ASSOC_CLAIM_MODE) since boot. rounds/matched/
# conflicts/anchored_inputs are all live in shadow too — _claim_round
diff --git a/backend/routes/test.py b/backend/routes/test.py
index e4018b36..b01ccd4f 100644
--- a/backend/routes/test.py
+++ b/backend/routes/test.py
@@ -210,10 +210,17 @@ def _build_dashboard_data() -> bytes:
# not the queue size.
"stale_drops": state.solver_stale_drops,
# Duplicate candidates for an aircraft already solved this
- # window (see solver.py's _claim_resolve_slot). Read it
+ # window (see solver.py's _resolve_slot_covered). Read it
# against stale_drops: skips are work correctly not done,
# stale drops are work lost.
"resolve_skips": state.solver_resolve_skips,
+ # Confirmed tracks withheld from association because their
+ # newest real detection had aged past TRACK_MAX_STALE_S, and
+ # solver inputs the epoch alignment could not correct because a
+ # measurement carried no sample time (see frame_processor's
+ # confirmed_track_views and solver's align_measurement_epochs).
+ "tracks_stale_skipped": state.tracks_stale_skipped,
+ "epoch_align_skipped": state.solver_epoch_align_skipped,
# Multinode entries replaced because a later solve consumed
# the same source tracks under a new key (fragmented re-solve).
"mn_superseded": state.mn_superseded,
@@ -734,7 +741,8 @@ def _window_effective_minutes(records: list[dict], minutes: float) -> float:
def _record_lane(rec: dict) -> str:
- """Which solver lane produced one history record: "known", "dark" or "adsb".
+ """Which solver lane produced one history record: "known", "dark_follow",
+ "dark" or "adsb".
``known_lane`` is stamped by known_lane._attempt via ``extra``. For the
regular pipeline the authority is the minted track key (mn-dark-* vs
@@ -742,9 +750,16 @@ def _record_lane(rec: dict) -> str:
key exists, so it falls back to the same predicate that key decision uses
— whether the solver input carried a transponder-shaped identity, which is
also what picked its displacement cap.
+
+ ``lane`` is checked before the key, because a dark-follow record
+ (services/dark_follow.py) is keyed mn-dark-* by design — it is the same
+ aircraft the dark lane tracks, reached top-down — and would otherwise land
+ in the bottom-up funnel whose attempts and rejects it is not one of.
"""
if rec.get("known_lane"):
return "known"
+ if rec.get("lane") == "dark_follow":
+ return "dark_follow"
key = rec.get("solve_key")
if key:
return "dark" if key.startswith("mn-dark-") else "adsb"
@@ -752,11 +767,40 @@ def _record_lane(rec: dict) -> str:
return "adsb" if hexn and is_transponder_hex(hexn) else "dark"
+_LANES = ("dark", "known", "adsb", "dark_follow")
+
+
+def _cap_per_lane(records: list[dict], limit: int) -> list[dict]:
+ """Keep the ``limit`` newest records OF EACH LANE, newest first.
+
+ ``records`` must already be newest-first. A single flat ``[:limit]``
+ made the cap a race between lanes rather than a retention rule, exactly
+ as the shared deque did before PR #289 split it: the known lane writes
+ ~16x the dark lane's volume, so a flat 1 000-record answer to a 30 min
+ request held only the newest ~6 min of dark records and the rest of the
+ window read as a quiet period. Capping per lane means known-lane volume
+ can never evict a dark record from a response.
+ """
+ kept: list[dict] = []
+ counts: dict[str, int] = {}
+ for r in records:
+ lane = _record_lane(r)
+ n = counts.get(lane, 0)
+ if n >= limit:
+ continue
+ counts[lane] = n + 1
+ kept.append(r)
+ return kept
+
+
@router.get("/api/test/mlat-history")
async def mlat_history(
hex: str | None = None,
all: int = 0,
minutes: float = 30.0,
+ lane: str = "all",
+ limit: int = 1000,
+ kind: str = "solves",
):
"""Per-solve MLAT history from the last ~30 minutes.
@@ -770,22 +814,73 @@ async def mlat_history(
merged here, so both lanes answer either query exactly as they did when
they shared a deque.
+ ``?lane=dark|known|adsb`` narrows the answer to one lane (default
+ ``all``, classified by ``_record_lane``); ``?limit=`` caps the record
+ list (default 1 000, max 5 000) and is applied PER LANE, so a known-lane
+ burst can never push dark records out of an ``all`` response — see
+ _cap_per_lane.
+
+ ``?kind=resolve_skips`` dumps a different store entirely: the solver's
+ recent resolve-slot refusals (state.solver_resolve_skips_recent), each
+ with the claims that blocked it. Those are not solve outcomes and
+ deliberately do not live in the solve-history deques.
+
``window_effective_minutes`` is how much of the requested window the
stores actually hold — below ``window_minutes`` the answer is truncated.
"""
+ if lane not in ("all", *_LANES):
+ return Response(
+ content=orjson.dumps({"error": f"lane must be one of all,{','.join(_LANES)}"}),
+ media_type="application/json",
+ status_code=400,
+ )
+ if kind not in ("solves", "resolve_skips"):
+ return Response(
+ content=orjson.dumps({"error": "kind must be solves or resolve_skips"}),
+ media_type="application/json",
+ status_code=400,
+ )
minutes = max(0.0, min(minutes, 35.0))
+ limit = max(1, min(int(limit), 5000))
cutoff_ms = int((time.time() - minutes * 60.0) * 1000)
+
+ if kind == "resolve_skips":
+ skips = [
+ s
+ for s in list(state.solver_resolve_skips_recent)
+ if s["ts_ms"] >= cutoff_ms and (lane == "all" or s["lane"] == lane)
+ ]
+ skips.reverse() # newest first
+ payload = {
+ "kind": "resolve_skips",
+ "window_minutes": minutes,
+ "lane": lane,
+ "lane_counts": {ln: sum(1 for s in skips if s["lane"] == ln) for ln in _LANES},
+ "n_records": len(skips),
+ "records": skips[:limit],
+ }
+ return Response(content=orjson.dumps(payload), media_type="application/json")
+
merged = _merged_solve_history()
effective_minutes = _window_effective_minutes(merged, minutes)
records = [r for r in merged if r["ts_ms"] >= cutoff_ms]
records.reverse() # newest first
+ if lane != "all":
+ records = [r for r in records if _record_lane(r) == lane]
+ lane_counts = dict.fromkeys(_LANES, 0)
+ for r in records:
+ lane_counts[_record_lane(r)] += 1
if all:
payload = {
"window_minutes": minutes,
"window_effective_minutes": effective_minutes,
+ "lane": lane,
+ # Pre-cap, so a truncated `records` can be read against what the
+ # window actually held.
+ "lane_counts": lane_counts,
"n_records": len(records),
- "records": records[:1000],
+ "records": _cap_per_lane(records, limit),
}
return Response(content=orjson.dumps(payload), media_type="application/json")
@@ -813,6 +908,8 @@ async def mlat_history(
"hex": norm,
"window_minutes": minutes,
"window_effective_minutes": effective_minutes,
+ "lane": lane,
+ "lane_counts": lane_counts,
"n_solves": len(solves),
"solves": solves[:500],
"rejects_nearby": {
@@ -882,7 +979,7 @@ def _solver_window_stats(minutes: float) -> dict:
effective_minutes = _window_effective_minutes(merged, minutes)
all_records = [r for r in merged if r["ts_ms"] >= cutoff_ms]
- lane_split = {"dark": 0, "adsb": 0, "known": 0}
+ lane_split = {"dark": 0, "adsb": 0, "known": 0, "dark_follow": 0}
records: list[dict] = []
known_records: list[dict] = []
for r in all_records:
@@ -914,6 +1011,46 @@ def _solver_window_stats(minutes: float) -> dict:
reason = outcome[len("rejected_") :] if outcome.startswith("rejected_") else outcome
by_reason[reason] = by_reason.get(reason, 0) + 1
+ # ── cluster contamination (dark, windowed) ──────────────────────────────
+ # Of the dark records this window that matched ground truth, how many
+ # carried a node that could not see the aircraft they were matched to —
+ # the live version of the offline number Phase 2 exists to move (~60 %).
+ # Records without the stamp are records nothing could be asked about (no
+ # GT match, or no registered geometry for any contributing node) and stay
+ # out of the denominator rather than counting as clean; see
+ # solver._stamp_foreign_nodes.
+ judged = [r for r in records if r.get("foreign_node_ids") is not None]
+ contaminated = [r for r in judged if r.get("contaminated")]
+ n_judged = len(judged)
+ contamination = {
+ "records_with_gt": n_judged,
+ "contaminated": len(contaminated),
+ "pct": round(100.0 * len(contaminated) / n_judged, 1) if n_judged else None,
+ "foreign_nodes_per_record": (
+ round(sum(len(r["foreign_node_ids"]) for r in judged) / n_judged, 2) if n_judged else None
+ ),
+ }
+
+ # ── resolve-slot skips (windowed, from the skip deque) ──────────────────
+ # The counter in "counters" below is since-boot; these are the skips that
+ # happened inside this window, so they can be read against the attempts in
+ # the same window. attempts_ratio is (all-lane skips / DARK attempts) —
+ # the shape the acceptance target for the claim-on-publish fix is quoted
+ # in (live baseline ~1 537 / 646 = 2.4), not a per-lane rate. The dark
+ # numerator is published beside it for anyone who wants one.
+ all_skips = list(state.solver_resolve_skips_recent)
+ skips = [s for s in all_skips if s["ts_ms"] >= cutoff_ms]
+ resolve_skips = {
+ "total": len(skips),
+ "dark": sum(1 for s in skips if s["lane"] == "dark"),
+ "attempts_ratio": round(len(skips) / attempts, 3) if attempts else None,
+ # The skip deque is 500 entries against a live rate of ~50/min, so a
+ # long window IS truncated here even when the solve-history stores
+ # cover it. Same honesty rule as window_effective_minutes above: read
+ # it before reading total as a window count.
+ "window_effective_minutes": _window_effective_minutes(all_skips, minutes),
+ }
+
pos_errors.sort()
n_err = len(pos_errors)
median_err = pos_errors[n_err // 2] if n_err else None
@@ -1066,6 +1203,9 @@ def _solver_window_stats(minutes: float) -> dict:
"published": {"total": n2 + n3plus, "n2": n2, "n3plus": n3plus},
"rejects": {"total": reject_total, "by_reason": by_reason},
"position_error_km": {"median": median_err, "p90": p90_err, "n": n_err},
+ # Both windowed and both DARK-lane, like the funnel above them.
+ "contamination": contamination,
+ "resolve_skips": resolve_skips,
"ghosts": {
# Scoped to dark tracks: precision_pct's denominator is
# dark_tracks, and it is None (not 100.0) when there are none.
@@ -1192,9 +1332,35 @@ def _solver_window_stats(minutes: float) -> dict:
"solver_trimmed": state.solver_trimmed,
"stale_drops": state.solver_stale_drops,
"resolve_skips": state.solver_resolve_skips,
+ "tracks_stale_skipped": state.tracks_stale_skipped,
+ "epoch_align_skipped": state.solver_epoch_align_skipped,
+ # Dark share of the line above. The windowed version, with the
+ # blocking claims, is the "resolve_skips" block further up.
+ "resolve_skips_dark": state.solver_resolve_skips_dark,
"queue_drops": state.solver_queue_drops,
+ # Frames the per-node rate limiter refused before the tracker ever
+ # saw them (tcp_handler's NODE_FRAME_MIN_INTERVAL_S). Not the
+ # same event as /api/admin/metrics' frames_dropped, which is
+ # frame_queue saturation and normally reads zero.
+ "node_frames_rate_limited": state.node_frames_rate_limited,
"worker_errors": state.solver_worker_errors,
"vel_untrusted_published": state.solver_vel_untrusted_published,
+ # Dark track following (services/dark_follow.py), since boot except
+ # targets, which is a live gauge of the current pseudo-state list.
+ # The funnel is targets -> claims -> inputs -> published; dropped is
+ # the ghost guard's tally of keys it stopped following.
+ "dark_follow_targets": state.dark_follow_targets,
+ "dark_follow_claims": state.dark_follow_claims,
+ "dark_follow_inputs": state.dark_follow_inputs,
+ "dark_follow_published": state.dark_follow_published,
+ "dark_follow_dropped": state.dark_follow_dropped,
+ # The other side of the lane: bottom-up dark solves refused at
+ # keying because the follow lane owns the key they landed on. It
+ # belongs beside the funnel because it is the same trade — the
+ # lane keeps a key only if it also stops the bottom-up lane from
+ # corrupting it — and the matching per-solve records are the
+ # "shadowed_by_follow" entries in rejects.by_reason.
+ "dark_bottomup_shadowed": state.dark_bottomup_shadowed,
},
}
diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py
index e08999bb..1bbe8967 100644
--- a/backend/scripts/association_bench.py
+++ b/backend/scripts/association_bench.py
@@ -69,6 +69,7 @@
# superseded detection path that --mode detection measures as the baseline, so
# constructing it unconditionally leaves --mode track unaffected.
import retina_analytics.association as _assoc_module # noqa: E402
+from retina_analytics.association import predict_observation # noqa: E402
from retina_analytics.detection_association import DetectionAssociator # noqa: E402
from retina_analytics.manager import NodeAnalyticsManager # noqa: E402
from retina_geolocator.consensus import solve_consensus # noqa: E402
@@ -248,6 +249,133 @@ def _frame_to_detections(frame: dict) -> list[dict]:
return dets
+# ── Contamination scoring (truth side-channel) ───────────────────────────
+# How far a detection may sit from an aircraft's noiseless (delay, doppler)
+# and still be attributed to it. The simulator's own measurement noise is
+# gauss(0, 0.1-0.2 us) in delay and gauss(0, 2-4 Hz) in Doppler
+# (world.generate_detections_for_node), so these are ~5 sigma: wide enough
+# that a real echo is never mistaken for clutter, tight enough that clutter
+# — uniform over tens of us — almost never lands on an aircraft.
+_TRUTH_DELAY_GATE_US = 1.0
+_TRUTH_DOPPLER_GATE_HZ = 25.0
+# Sentinel for "this measurement matched two different aircraft equally
+# well". Neither foreign nor own — excluded from the numerator so an
+# ambiguity in the scorer is never reported as a contamination.
+_TRUTH_AMBIGUOUS = "?ambiguous"
+
+
+def _index_detection_truth(det_truth: dict, geo, node_id: str, frame: dict, aircraft: list) -> None:
+ """Record which aircraft produced each detection in one node's frame.
+
+ Truth side-channel, for the CONTAMINATION metric only: it is built from
+ the frame BEFORE _strip_adsb and never reaches association, so the blind
+ discipline is intact. It cannot be read off the frame's own ``adsb``
+ list either — the simulator appends None there for every aircraft with
+ has_adsb False, and dark aircraft are exactly the population this metric
+ exists to score. So each detection is instead matched back to the
+ aircraft whose noiseless observation it is nearest.
+
+ Keyed on the (delay, doppler) floats themselves because that is the only
+ handle the metric gets downstream: a solver input's measurement carries
+ its track's latest delay/doppler verbatim (history[-1] -> the detection
+ dict -> here), and the simulator rounds both to 2 dp, so the equality is
+ exact rather than approximate.
+ """
+ delays = frame.get("delay") or []
+ if not delays:
+ return
+ dopplers = frame.get("doppler") or []
+ preds = []
+ for ac in aircraft:
+ d_us, f_hz = predict_observation(
+ geo,
+ ac.lat,
+ ac.lon,
+ ac.alt_km,
+ ac.vel_east * 1000.0,
+ ac.vel_north * 1000.0,
+ ac.vel_up * 1000.0,
+ )
+ preds.append((d_us, f_hz, ac.object_id))
+ for d, f in zip(delays, dopplers):
+ best = best2 = None
+ for d_us, f_hz, oid in preds:
+ dd, df = abs(d - d_us), abs(f - f_hz)
+ if dd > _TRUTH_DELAY_GATE_US or df > _TRUTH_DOPPLER_GATE_HZ:
+ continue
+ # Normalised so the two axes are comparable at their own gates.
+ cost = (dd / _TRUTH_DELAY_GATE_US) ** 2 + (df / _TRUTH_DOPPLER_GATE_HZ) ** 2
+ if best is None or cost < best[0]:
+ best, best2 = (cost, oid), best
+ elif best2 is None or cost < best2[0]:
+ best2 = (cost, oid)
+ if best is None:
+ continue # clutter: left absent, which the scorer reads as foreign
+ oid = best[1]
+ if best2 is not None and best2[0] < 4.0 * best[0]:
+ oid = _TRUTH_AMBIGUOUS
+ key = (node_id, float(d), float(f))
+ prev = det_truth.get(key)
+ # The same (node, delay, doppler) recurring for a different aircraft
+ # later in the run would silently relabel an earlier measurement, so
+ # a collision demotes the key rather than overwriting it.
+ det_truth[key] = oid if (prev is None or prev == oid) else _TRUTH_AMBIGUOUS
+
+
+def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) -> int | None:
+ """Count the nodes in one solver input that are not looking at its aircraft.
+
+ The input's own aircraft is the plurality of its measurements' true
+ aircraft — the honest reading of "what is this candidate mostly about",
+ and the one that does not assume the (possibly contaminated) initial
+ guess is anywhere near a target. Ties are broken by the nearest ground
+ truth to the initial guess, which is the criterion the ghost/matched
+ split already uses.
+
+ A node is foreign when its measurement belongs to a different aircraft,
+ or to no aircraft at all (clutter that survived the tracker's M-of-N and
+ the delay grid). Ambiguous attributions are counted in neither.
+
+ Returns the foreign-node count, so the caller can score the same input
+ again at the publish point (see the PUBLISHED counters on Result: the
+ candidate-level rate has a denominator the association layer itself moves,
+ and a change that emits more, cleaner candidates reads as a regression on
+ it while being an improvement on what actually reaches the map). None
+ when nothing could be attributed at all.
+ """
+ oids = [
+ det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"])))
+ for m in s_in.get("measurements") or []
+ ]
+ if not oids:
+ return None
+ counts = Counter(o for o in oids if o is not None and o != _TRUTH_AMBIGUOUS)
+ if not counts:
+ return None
+ top_n = max(counts.values())
+ contenders = sorted(o for o, c in counts.items() if c == top_n)
+ if len(contenders) > 1:
+ guess = s_in.get("initial_guess") or {}
+ contenders.sort(
+ key=lambda o: min(
+ (
+ _haversine_km(guess.get("lat", 0.0), guess.get("lon", 0.0), a, b)
+ for a, b, oid, _ in truth
+ if oid == o
+ ),
+ default=float("inf"),
+ )
+ )
+ own = contenders[0]
+ foreign = sum(1 for o in oids if o != own and o != _TRUTH_AMBIGUOUS)
+ res.inputs_scored += 1
+ res.input_nodes += len(oids)
+ res.foreign_nodes += foreign
+ if foreign:
+ res.inputs_contaminated += 1
+ return foreign
+
+
def _strip_adsb(frame: dict) -> dict:
"""Return the frame as a real receiver would see it.
@@ -482,6 +610,9 @@ class Result:
gate_accepted: int = 0
gate_unfitted: int = 0
gate_superseded: int = 0
+ # Position clusters that held two different tracks of one node and were
+ # split into one solver input each, straight off the associator.
+ cluster_splits: int = 0
# Deferred mode only: what the *solver-side* n=2 gate did. In production the
# associator emits unscored pairings and this gate is the one that runs, so
# without these the shipped configuration's selection is invisible.
@@ -531,6 +662,30 @@ class Result:
keys_real: int = 0
keys_ghost: int = 0
+ # ── Candidate contamination (--mode track) ────────────────────────────
+ # Scored on every solver input association emits, BEFORE the solve and
+ # before every downstream gate: the question is what association handed
+ # the solver, not what survived it. A contaminated input is one whose
+ # measurements do not all belong to the same aircraft — the failure the
+ # cluster-merge rework targets, and the one the ghost rate cannot see
+ # (a two-aircraft merge usually still solves within MATCH_KM of one of
+ # them, so it counts as matched while carrying 4-5 km of position
+ # error). See _score_contamination.
+ inputs_scored: int = 0
+ inputs_contaminated: int = 0
+ foreign_nodes: int = 0
+ input_nodes: int = 0
+ # The same score restricted to inputs that cleared every gate and bound
+ # to a real aircraft — what actually reached the map. Reported next to
+ # the candidate rate because the two answer different questions: the
+ # candidate rate's denominator is the number of candidates association
+ # chooses to emit, so splitting one contaminated cluster into several
+ # clean ones plus the false pairing it was hiding *raises* it while
+ # lowering this one.
+ published_inputs: int = 0
+ published_contaminated: int = 0
+ published_foreign_nodes: int = 0
+
# Stone-Soup GOSPA/SIAP scalars for this one run (--ss-metrics), or None
# when it was off, stonesoup wasn't available, or the recorder had
# nothing to score (see stonesoup_metrics.MetricRecorder.compute). Not
@@ -634,6 +789,14 @@ def keys_per_object(self):
"distinct_keys",
"keys_real",
"keys_ghost",
+ "inputs_scored",
+ "inputs_contaminated",
+ "foreign_nodes",
+ "input_nodes",
+ "published_inputs",
+ "published_contaminated",
+ "published_foreign_nodes",
+ "cluster_splits",
)
_EXTEND_FIELDS = (
"errors_km",
@@ -685,6 +848,22 @@ def merge(self, other: Result, tag: str) -> None:
def ghost_pct(self):
return 100.0 * self.ghosts / self.total if self.total else 0.0
+ @property
+ def contaminated_inputs_pct(self):
+ return 100.0 * self.inputs_contaminated / self.inputs_scored if self.inputs_scored else 0.0
+
+ @property
+ def foreign_nodes_per_input(self):
+ return self.foreign_nodes / self.inputs_scored if self.inputs_scored else 0.0
+
+ @property
+ def published_contaminated_pct(self):
+ return 100.0 * self.published_contaminated / self.published_inputs if self.published_inputs else 0.0
+
+ @property
+ def published_foreign_per_solve(self):
+ return self.published_foreign_nodes / self.published_inputs if self.published_inputs else 0.0
+
def build_scene(
seed: int,
@@ -789,6 +968,7 @@ def run(
ss_metric_dt=5.0,
ss_hold_s=12.0,
smoother_legs=None,
+ cluster_opts=None,
) -> Result:
import random
@@ -861,12 +1041,17 @@ def fov_provider(node_id):
# emits unscored pairings and the solver worker fits and arbitrates. The
# two are different code paths, so they need separate baselines.
deferred = mode == "track" and cv_fit_mode == "deferred"
+ # The cluster-merge knobs are passed only when the caller overrode them,
+ # so a plain run measures whatever the library currently ships rather than
+ # freezing today's defaults into the bench.
+ _cluster_kwargs = {k: v for k, v in (cluster_opts or {}).items() if v is not None}
assoc = DetectionAssociator(
grid_step_km=3.0,
cv_fit=(fit_constant_velocity if (mode == "track" and not deferred) else None),
cv_chi2_max=chi2_max,
cv_min_span_s=min_span_s,
cv_exclusive=exclusive,
+ **_cluster_kwargs,
)
n2_gate = DeferredN2Gate(chi2_max, claim_ttl_s=claim_ttl_s, claim_policy=claim_policy) if deferred else None
# One tracker per node, driven by every frame — mirrors
@@ -915,6 +1100,10 @@ def fov_provider(node_id):
_BENCH_MN_MAX_AGE_MS = 60_000
res = Result()
+ # (node_id, delay_us, doppler_hz) -> the aircraft that produced that
+ # detection. Truth side-channel for the contamination metric only, built
+ # from the un-stripped frame below — see _index_detection_truth.
+ det_truth: dict = {}
_all_keys_seen: set = set()
_keys_real: set = set()
_keys_ghost: set = set()
@@ -960,6 +1149,10 @@ def _geo_key(nid):
for nid in due_nodes:
next_send[nid] += frame_interval
frame = world.generate_detections_for_node(nid, ts_ms)
+ if mode == "track":
+ # Before _strip_adsb, and never fed to association: the
+ # contamination metric's truth channel.
+ _index_detection_truth(det_truth, assoc.node_geometries[nid], nid, frame, world.aircraft)
if fov_analytics is not None:
# The truth channel, not the (possibly blind) association
# stream below -- a real node's ADS-B calibration reaches
@@ -978,7 +1171,7 @@ def _geo_key(nid):
# frame is what association pairs against — but only let a node
# *trigger* a round on its own cadence.
if mode == "track":
- assoc._pending_tracks[nid] = confirmed_track_views(trackers[nid], history_n)
+ assoc._pending_tracks[nid] = confirmed_track_views(trackers[nid], history_n, ts_ms)
else:
assoc._pending_frames[nid] = frame
if (t - last_assoc.get(nid, -1e9)) < assoc_interval:
@@ -1001,6 +1194,13 @@ def _geo_key(nid):
res.cluster_sizes[(_k, len(s_in.get("track_ids") or []))] += 1
if s_in.get("n_nodes", 0) < 2:
continue
+ _foreign = None
+ if mode == "track":
+ # Scored here, ahead of the solve and every gate below:
+ # this measures what association emitted, which is the
+ # thing the cluster-merge rework changes. Re-scored at
+ # the publish point further down, on the same number.
+ _foreign = _score_contamination(res, s_in, det_truth, truth)
try:
_t0 = time.perf_counter()
out = solve_fn(s_in, node_cfgs)
@@ -1104,6 +1304,17 @@ def _geo_key(nid):
(_keys_real if d <= MATCH_KM else _keys_ghost).add(_key)
if d <= MATCH_KM:
res.matched += 1
+ if _foreign is not None:
+ # Same input, scored again now that every gate has
+ # accepted it and it has bound to a real aircraft:
+ # this is the population the live audit sampled (45%
+ # of published dark solves carried a foreign node),
+ # and unlike the candidate rate its denominator is
+ # not something association can inflate.
+ res.published_inputs += 1
+ res.published_foreign_nodes += _foreign
+ if _foreign:
+ res.published_contaminated += 1
res.errors_km.append(d)
res.n_nodes_matched[nn] += 1
# Broken out because the whole dual-site hypothesis is
@@ -1153,6 +1364,7 @@ def _geo_key(nid):
res.gate_accepted = assoc.track_pairs_accepted
res.gate_unfitted = assoc.track_pairs_unfitted
res.gate_superseded = assoc.track_pairs_superseded
+ res.cluster_splits = getattr(assoc, "cluster_splits", 0)
res.claims_matched = assoc.claims_matched
res.claim_conflicts = assoc.claim_conflicts
res.anchored_inputs = assoc.anchored_inputs_emitted
@@ -1329,6 +1541,21 @@ def report(label: str, r: Result, truth_max_kt: float | None = None):
f" solves faster than any real aircraft ({truth_max_kt:.0f} kt): "
f"{over} ({100 * over / len(r.speeds_kt):.0f}%)"
)
+ if r.inputs_scored:
+ print(
+ f" CONTAMINATION: {r.inputs_contaminated}/{r.inputs_scored} solver inputs carry a foreign node"
+ f" -> {r.contaminated_inputs_pct:5.1f}% "
+ f"foreign nodes/input {r.foreign_nodes_per_input:.2f}"
+ f" ({r.foreign_nodes}/{r.input_nodes} nodes)"
+ )
+ if r.published_inputs:
+ print(
+ f" CONTAMINATION (published): {r.published_contaminated}/{r.published_inputs} matched solves"
+ f" -> {r.published_contaminated_pct:5.1f}% "
+ f"foreign nodes/solve {r.published_foreign_per_solve:.2f}"
+ )
+ if r.cluster_splits:
+ print(f" cluster splits (same-node track conflict): {r.cluster_splits}")
if r.gate_gated:
print(
f" CV gate: {r.gate_gated} pairings past the delay grid "
@@ -1498,6 +1725,32 @@ def main():
help="track mode: disable one-to-one hypothesis selection "
"(each pairing then answers only to the chi2 threshold)",
)
+ # Cluster-merge knobs (track mode). Each defaults to None, meaning "leave
+ # the library's own default alone", so the bench does not silently pin a
+ # value the library later changes — and so a sweep leg reads as exactly
+ # the deviation it is testing.
+ p.add_argument(
+ "--merge-dist-km",
+ type=float,
+ default=None,
+ help="track mode: how close two pairings must be to merge into one solver input (association._MERGE_DIST_KM)",
+ )
+ p.add_argument(
+ "--pair-vel-exclusive",
+ choices=("on", "off"),
+ default=None,
+ help="track mode, deferred only: drop a pairing whose implied velocity "
+ "contradicts a better-scoring pairing that claims the same track",
+ )
+ p.add_argument(
+ "--merge-vel-consistent",
+ choices=("on", "off"),
+ default=None,
+ help="track mode: require implied-velocity agreement, not just "
+ "proximity, before two pairings are merged into one cluster",
+ )
+ p.add_argument("--pair-vel-dv-ms", type=float, default=None, help="velocity-conflict speed threshold (m/s)")
+ p.add_argument("--pair-vel-dtheta-deg", type=float, default=None, help="velocity-conflict heading threshold (deg)")
p.add_argument("--min-aircraft", type=int, default=10, help="matches FLEET_AIRCRAFT lower bound")
p.add_argument("--max-aircraft", type=int, default=20)
p.add_argument("--metro-traffic-frac", type=float, default=0.85, help="matches FLEET_METRO_TRAFFIC_FRAC")
@@ -1568,6 +1821,14 @@ def main():
)
args = p.parse_args()
+ cluster_opts = {
+ "merge_dist_km": args.merge_dist_km,
+ "pair_vel_exclusive": None if args.pair_vel_exclusive is None else args.pair_vel_exclusive == "on",
+ "merge_vel_consistent": None if args.merge_vel_consistent is None else args.merge_vel_consistent == "on",
+ "pair_vel_dv_ms": args.pair_vel_dv_ms,
+ "pair_vel_dtheta_deg": args.pair_vel_dtheta_deg,
+ }
+
# --ss-metrics auto/on/off resolution. "on" without stonesoup installed
# is a hard error (the user explicitly asked for numbers this image
# cannot produce); "auto" degrades quietly except for one notice line so
@@ -1622,6 +1883,7 @@ def main():
+ f", fov={args.fov}"
+ f", ss-metrics={'on' if ss_metrics_enabled else 'off'}"
+ (f", smoother-legs={','.join(lbl for lbl, _, _ in smoother_legs)}" if smoother_legs else "")
+ + "".join(f", {k.replace('_', '-')}={v}" for k, v in sorted(cluster_opts.items()) if v is not None)
)
# chi2 only means anything in track mode; keep one pass otherwise.
@@ -1632,6 +1894,7 @@ def main():
solve_fn = _ESTIMATORS[estimator_name]
rates, solve_rates, reals, fakes, speed_errs = [], [], [], [], []
n2_rates = []
+ contam_rates, foreign_rates, med_errs, pub_contam_rates = [], [], [], []
agg = Result()
last = None
for k in range(args.repeat):
@@ -1666,6 +1929,7 @@ def main():
ss_metric_dt=args.ss_metric_dt,
ss_hold_s=args.ss_hold_s,
smoother_legs=smoother_legs,
+ cluster_opts=cluster_opts,
)
agg.merge(last, tag=f"s{args.seed + k}")
# Track-level is the comparable metric — solve-level and
@@ -1678,6 +1942,10 @@ def main():
fakes.append(len(last.ghost_tracks))
if last.speed_err_ms:
speed_errs.append(statistics.median(last.speed_err_ms))
+ contam_rates.append(last.contaminated_inputs_pct)
+ foreign_rates.append(last.foreign_nodes_per_input)
+ pub_contam_rates.append(last.published_contaminated_pct)
+ med_errs.append(statistics.median(last.errors_km) if last.errors_km else float("nan"))
label = f"assoc_interval={interval:g}s"
if chi2_max is not None:
label += f" chi2/dof<={chi2_max:g}"
@@ -1700,6 +1968,27 @@ def main():
f"({', '.join(f'{x:.0f}%' for x in n2_rates)})"
)
print(f" by solve: {', '.join(f'{x:.1f}%' for x in solve_rates)}")
+ if any(contam_rates):
+ print(
+ f" contaminated inputs per seed: "
+ f"{', '.join(f'{x:.0f}%' for x in contam_rates)}"
+ f" mean {statistics.mean(contam_rates):.1f}%"
+ )
+ print(
+ f" foreign nodes/input per seed: "
+ f"{', '.join(f'{x:.2f}' for x in foreign_rates)}"
+ f" mean {statistics.mean(foreign_rates):.2f}"
+ )
+ print(
+ f" published contaminated per seed: "
+ f"{', '.join(f'{x:.0f}%' for x in pub_contam_rates)}"
+ f" mean {statistics.mean(pub_contam_rates):.1f}%"
+ )
+ print(
+ f" median matched error per seed: "
+ f"{', '.join(f'{x:.2f}' for x in med_errs)} km"
+ f" real tracks {', '.join(str(x) for x in reals)}"
+ )
if speed_errs:
print(
f" median speed error per seed: "
diff --git a/backend/services/aircraft_feed.py b/backend/services/aircraft_feed.py
index 1befcf04..24cfbabd 100644
--- a/backend/services/aircraft_feed.py
+++ b/backend/services/aircraft_feed.py
@@ -7,6 +7,7 @@
in services.feed_gc.
"""
+import logging
import math
import os
import time
@@ -17,6 +18,8 @@
ARC_REFRESH_S,
CLAIMED_DISPLAY_FRESH_S,
GT_REFRESH_S,
+ MN_DARK_EXPIRY_S,
+ MN_DR_CAP_S,
MN_N2_MIN_SOLVES,
MN_ONESHOT_TTL_S,
STALE_TRACK_S,
@@ -56,12 +59,18 @@ def _reset_for_tests() -> None:
"""
global _cached_pending_arcs, _cached_detecting_nodes, _arcs_last_ts
global _cached_gt_snapshot, _cached_gt_meta, _gt_last_ts
+ global _mn_entry_fail_logged_at, _mn_entry_fail_count
_cached_pending_arcs = []
_cached_detecting_nodes = {}
_arcs_last_ts = 0.0
_cached_gt_snapshot = {}
_cached_gt_meta = {}
_gt_last_ts = 0.0
+ # Same reason as the wall-clock caches above: the log throttle is
+ # monotonic-time state, so one test's skipped entry would otherwise
+ # silence the next test's.
+ _mn_entry_fail_logged_at = 0.0
+ _mn_entry_fail_count = 0
_MN_ADSB_PREFIX = "mn-adsb-"
@@ -177,6 +186,71 @@ def multinode_to_aircraft(key: str, r: dict) -> dict:
return entry
+# One multinode entry failing is a bug worth a log line, but the feed builds
+# at 1 Hz over ~40 live keys, so an unguarded logger would turn one sick
+# aircraft into thousands of identical lines an hour and bury everything else.
+# Same shape of throttle detection_mirror.py uses for its dropped-frame line.
+_MN_ENTRY_FAIL_LOG_INTERVAL_S = 60.0
+_mn_entry_fail_logged_at = 0.0
+_mn_entry_fail_count = 0
+
+
+def _note_multinode_entry_failure(key: str) -> None:
+ """Count a skipped multinode entry, logging at most once a minute."""
+ global _mn_entry_fail_logged_at, _mn_entry_fail_count
+ _mn_entry_fail_count += 1
+ now = time.monotonic()
+ if now - _mn_entry_fail_logged_at < _MN_ENTRY_FAIL_LOG_INTERVAL_S:
+ return
+ _mn_entry_fail_logged_at = now
+ logging.exception(
+ "Multinode feed entry failed for key=%s, skipping it (%d total since boot)",
+ key,
+ _mn_entry_fail_count,
+ )
+
+
+def _multinode_entry(key: str, r: dict, now: float) -> dict:
+ """One multinode solve as a feed entry, dead-reckoned to ``now``.
+
+ Split out of build_combined_aircraft_json so the whole per-entry
+ computation — multinode_to_aircraft, the learned-velocity lookup, the
+ dead-reckon — sits behind one try/except there. Inline, an exception
+ from any of them took the entire flush with it.
+ """
+ ac = multinode_to_aircraft(key, r)
+ # Dead-reckon position using solver velocity (vel_east/vel_north in
+ # m/s), capped at MN_DR_CAP_S: beyond that a velocity error dominates
+ # any solve accuracy, so an old solve holds its last dead-reckoned
+ # point until the entry expiry rather than drifting further.
+ ts_fix = r.get("timestamp_ms", 0) / 1000.0
+ elapsed = min(now - ts_fix, MN_DR_CAP_S)
+ vel_east_m_s = r.get("vel_east", 0.0)
+ vel_north_m_s = r.get("vel_north", 0.0)
+ # TRACK_DR_SOURCE, read per call like TRACK_SMOOTHER: "kf" (default)
+ # dead-reckons with the display filter's LEARNED velocity when one
+ # exists — the solved velocity this block used to trust was measured
+ # (2026-08-09, n=93) at median 127 m/s vector error, i.e. ~3.8 km of
+ # drift at the 30 s cap below, worse than the solve error itself.
+ # "solve" restores the old behaviour (rollback, env only). The KF
+ # accessor returns None whenever the KF never saw this key (smoother
+ # in ewma/off mode, first solve, TTL-swept) so the fallback below is
+ # also the natural off-path, not a separate mode.
+ if (os.getenv("TRACK_DR_SOURCE", "kf") or "kf").strip().lower() != "solve":
+ _lv = track_filter.learned_velocity(key)
+ if _lv is not None:
+ vel_east_m_s, vel_north_m_s = _lv[0], _lv[1]
+ if elapsed > 0.0 and (vel_east_m_s != 0.0 or vel_north_m_s != 0.0):
+ _dr_lat, _dr_lon = offset_latlon_m(
+ ac["lat"],
+ ac["lon"],
+ east_m=vel_east_m_s * elapsed,
+ north_m=vel_north_m_s * elapsed,
+ )
+ ac["lat"], ac["lon"] = round(_dr_lat, 5), round(_dr_lon, 5)
+ return ac
+
+
def _claimed_single_node_entries(now: float) -> list[dict]:
"""Feed entries for hexes exactly ONE node is currently claiming.
@@ -340,54 +414,54 @@ def build_combined_aircraft_json(default_pipeline: PassiveRadarPipeline) -> dict
# 3. Multi-node solver
stale_mn = []
- for key, r in list(state.multinode_tracks.items()):
+ # Snapshot and evict under the solver's track lock: the solver worker
+ # iterates state.multinode_tracks inside multinode_key_decision while
+ # holding it, and a pop from this thread mid-iteration raised
+ # "dictionary changed size during iteration" live (2026-09-05). Lazy
+ # import: solver.py owns the lock and importing it at module level here
+ # would create a cycle through the task modules.
+ from services.tasks import solver as _solver_mod
+
+ with _solver_mod._MN_TRACKS_LOCK:
+ _mn_snapshot = list(state.multinode_tracks.items())
+ for key, r in _mn_snapshot:
age_s = now - r.get("timestamp_ms", 0) / 1000
- if age_s > 60:
+ # Lane-aware expiry, on the same key-prefix truth multinode_to_aircraft
+ # reads adsb_assisted off. An assisted entry is anchored to a
+ # transponder hex, so a long gap is the ADS-B feed breathing and the
+ # historic 60 s still fits it. A dark entry has nothing holding it in
+ # place: at the current 1–3 s dark solve cadence a 30 s gap is a lost
+ # track, and the 30–60 s band measured 3.99 km median error (32% over
+ # 5 km) — a confident icon kilometres from any aircraft.
+ _expiry_s = 60.0 if key.startswith(_MN_ADSB_PREFIX) else MN_DARK_EXPIRY_S
+ if age_s > _expiry_s:
stale_mn.append(key)
continue
# Display gates below are NOT staleness — a gated entry stays in
# state.multinode_tracks so the next solve can confirm it (n=2) or
- # supersede it, and only the age_s > 60 branch above discards its
+ # supersede it, and only the expiry branch above discards its
# anomaly hex. A one-shot solve renders nothing at all: a 2-node
# track needs a second solve to prove it isn't a mirror-point ghost,
# and a 3+-node one-shot gets a short preview window instead of the
- # full 60 s entry lifetime before it either confirms or expires.
+ # full entry lifetime before it either confirms or expires.
solve_count = int(r.get("solve_count") or 1)
if r.get("n_nodes") == 2 and solve_count < MN_N2_MIN_SOLVES:
continue
if r.get("n_nodes", 0) >= 3 and solve_count == 1 and age_s > MN_ONESHOT_TTL_S:
continue
- ac = multinode_to_aircraft(key, r)
- # Dead-reckon position using solver velocity (vel_east/vel_north in
- # m/s), capped at 30 s: beyond that a velocity error dominates any
- # solve accuracy (a 15 m/s error is already 450 m of drift at the
- # cap), so an old solve holds its last dead-reckoned point until the
- # 60 s entry expiry rather than drifting further.
- ts_fix = r.get("timestamp_ms", 0) / 1000.0
- elapsed = min(now - ts_fix, 30.0)
- vel_east_m_s = r.get("vel_east", 0.0)
- vel_north_m_s = r.get("vel_north", 0.0)
- # TRACK_DR_SOURCE, read per call like TRACK_SMOOTHER: "kf" (default)
- # dead-reckons with the display filter's LEARNED velocity when one
- # exists — the solved velocity this block used to trust was measured
- # (2026-08-09, n=93) at median 127 m/s vector error, i.e. ~3.8 km of
- # drift at the 30 s cap below, worse than the solve error itself.
- # "solve" restores the old behaviour (rollback, env only). The KF
- # accessor returns None whenever the KF never saw this key (smoother
- # in ewma/off mode, first solve, TTL-swept) so the fallback below is
- # also the natural off-path, not a separate mode.
- if (os.getenv("TRACK_DR_SOURCE", "kf") or "kf").strip().lower() != "solve":
- _lv = track_filter.learned_velocity(key)
- if _lv is not None:
- vel_east_m_s, vel_north_m_s = _lv[0], _lv[1]
- if elapsed > 0.0 and (vel_east_m_s != 0.0 or vel_north_m_s != 0.0):
- _dr_lat, _dr_lon = offset_latlon_m(
- ac["lat"],
- ac["lon"],
- east_m=vel_east_m_s * elapsed,
- north_m=vel_north_m_s * elapsed,
- )
- ac["lat"], ac["lon"] = round(_dr_lat, 5), round(_dr_lon, 5)
+ # One sick solve must not cost the whole broadcast. Everything from
+ # here to the append reads a single multinode entry, and an exception
+ # anywhere in it used to propagate out of the flush task ("Aircraft
+ # flush failed") and drop the ENTIRE tick's feed — every other
+ # aircraft with it — for one bad key. A skipped entry ages out of
+ # state.multinode_tracks on its own within 60 s, so degrading to
+ # "this one aircraft is missing for a few ticks" is strictly better
+ # than an empty map.
+ try:
+ ac = _multinode_entry(key, r, now)
+ except Exception:
+ _note_multinode_entry_failure(key)
+ continue
if ac["hex"] not in seen_hex:
seen_hex.add(ac["hex"])
append_track_history(ac["hex"], ac["lat"], ac["lon"], ac["alt_baro"], now)
@@ -410,7 +484,8 @@ def build_combined_aircraft_json(default_pipeline: PassiveRadarPipeline) -> dict
# without bound — enough to trip the anomaly_flood health check.
with state.anomaly_lock:
state.anomaly_hexes.discard(multinode_hex_from_key(k))
- state.multinode_tracks.pop(k, None)
+ with _solver_mod._MN_TRACKS_LOCK:
+ state.multinode_tracks.pop(k, None)
# 3b. Singly-claimed ADS-B targets — no seen_hex guard on purpose. A
# partially-claimed aircraft can still carry a tracker track keyed by the
diff --git a/backend/services/dark_follow.py b/backend/services/dark_follow.py
new file mode 100644
index 00000000..e949195e
--- /dev/null
+++ b/backend/services/dark_follow.py
@@ -0,0 +1,445 @@
+"""Dark track following (DARK_FOLLOW_MODE): the pseudo-state store behind the
+top-down claiming of aircraft that have no transponder.
+
+The dark lane is bottom-up all the way down — per-node tracker tracks, delay
+grid pairing, clustering, solve, and only THEN a key, chosen by proximity to
+whatever multinode entry happens to be nearest (solver.multinode_key_decision).
+Two consequences the map shows directly. Continuity: consecutive solves of one
+aircraft less than 5 s apart land on a different key 15% of the time, any gap
+in solving re-mints the key from scratch, and two aircraft 3 km apart can share
+one. Accuracy: nothing ever tells the solver where the aircraft is EXPECTED to
+be, so a solve starts from a 3 km grid centroid.
+
+The known lane already solved both problems for ADS-B aircraft, by inverting
+the order: identity first (services/known_claiming.py claims detections against
+a dead-reckoned transponder fix), then a solve seeded with that fix. This
+module supplies the missing half of the analogy — an identity for a dark
+aircraft. There is one available and it is already good: an established
+multinode track with a Kalman state. Its dead-reckoned position and velocity
+predict (delay, Doppler) at any node exactly the way an ADS-B fix does, so the
+same claiming machinery works against it unchanged.
+
+WHY NOT NODE-TRACK IDS. The obvious alternative — carry ``source_track_ids``
+forward and attach each new solve to the newest key sharing a node-track id —
+was simulated against a dense metro cluster and linked the WRONG aircraft 12%
+of the time. Single-node tracker tracks are genuinely shared between the
+association candidates of different aircraft (the same measurement that made
+solver._supersession_match stop trusting a bare shared id), so a node-track id
+is evidence about a detection, not about an aircraft. A predicted observation
+is evidence about an aircraft, which is what keying needs.
+
+THE GHOST RISK, AND THE GUARD. Following a track is a positive feedback loop:
+a solve keeps a key alive, the key keeps predicting, the prediction keeps
+claiming detections. Left alone that locks a ghost onto the map forever — the
+bottom-up lane can no longer disagree with it, because binding mode takes the
+detections away before the lane sees them. So a followed key is dropped (and
+put in cooldown, letting the bottom-up lane re-find it or not) as soon as it
+stops earning its place: two rejected follow-solves in a row, or a filter
+velocity sigma past DARK_FOLLOW_MAX_VEL_SIGMA_MS. The guard is the reason this
+lane is safe to bind; it is not optional tidiness.
+
+Modes (state.DARK_FOLLOW_MODE), same three-way shape as KNOWN_LANE_MODE:
+ off — nothing; no targets are built, so no claim can form.
+ shadow — claim, solve and record, but the claimed detections stay in the
+ dark pool and nothing is published. Default: the lane changes
+ which aircraft the map believes in, and that earns a soak.
+ binding — claimed detections leave the dark pool (the same
+ strip_claimed_detections the known lane uses) and the solve goes
+ onto the normal solver queue, keyed onto the followed track by
+ its anchor. Binding also gives the lane OWNERSHIP of the keys it
+ follows: a bottom-up solve may not join a key this lane published
+ on in the last DARK_FOLLOW_OWN_S — see that constant for the
+ measurement, and recently_followed for the reader.
+"""
+
+import logging
+import os
+import threading
+import time
+
+from config.constants import C_KM_US
+from core import state
+from services import track_filter
+
+_logger = logging.getLogger(__name__)
+
+# ── Target eligibility ───────────────────────────────────────────────────────
+# How stale a dark track's last solve may be and still be followed. Shorter
+# than the map's own 60 s expiry on purpose: past ~20 s the dead-reckoned
+# position is the KF's extrapolation rather than a measurement, and a claim
+# made against it would be the lane inventing its own evidence.
+DARK_FOLLOW_MAX_AGE_S = float(os.getenv("DARK_FOLLOW_MAX_AGE_S", "20"))
+# Minimum solves before a key may be followed. A key with one or two solves
+# behind it is exactly what the bottom-up lane mints for a mis-associated
+# fragment; requiring three means the aircraft has survived the whole gate
+# stack repeatedly before anything is claimed on its behalf. Same intent as
+# the library's CLAIM_ELIGIBLE_MIN_SOLVE_COUNT, one notch stricter because a
+# claim here also REMOVES detections from the lane that would disagree.
+DARK_FOLLOW_MIN_SOLVES = int(os.getenv("DARK_FOLLOW_MIN_SOLVES", "3"))
+# ...and how many nodes the last solve used. n>=3 is where the multinode
+# position is overdetermined; an n=2 track is a bistatic intersection that the
+# displacement and beam gates are still arguing about.
+DARK_FOLLOW_MIN_NODES = 3
+# Velocity sigma ceiling. The KF's own statement about how well it knows the
+# track's velocity, and the term that dominates the prediction error the moment
+# the track is coasted: at 60 m/s a 2 s coast is already 120 m of position
+# uncertainty, and the Doppler allowance the gate below derives from it is
+# ~78 Hz against a 25 Hz base — wider than the base gate, which is the point
+# past which the "prediction" stops constraining anything.
+DARK_FOLLOW_MAX_VEL_SIGMA_MS = float(os.getenv("DARK_FOLLOW_MAX_VEL_SIGMA_MS", "60"))
+# How long a dropped key stays un-followable. Long enough that the bottom-up
+# lane gets several association rounds (ASSOC_MIN_INTERVAL_S is 30 s at its
+# widest, ~2 s at its narrowest) to re-find the aircraft on its own evidence
+# before this lane is allowed to assert it again.
+DARK_FOLLOW_COOLDOWN_S = float(os.getenv("DARK_FOLLOW_COOLDOWN_S", "30"))
+# Minimum spacing between follow-solves for one key. Matched to the known
+# lane's pass interval: the aircraft is already being solved bottom-up too, and
+# a follow-solve every 2 s is four refreshes inside the map's 60 s expiry.
+DARK_FOLLOW_INTERVAL_S = float(os.getenv("DARK_FOLLOW_INTERVAL_S", "2.0"))
+
+# ── Key ownership ────────────────────────────────────────────────────────────
+# How long after a follow-solve publishes on a key that key stays the follow
+# lane's, i.e. un-joinable by a bottom-up solve (solver.multinode_key_decision,
+# binding mode only). Three follow-solve intervals: a followed track is solved
+# every DARK_FOLLOW_INTERVAL_S, so a key still inside this window is one the
+# lane is actively refreshing and does not need help keeping alive, while a key
+# that has missed three turns is one the lane has stopped answering for and the
+# bottom-up lane should be free to claim again.
+#
+# WHY OWNERSHIP AT ALL. Measured on test with the lane binding (20 min, 625
+# six-plus-node dark samples): of 425 bottom-up solves keyed by proximity onto
+# an existing key, 90 landed on a key belonging to a DIFFERENT aircraft (21%),
+# 12 of them onto a key the follow lane had published on within the previous
+# 6 s. A cross-keyed solve moves the entry 5+ km, corrupts the KF velocity it
+# feeds, and can supersede the right key. A tighter spatial gate cannot
+# separate the two cases: same-aircraft re-key distances are p50 1.5 km /
+# p90 4.3 km, overlapping the wrong-aircraft population entirely. What CAN
+# separate them is that the follow lane already supplies every solve an
+# established track needs — so near a freshly-followed key a bottom-up solve is
+# either a duplicate (it competes with the anchored solve and drags the filter)
+# or a different aircraft (it steals the key). Neither should join.
+DARK_FOLLOW_OWN_S = float(os.getenv("DARK_FOLLOW_OWN_S", "6.0"))
+# How close a bottom-up solve has to land to a followed key before it is
+# refused outright rather than merely kept off that key. Inside this radius
+# the two are the same aircraft often enough that minting a second key would
+# just fragment the track; outside it the solve is plausibly a neighbour the
+# follow lane knows nothing about and deserves a key of its own. Well under
+# the 6 km proximity gate on purpose — this is a "these are the same target"
+# radius, not an association gate.
+DARK_FOLLOW_SHADOW_KM = float(os.getenv("DARK_FOLLOW_SHADOW_KM", "2.0"))
+
+# Consecutive rejected follow-solves that drop a key. Two, not one: a single
+# reject is routinely a bad epoch (one node's contaminated measurement trips
+# the rms gate), while two in a row is the prediction itself being wrong.
+_MAX_CONSECUTIVE_REJECTS = 2
+
+# Position sigma to assume for a followed entry whose record carries none —
+# the KF's own cold-start value (track_filter._KF_DEFAULT_POS_SIGMA_M). An
+# entry without a filter-reported sigma is one the smoother has not updated,
+# so the cold-start number is the honest statement about it; deliberately not
+# a tighter guess, because understating sigma here narrows a gate.
+_DEFAULT_POS_SIGMA_M = 1200.0
+
+# Ceilings on the widened gates (see follow_gates). The uncertainty terms are
+# bounded by the thresholds above in normal operation, but the filter's own
+# sigma cap is 8 km, which alone would open a 53 µs delay gate — at that width
+# the claim is no longer a prediction test. 4x the base gates: past there the
+# aircraft should be re-found bottom-up, not followed.
+_MAX_DELAY_GATE_US = 40.0
+_MAX_DOPPLER_GATE_HZ = 100.0
+
+# Metres per microsecond and metres per second, from the one speed-of-light
+# constant this repo has.
+_C_M_PER_US = C_KM_US * 1000.0
+_C_M_PER_S = C_KM_US * 1.0e9
+
+# ── Target cache ─────────────────────────────────────────────────────────────
+# Rebuilt at most this often. The claiming stage runs once per frame per node
+# — ~37 calls/s on the test fleet — and each rebuild walks multinode_tracks and
+# takes track_filter's lock once per dark key. One second is short against
+# DARK_FOLLOW_MAX_AGE_S (so a target never survives its own staleness by more
+# than a rebuild) and long enough that the walk is amortised ~37x. The
+# per-frame dead-reckoning happens at the CALL site against the frame's own
+# timestamp, so caching the base list costs no prediction accuracy.
+_TARGETS_TTL_S = 1.0
+
+_TARGETS_LOCK = threading.Lock()
+_targets: list[dict] = []
+_targets_built_mono = 0.0
+
+# Guard state: consecutive rejects per key, and the cooldown each drop starts.
+_GUARD_LOCK = threading.Lock()
+_reject_streak: dict[str, int] = {}
+_cooldown_until: dict[str, float] = {}
+
+# Ownership state: key → the measurement epoch of the newest follow-solve that
+# published on it (see note_follow_publish for why the measurement clock and
+# not wall time). Dict-level TTL, same shape and reason as the guard maps
+# above: mn-dark-* keys churn for the process lifetime, so nothing keyed by one
+# may grow unbounded. 60 s is the map's own expiry — a key with no follow
+# publish for that long has no entry left to own.
+_FOLLOWED_TTL_S = 60.0
+_FOLLOWED_LOCK = threading.Lock()
+_last_follow_publish: dict[str, float] = {}
+
+
+def _reset_for_tests() -> None:
+ """Drop the target cache and the guard state. Tests only."""
+ global _targets, _targets_built_mono
+ with _TARGETS_LOCK:
+ _targets = []
+ _targets_built_mono = 0.0
+ with _GUARD_LOCK:
+ _reject_streak.clear()
+ _cooldown_until.clear()
+ with _FOLLOWED_LOCK:
+ _last_follow_publish.clear()
+ state.dark_follow_targets = 0
+
+
+def _expire_targets_for_tests() -> None:
+ """Force the next follow_targets() call to rebuild, KEEPING the guard state.
+
+ Tests only, and distinct from _reset_for_tests for exactly that reason: the
+ guard's whole behaviour is "a key that was a target stops being one", which
+ is unobservable if the only way to re-read the list also forgets the drop.
+ """
+ global _targets_built_mono
+ with _TARGETS_LOCK:
+ _targets_built_mono = 0.0
+
+
+def mode() -> str:
+ """The follow mode, defensively — the known lane's ``_mode`` precedent.
+
+ An absent or unrecognised value is "off": a flag this module cannot read
+ must disable the lane, never bind it.
+ """
+ m = getattr(state, "DARK_FOLLOW_MODE", "off")
+ return m if m in ("off", "shadow", "binding") else "off"
+
+
+def drop_target(key: str, reason: str) -> None:
+ """Stop following ``key`` for DARK_FOLLOW_COOLDOWN_S.
+
+ Idempotent inside a cooldown window: a key already in cooldown is not
+ re-dropped, so the counter reads "keys dropped", not "times the drop
+ condition was re-observed". The target cache is not invalidated — it
+ expires within _TARGETS_TTL_S, and the cooldown is re-tested on rebuild.
+ """
+ now = time.monotonic()
+ with _GUARD_LOCK:
+ if _cooldown_until.get(key, 0.0) > now:
+ return
+ _cooldown_until[key] = now + DARK_FOLLOW_COOLDOWN_S
+ _reject_streak.pop(key, None)
+ state.bump_counter("dark_follow_dropped")
+ _logger.debug("dark-follow: dropped %s (%s)", key, reason)
+
+
+def record_outcome(key: str, ok: bool) -> None:
+ """Feed one follow-solve verdict to the ghost guard.
+
+ ``ok`` is "this solve reached the feed" in binding mode and "this solve
+ landed within the dark displacement cap of its own prediction" in shadow —
+ the shadow pass has no gates to be rejected by, and a guard that stayed
+ inert there would leave the soak measuring a lane the binding one does not
+ have. Either way a good solve clears the streak and
+ _MAX_CONSECUTIVE_REJECTS bad ones in a row drop the key.
+ """
+ if ok:
+ with _GUARD_LOCK:
+ _reject_streak.pop(key, None)
+ return
+ with _GUARD_LOCK:
+ streak = _reject_streak.get(key, 0) + 1
+ _reject_streak[key] = streak
+ if streak >= _MAX_CONSECUTIVE_REJECTS:
+ drop_target(key, f"{streak} consecutive rejected follow-solves")
+
+
+def note_follow_publish(key: str, ts_s: float) -> None:
+ """Record that a follow-lane solve published on ``key`` at epoch ``ts_s``.
+
+ ``ts_s`` is the solve's MEASUREMENT epoch, not wall time, and that is not
+ an accident: the only reader is solver.multinode_key_decision, which is
+ deliberately clock-free — every time-of-day it uses arrives on the solve it
+ is judging — so that the keying rule stays replayable against recorded
+ history. Feeding it wall time here would make the one gate that decides
+ key ownership the one thing a replay could not reproduce.
+
+ Called from the single point every follow-solve outcome passes through
+ (solver._record_solve_history), so there is no path that publishes on a
+ followed key without marking it.
+ """
+ if not key or ts_s <= 0.0:
+ return
+ with _FOLLOWED_LOCK:
+ _last_follow_publish[key] = ts_s
+ cutoff = ts_s - _FOLLOWED_TTL_S
+ for k in [k for k, t in _last_follow_publish.items() if t < cutoff]:
+ del _last_follow_publish[k]
+
+
+def recently_followed(key: str, now_s: float, within_s: float = DARK_FOLLOW_OWN_S) -> bool:
+ """Did the follow lane publish on ``key`` within ``within_s`` of ``now_s``?
+
+ Symmetric in time on purpose. Solves reach the worker out of order (three
+ worker threads, a queue, and a per-key rate limit that batches claims), so
+ a bottom-up solve whose epoch sits just BEFORE the follow-solve's is
+ looking at the same instant of the same aircraft as one just after it, and
+ the ownership answer has to be the same for both.
+ """
+ with _FOLLOWED_LOCK:
+ last = _last_follow_publish.get(key)
+ return last is not None and abs(now_s - last) <= within_s
+
+
+def _in_cooldown(key: str, now_mono: float) -> bool:
+ with _GUARD_LOCK:
+ return _cooldown_until.get(key, 0.0) > now_mono
+
+
+def _sweep_guard(now_mono: float) -> None:
+ """Drop expired cooldowns. Both maps are keyed by mn-dark-* ids, which
+ churn for the process lifetime, so neither may grow unbounded."""
+ with _GUARD_LOCK:
+ for k in [k for k, until in _cooldown_until.items() if until <= now_mono]:
+ del _cooldown_until[k]
+ _reject_streak.pop(k, None)
+
+
+def _pos_sigma_m(rec: dict) -> float:
+ """The followed entry's own position uncertainty, in metres.
+
+ ``kf_pos_sigma_m`` is the smoother's post-update marginal (the honest
+ number, and the one solve_uncertainty already calibrates against);
+ ``pos_sigma_km`` is the solver's formal pre-inflation sigma, used only when
+ the filter never ran on this key. Both absent means a cold entry, which
+ gets the filter's own cold-start sigma rather than a flattering guess.
+ """
+ kf = rec.get("kf_pos_sigma_m")
+ if isinstance(kf, (int, float)) and kf > 0:
+ return float(kf)
+ formal = rec.get("pos_sigma_km")
+ if isinstance(formal, (int, float)) and formal > 0:
+ return float(formal) * 1000.0
+ return _DEFAULT_POS_SIGMA_M
+
+
+def _build_targets(now_s: float, now_mono: float) -> list[dict]:
+ """One pseudo-state per followable dark track. Cheap, and cached."""
+ out: list[dict] = []
+ for key, rec in list(state.multinode_tracks.items()):
+ if not key.startswith("mn-dark-"):
+ continue
+ if _in_cooldown(key, now_mono):
+ continue
+ lat, lon = rec.get("lat"), rec.get("lon")
+ if lat is None or lon is None:
+ continue
+ ts_ms = rec.get("timestamp_ms") or 0
+ age_s = now_s - ts_ms / 1000.0
+ if not (0.0 <= age_s <= DARK_FOLLOW_MAX_AGE_S):
+ continue
+ if int(rec.get("solve_count") or 0) < DARK_FOLLOW_MIN_SOLVES:
+ continue
+ if int(rec.get("n_nodes") or 0) < DARK_FOLLOW_MIN_NODES:
+ continue
+ # No filter state, no follow. The velocity and its sigma are the whole
+ # prediction: without them there is nothing to dead-reckon with and no
+ # way to widen a gate honestly, and the raw solved velocity is exactly
+ # the under-determined quantity (n<=3 Doppler) the KF exists to fix.
+ lv = track_filter.learned_velocity(key)
+ if lv is None:
+ continue
+ vel_east, vel_north, vel_sigma_ms, _last_ts_s = lv
+ if vel_sigma_ms > DARK_FOLLOW_MAX_VEL_SIGMA_MS:
+ drop_target(key, f"velocity sigma {vel_sigma_ms:.0f} m/s")
+ continue
+ # World tag. The overlap-zone world gate means every node that
+ # contributed to one multinode entry is from a single world, so the
+ # first contributor answers for all of them; an entry with no
+ # contributor list left (an old record shape) is untagged, and untagged
+ # passes every world — the same rule known_claiming's ADS-B world gate
+ # applies to a cache entry written before worlds existed.
+ node_ids = list(rec.get("contributing_node_ids") or ())
+ world = state.node_world(node_ids[0]) if node_ids else None
+ out.append(
+ {
+ "key": key,
+ "lat": float(lat),
+ "lon": float(lon),
+ "alt_m": float(rec.get("alt_m") or 0.0),
+ "vel_east": float(vel_east),
+ "vel_north": float(vel_north),
+ "vel_sigma_ms": float(vel_sigma_ms),
+ "pos_sigma_m": _pos_sigma_m(rec),
+ "timestamp_ms": int(ts_ms),
+ "world": world,
+ }
+ )
+ return out
+
+
+def follow_targets() -> list[dict]:
+ """The current dark follow targets, rebuilt at most once per _TARGETS_TTL_S.
+
+ Returns the shared cached list; callers must treat it (and the dicts in it)
+ as read-only. Empty in ``off`` mode, which is what makes the whole lane
+ cost one attribute read per frame when it is switched off.
+ """
+ global _targets, _targets_built_mono
+ if mode() == "off":
+ return []
+ now_mono = time.monotonic()
+ with _TARGETS_LOCK:
+ if _targets_built_mono and now_mono - _targets_built_mono < _TARGETS_TTL_S:
+ return _targets
+ _sweep_guard(now_mono)
+ _targets = _build_targets(time.time(), now_mono)
+ _targets_built_mono = now_mono
+ # A gauge, not a counter: it is the size of the list above, so it is
+ # assigned rather than bumped (state.bump_counter only adds).
+ state.dark_follow_targets = len(_targets)
+ return _targets
+
+
+def follow_gates(
+ target: dict, dt_s: float, base_delay_us: float, base_doppler_hz: float, fc_hz: float
+) -> tuple[float, float]:
+ """Claim gates for one follow target coasted ``dt_s`` seconds, in
+ (delay µs, Doppler Hz).
+
+ Two terms, because there are two independent error sources and the ADS-B
+ path only has the first:
+
+ MEASUREMENT. ``base * _gate_scale(dt)`` — the known lane's own gate,
+ age-scaled exactly as it is there (services/known_claiming._gate_scale).
+ This covers detection noise and the node's own bias.
+
+ STATE. The followed track's own uncertainty, converted into observation
+ space. A position error of ``s`` metres moves the bistatic range by at
+ most ``2s`` (the target can be displaced toward both the transmitter and
+ the receiver), so it is worth ``2s / c`` microseconds of delay; the
+ position error at claim time is the filter's position sigma plus its
+ velocity sigma coasted over ``dt``. A velocity error of ``u`` m/s
+ likewise moves the bistatic Doppler by at most ``2u / λ`` = ``2u·fc / c``
+ hertz. Both are worst-case projections (the true geometry factor is a
+ cosine ≤ 1), which is the right direction for a gate: it may admit a
+ detection the geometry would have excluded, never exclude one it should
+ have admitted.
+
+ Capped at _MAX_DELAY_GATE_US / _MAX_DOPPLER_GATE_HZ — see those constants.
+ """
+ # Imported here rather than at module scope: known_claiming imports this
+ # module for the claiming path, and a top-level import back would be a
+ # cycle. _gate_scale is the age-scaling rule itself, and duplicating it
+ # would let the two lanes drift apart on the same physics.
+ from services.known_claiming import _gate_scale
+
+ scale = _gate_scale(dt_s)
+ pos_err_m = target["pos_sigma_m"] + target["vel_sigma_ms"] * max(dt_s, 0.0)
+ d_gate = base_delay_us * scale + 2.0 * pos_err_m / _C_M_PER_US
+ f_gate = base_doppler_hz * scale + 2.0 * target["vel_sigma_ms"] * fc_hz / _C_M_PER_S
+ return min(d_gate, _MAX_DELAY_GATE_US), min(f_gate, _MAX_DOPPLER_GATE_HZ)
diff --git a/backend/services/frame_processor.py b/backend/services/frame_processor.py
index 0fb2bbd5..1b599084 100644
--- a/backend/services/frame_processor.py
+++ b/backend/services/frame_processor.py
@@ -18,6 +18,7 @@
ARCHIVE_BATCH_MAX,
ARCHIVE_FLUSH_INTERVAL_S,
N2_TRACK_HISTORY_MAX,
+ TRACK_MAX_STALE_S,
)
from core import state
from pipeline.passive_radar import PassiveRadarPipeline
@@ -156,6 +157,27 @@ def get_node_configs() -> dict[str, dict]:
return configs
+def configs_for_solver_input(node_cfgs: dict[str, dict], s_in: dict) -> dict[str, dict]:
+ """The subset of ``node_cfgs`` a solver input can actually reach.
+
+ The solver runs in a *spawn* process pool, so everything queued with an
+ input is pickled and shipped to a child on every call — and the fleet is
+ 58 nodes while a candidate carries 2-8 measurements. Sending the whole
+ set meant ~50 configs per solve that no code path could look at.
+
+ Nothing downstream needs the rest. The solver builds NodeSetups from the
+ measurements only; trimming and consensus both narrow that set further
+ (_filter_s_in_to_nodes) and never widen it; the beam gate iterates the
+ result's contributing_node_ids, which are measurement node ids by
+ construction; and cv_epochs is built from the same matched nodes as the
+ measurements in all three input shapes association emits. The known lane
+ is unaffected — it fetches its own configs (known_lane.run_known_lane_pass)
+ rather than reusing what was queued here.
+ """
+ wanted = {m.get("node_id") for m in (s_in.get("measurements") or ())}
+ return {nid: cfg for nid, cfg in node_cfgs.items() if nid in wanted}
+
+
# ── Per-node pipeline factory ─────────────────────────────────────────────────
@@ -254,7 +276,11 @@ def _view_adsb_hex(track, hist) -> str | None:
return hexn
-def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> list[dict]:
+def confirmed_track_views(
+ tracker,
+ history_n: int = N2_TRACK_HISTORY_MAX,
+ now_ts_ms: int | None = None,
+) -> list[dict]:
"""A tracker's confirmed tracks, in the shape submit_tracks takes.
TENTATIVE tracks are excluded, the same filter the arc builder applies: they
@@ -264,9 +290,26 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis
reason arcs keep it — at 22 fps a single missed frame flips ACTIVE →
COASTING and the next flips it back.
+ But COASTING is kept only while its newest REAL detection is fresh. What
+ travels downstream is ``history[-1]``, and association hands that sample to
+ the solver as the node's current measurement — so a track coasting toward
+ its N_DELETE deletion point contributes a delay from wherever the aircraft
+ was several seconds ago. That is the out-of-cone node the rms trim then
+ has to discard (see TRACK_MAX_STALE_S). The staleness test reads
+ ``hist[-1]["timestamp"]`` rather than the track's coast count because
+ get_recent_detections returns only ASSOCIATED samples — mark_missed appends
+ None to ``history["measurements"]`` and the reverse scan skips those — so
+ that timestamp IS the last real detection's, exactly the honest signal,
+ while n_missed only counts frames the node happened to process. Compared
+ against *now_ts_ms*, the frame timestamp being processed, never wall clock:
+ the fleet replays and backfills, and a filter keyed on wall clock would
+ silently empty every view in those runs. Skipped when the caller supplies
+ no frame time, or when TRACK_MAX_STALE_S is 0.
+
Shared with scripts/association_bench.py (which carried a near-verbatim
copy) so the bench feeds association exactly what production does.
"""
+ max_stale_ms = TRACK_MAX_STALE_S * 1000.0 if now_ts_ms is not None else 0.0
views = []
for tr in tracker.tracks:
if tr.state_status == TrackState.TENTATIVE:
@@ -274,6 +317,9 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis
hist = tr.get_recent_detections(history_n)
if len(hist) < 2:
continue
+ if max_stale_ms > 0 and (now_ts_ms - hist[-1]["timestamp"]) > max_stale_ms:
+ state.bump_counter("tracks_stale_skipped")
+ continue
views.append(
{
"track_id": tr.id or f"tmp-{id(tr)}",
@@ -292,8 +338,8 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis
return views
-def _node_track_views(pipeline: PassiveRadarPipeline) -> list[dict]:
- return confirmed_track_views(pipeline.tracker)
+def _node_track_views(pipeline: PassiveRadarPipeline, now_ts_ms: int | None = None) -> list[dict]:
+ return confirmed_track_views(pipeline.tracker, now_ts_ms=now_ts_ms)
def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarPipeline):
@@ -339,7 +385,9 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP
# cannot cross-pair into a phantom solve). _pframe is what the dark
# lane processes from here down; the original frame is untouched, so
# the archive and the ADS-B cache extraction below still see everything
- # the node sent.
+ # the node sent. Dark track following (DARK_FOLLOW_MODE) rides the same
+ # stage — it claims what the ADS-B paths leave, so it cannot run without
+ # them, which is why it is gated on KNOWN_LANE_MODE too.
_pframe = frame
if state.KNOWN_LANE_MODE != "off" and frame.get("delay"):
# Fail open: the known lane is an overlay on the dark lane, and in
@@ -347,10 +395,20 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP
# this guard, one ADS-B record with alt_baro="ground" threw here and
# took every frame down with it until the record aged out.
try:
- _claimed = claim_known_targets(node_id, frame)
+ # Two lanes claim here, with independent binding modes, so the
+ # strip is computed as one union and applied once: strip_claimed_
+ # detections re-bases the indices it keeps, and a second strip
+ # against the first's output would delete the wrong detections.
+ _followed: set[int] = set()
+ _claimed = claim_known_targets(node_id, frame, follow_claimed=_followed)
+ _strip: set[int] = set()
if _claimed and state.KNOWN_LANE_MODE == "binding":
- _pframe = strip_claimed_detections(frame, _claimed)
+ _strip |= _claimed
state.bump_counter("known_claims_bound", len(_claimed))
+ if _followed and state.DARK_FOLLOW_MODE == "binding":
+ _strip |= _followed
+ if _strip:
+ _pframe = strip_claimed_detections(frame, _strip)
except Exception:
state.bump_counter("known_claims_errors")
now = time.time()
@@ -397,7 +455,7 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP
# Track-level association. The detection-level path it replaced now lives
# in retina_analytics.detection_association, reachable only from the
# offline bench, which keeps it as the A/B baseline.
- _track_views = _node_track_views(pipeline)
+ _track_views = _node_track_views(pipeline, _ts_ms_assoc or None)
# Feed the per-node distinct-track counters — total_tracks /
# geolocated_tracks were exported (and read by the admin API) but never
# written anywhere.
@@ -427,7 +485,7 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP
if s_in["n_nodes"] < 2:
continue
try:
- state.solver_queue.put_nowait((s_in, node_cfgs, time.time()))
+ state.solver_queue.put_nowait((s_in, configs_for_solver_input(node_cfgs, s_in), time.time()))
except Exception:
state.bump_counter("solver_queue_drops")
if state.solver_queue_drops % 100 == 1:
diff --git a/backend/services/known_claiming.py b/backend/services/known_claiming.py
index 684d9a16..cf44df71 100644
--- a/backend/services/known_claiming.py
+++ b/backend/services/known_claiming.py
@@ -65,6 +65,7 @@
from config.constants import FT_TO_M, as_num
from core import state
+from services import dark_follow
from services.id_utils import normalize_hex_key
# Same base constants as the seeding path: the comparison is the identical
@@ -195,7 +196,143 @@ def _is_contested(delay_us: float, doppler_hz: float, projections: list[tuple[fl
)
-def claim_known_targets(node_id: str, frame: dict) -> set[int]:
+def _claim_dark_follow(
+ node_id: str,
+ geo,
+ frame_ts_s: float,
+ ts_ms: int,
+ delays: list,
+ dopplers: list,
+ free: list[int],
+) -> set[int]:
+ """Path 3: claim the leftover detections against followed dark tracks.
+
+ The ADS-B paths' assignment, run a second time with the dark pseudo-states
+ (services/dark_follow.py) standing in for cached transponder fixes — same
+ dead-reckoning, same visibility gate, same Hungarian one-to-one, same
+ per-axis normalised score. Only the gate widths differ, because a dark
+ pseudo-state carries its own uncertainty and an ADS-B fix is treated as
+ truth (see dark_follow.follow_gates).
+
+ Runs on ``free`` — what the ADS-B paths did not take — which IS the
+ precedence rule: an aircraft with a transponder can never lose a detection
+ to a dark track's prediction, whatever the residuals say. The reverse is
+ tolerable; a dark track that loses a detection is solved from its other
+ nodes, and a wrong ADS-B claim would charge a fix the node never saw to
+ that node's trust.
+
+ Claims land in state.known_claims under the mn-dark-* KEY rather than a
+ hex, marked ``dark_follow`` so known_lane's two passes can tell them apart.
+ They carry ``follow_fix`` rather than ``adsb_fix`` deliberately: every
+ other reader of the registry (the feed's single-node ADS-B section, the
+ per-node trust residuals) keys on ``adsb_fix``, and a follow claim has no
+ transponder fix to offer them — its absence is what keeps those readers
+ unchanged.
+ """
+ if not free or dark_follow.mode() == "off":
+ return set()
+ targets = dark_follow.follow_targets()
+ if not targets:
+ return set()
+
+ node_world = state.node_world(node_id)
+ cands: list[tuple[dict, float, float, float, float, float, float]] = []
+ for t in targets:
+ # Same world gate as path 2, same reason: a synthetic node's echoes are
+ # only ever of simulated aircraft. Untagged targets pass.
+ if t["world"] is not None and t["world"] != node_world:
+ continue
+ dt = frame_ts_s - t["timestamp_ms"] / 1000.0
+ if not (0.0 <= dt <= dark_follow.DARK_FOLLOW_MAX_AGE_S):
+ continue
+ dr_lat, dr_lon = offset_latlon_m(
+ t["lat"],
+ t["lon"],
+ east_m=t["vel_east"] * dt,
+ north_m=t["vel_north"] * dt,
+ )
+ # The associator's own visibility predicate, applied whole — the same
+ # call path 2 makes, for the same asymmetry: a false accept binds a
+ # detection to an aircraft this node cannot see and takes it out of the
+ # lane that would have disagreed.
+ if not _point_in_beam(dr_lat, dr_lon, geo):
+ continue
+ alt_km = t["alt_m"] / 1000.0
+ pred_d, pred_f = predict_observation(
+ geo,
+ dr_lat,
+ dr_lon,
+ alt_km,
+ t["vel_east"],
+ t["vel_north"],
+ )
+ d_gate, f_gate = dark_follow.follow_gates(
+ t,
+ dt,
+ KNOWN_CLAIM_DELAY_GATE_US,
+ KNOWN_CLAIM_DOPPLER_GATE_HZ,
+ geo.fc_hz,
+ )
+ cands.append((t, pred_d, pred_f, d_gate, f_gate, dr_lat, dr_lon))
+ if not cands:
+ return set()
+
+ cost = np.full((len(free), len(cands)), _GATE_INFEASIBLE)
+ for c, (_t, pred_d, pred_f, d_gate, f_gate, _dr_lat, _dr_lon) in enumerate(cands):
+ for r, i in enumerate(free):
+ d_res = abs(pred_d - float(delays[i]))
+ f_res = abs(pred_f - float(dopplers[i]))
+ if d_res > d_gate or f_res > f_gate:
+ continue
+ cost[r, c] = d_res / d_gate + f_res / f_gate
+ rows, cols = linear_sum_assignment(cost)
+
+ claimed: set[int] = set()
+ for r, c in zip(rows, cols):
+ if cost[r, c] >= _GATE_INFEASIBLE:
+ continue
+ i = free[r]
+ t, pred_d, pred_f, _d_gate, _f_gate, dr_lat, dr_lon = cands[c]
+ dq = state.known_claims.get(t["key"])
+ if dq is None:
+ dq = state.known_claims.setdefault(t["key"], deque(maxlen=state.KNOWN_CLAIMS_PER_HEX_MAX))
+ dq.append(
+ {
+ "node_id": node_id,
+ "delay_us": float(delays[i]),
+ "doppler_hz": float(dopplers[i]),
+ "pred_delay_us": float(pred_d),
+ "pred_doppler_hz": float(pred_f),
+ "ts_ms": ts_ms,
+ "dark_follow": True,
+ # The prediction itself, at the frame epoch — this is what the
+ # follow solve uses as its initial guess, which is the second
+ # thing this lane exists for (the first being the key). Unlike
+ # path 2's REPORTED-position rule there is no reported position
+ # to prefer: the dead-reckoned state is the only estimate there
+ # has ever been.
+ "follow_fix": {
+ "lat": dr_lat,
+ "lon": dr_lon,
+ "alt_km": t["alt_m"] / 1000.0,
+ "vel_east": t["vel_east"],
+ "vel_north": t["vel_north"],
+ "fix_ts_ms": ts_ms,
+ },
+ # Contention is an ADS-B-vs-dark question (identity evidence
+ # beating a dark projection). A follow claim IS the dark
+ # projection, so there is nothing for it to contend with, and
+ # leaving the flag false is what lets known_lane's selection
+ # reuse _select_claims unchanged.
+ "contested": False,
+ }
+ )
+ claimed.add(i)
+ state.bump_counter("dark_follow_claims")
+ return claimed
+
+
+def claim_known_targets(node_id: str, frame: dict, follow_claimed: set[int] | None = None) -> set[int]:
"""Run the claiming stage for one frame; return the claimed detection
indices.
@@ -212,6 +349,15 @@ def claim_known_targets(node_id: str, frame: dict) -> set[int]:
2. Remaining detections × fresh cached ADS-B states whose dead-reckoned
position this node can see, global one-to-one via
linear_sum_assignment under age-scaled gates.
+ 3. Dark track following (DARK_FOLLOW_MODE) — the same assignment again,
+ against established dark tracks' predicted observations instead of
+ ADS-B fixes. See _claim_dark_follow.
+
+ ``follow_claimed``, when given, is the set path 3's indices are written
+ into. They are deliberately NOT part of the return value: the two lanes
+ have independent binding modes, so the caller must be able to strip one
+ lane's claims from the frame without the other's. Omit it and path 3 does
+ not run at all — a caller that cannot receive the split cannot honour it.
Claims nothing without a registered geometry: the registry contract
requires the predicted observation, and there is nothing to predict
@@ -412,12 +558,9 @@ def claim_known_targets(node_id: str, frame: dict) -> set[int]:
)
claimed_idx.add(i)
- if not claims:
- return set()
-
# ── Contention, registry, counters, residual hook ─────────────────────────
- projections = _dark_global_projections(geo, frame_ts_s)
- nb = _node_bias()
+ projections = _dark_global_projections(geo, frame_ts_s) if claims else []
+ nb = _node_bias() if claims else None
for i, hexn, fix, pred_d, pred_f in claims:
d_meas = float(delays[i])
f_meas = float(dopplers[i])
@@ -452,6 +595,21 @@ def claim_known_targets(node_id: str, frame: dict) -> set[int]:
# makes a bias a bias.
nb.record_claim_residual(node_id, hexn, d_meas - pred_d, f_meas - pred_f, ts_ms)
+ # ── Path 3: dark track following ─────────────────────────────────────────
+ # Last, on what the ADS-B paths left behind — see _claim_dark_follow for
+ # why that ordering is the precedence rule rather than an implementation
+ # detail.
+ if follow_claimed is not None:
+ follow_claimed |= _claim_dark_follow(
+ node_id,
+ geo,
+ frame_ts_s,
+ ts_ms,
+ delays,
+ dopplers,
+ [i for i in range(len(delays)) if i not in claimed_idx],
+ )
+
return claimed_idx
diff --git a/backend/services/solve_uncertainty.py b/backend/services/solve_uncertainty.py
index 04e58e06..72f6de50 100644
--- a/backend/services/solve_uncertainty.py
+++ b/backend/services/solve_uncertainty.py
@@ -81,12 +81,15 @@
_SIGMA_MIN_M = 50.0
_SIGMA_MAX_M = 5000.0
-# Velocity-sigma clamp band and the growth horizon. Past 60 s the frontend
-# stops dead-reckoning entirely, so growing the disc further would describe
-# a position nothing is drawing.
+# Velocity-sigma clamp band and the growth horizon. Past the horizon the
+# frontend stops dead-reckoning entirely, so growing the disc further would
+# describe a position nothing is drawing. It tracks the frontend's
+# UNCERTAINTY_DR_CAP_S, which in turn tracks MN_DARK_EXPIRY_S: a dark entry no
+# longer reaches 60 s at all, so the old 60 s horizon described entries that
+# cannot exist.
_VEL_MIN_MS = 5.0
_VEL_MAX_MS = 150.0
-_GROWTH_MAX_AGE_S = 60.0
+_GROWTH_MAX_AGE_S = 30.0
def _floor_m(n_nodes: int) -> float:
@@ -152,13 +155,13 @@ def grown_sigma_m(sigma_m: float, vel_sigma_ms: float, age_s: float) -> float:
Position error at the solve epoch and velocity error over the coast are
independent, so their variances add:
- sigma(t) = sqrt( sigma_solve^2 + (sigma_v * min(t, 60))^2 )
+ sigma(t) = sqrt( sigma_solve^2 + (sigma_v * min(t, _GROWTH_MAX_AGE_S))^2 )
- Age is clamped to [0, 60]: negative is a clock artefact, and past 60 s the
- frontend has stopped dead-reckoning, so a bigger disc would not correspond
- to anything on screen. The frontend's own uncertainty.ts is the mirror of
- this function; this one exists for backend callers and for the tests that
- pin the two to the same shape.
+ Age is clamped to [0, _GROWTH_MAX_AGE_S]: negative is a clock artefact, and
+ past the horizon the frontend has stopped dead-reckoning, so a bigger disc
+ would not correspond to anything on screen. The frontend's own
+ uncertainty.ts is the mirror of this function; this one exists for backend
+ callers and for the tests that pin the two to the same shape.
"""
t = min(max(age_s, 0.0), _GROWTH_MAX_AGE_S)
return float(math.sqrt(sigma_m**2 + (vel_sigma_ms * t) ** 2))
diff --git a/backend/services/tasks/known_lane.py b/backend/services/tasks/known_lane.py
index 6ffe29ad..e9be7f1a 100644
--- a/backend/services/tasks/known_lane.py
+++ b/backend/services/tasks/known_lane.py
@@ -41,6 +41,17 @@
published: a displaced solve under a real hex is a wrong map
marker, the same reason the regular displacement gate exists.
+A SECOND PASS lives here too: the dark-follow lane (DARK_FOLLOW_MODE, see
+services/dark_follow.py) applies the same inversion to aircraft that have no
+transponder, using an established mn-dark-* track's Kalman state where this
+lane uses an ADS-B fix. It shares this module's registry reader, pass lock and
+interval — the two passes read the same ``state.known_claims`` and differ only
+in which kind of claim they select — but not its solve path: a follow solve
+goes onto the normal solver queue so the dark gate stack judges it, whereas a
+known-lane solve is deliberately free of that stack (see the free-solve
+invariant above). ``run_dark_follow_pass`` and ``_build_follow_solver_input``
+carry the detail.
+
Neither ``state.known_claims`` nor ``state.KNOWN_LANE_MODE`` exists on this
branch — slice A owns core/state.py — so every access goes through getattr
with an inert default, and the counters below are registered onto the state
@@ -55,7 +66,7 @@
from config.constants import FT_TO_M
from core import state
-from services import track_filter
+from services import dark_follow, track_filter
from services.geo import haversine_km, offset_latlon_m
from services.id_utils import normalize_hex_key
@@ -128,6 +139,13 @@
_last_pass_ts = 0.0
_last_attempt_ts_ms: dict[str, int] = {}
_last_sample_mono: dict[str, float] = {}
+# Dark-follow pass bookkeeping, per followed key: when it last produced a
+# follow-solve (monotonic, the DARK_FOLLOW_INTERVAL_S rate limit) and the claim
+# epoch that solve was built from (the same "no newer claim, no new solve"
+# dedup _last_attempt_ts_ms gives the ADS-B pass). Both single-writer under
+# _PASS_LOCK, like the maps above.
+_last_follow_mono: dict[str, float] = {}
+_last_follow_ts_ms: dict[str, int] = {}
def _reset_for_tests() -> None:
@@ -137,6 +155,8 @@ def _reset_for_tests() -> None:
_last_pass_ts = 0.0
_last_attempt_ts_ms.clear()
_last_sample_mono.clear()
+ _last_follow_mono.clear()
+ _last_follow_ts_ms.clear()
with state.counters_lock:
for name in _COUNTERS:
setattr(state, name, 0)
@@ -157,7 +177,7 @@ def _num(v, fallback=0.0) -> float:
return float(fallback)
-def _select_claims(dq, now_ms: int) -> dict[str, dict]:
+def _select_claims(dq, now_ms: int, follow: bool = False) -> dict[str, dict]:
"""Newest usable claim per node from one hex's deque, or {} if fewer than
two nodes survive.
@@ -168,11 +188,20 @@ def _select_claims(dq, now_ms: int) -> dict[str, dict]:
is slice A's, written concurrently, and this reader must survive any
single bad entry. Nodes whose newest claim trails the newest overall by
more than _CLAIM_SPREAD_S are dropped rather than failing the whole hex.
+
+ ``follow`` selects which KIND of claim: the dark-follow claims a key's
+ deque holds (DARK_FOLLOW_MODE, see services/dark_follow.py) or the ADS-B
+ ones a hex's does. The two never share a deque in practice — a registry
+ entry is keyed either by transponder hex or by mn-dark-* key — but the
+ filter is explicit rather than implied by the key shape, so a stray entry
+ can only be ignored, never solved by the wrong lane.
"""
best: dict[str, dict] = {}
for c in list(dq):
if not isinstance(c, dict):
continue
+ if bool(c.get("dark_follow")) is not follow:
+ continue
try:
ts_ms = int(c["ts_ms"])
node_id = c["node_id"]
@@ -245,6 +274,12 @@ def _build_solver_input(hexn: str, claims: dict[str, dict]) -> dict | None:
"delay_us": float(c["delay_us"]),
"doppler_hz": float(c["doppler_hz"]),
"snr": _num(c.get("snr")),
+ # This lane does NOT have one epoch: _CLAIM_SPREAD_S admits
+ # claims up to 5 s apart, which at 300 m/s is ~1.5 km of target
+ # motion charged straight to the residual this lane exists to
+ # measure. Carrying each claim's own capture time lets
+ # _attempt reuse the regular lane's epoch alignment.
+ "t_s": int(c["ts_ms"]) / 1000.0,
}
for nid, c in sorted(claims.items())
],
@@ -362,6 +397,15 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non
record's displacement_km and the accuracy error are the same number.
"""
state.bump_counter("known_lane_attempts")
+ # Same correction, same flag, same helper as the regular lane — see
+ # solver.align_measurement_epochs. Applied here rather than in
+ # _build_solver_input because the alignment needs the node configs, and
+ # because the accuracy classification below compares the solve against an
+ # initial guess already dead-reckoned to the newest claim's epoch, which is
+ # exactly the t0 the helper aligns onto.
+ epoch_meta: dict = {"epoch_aligned": False}
+ if state.SOLVER_EPOCH_ALIGN:
+ s_in, epoch_meta = solver_mod.align_measurement_epochs(s_in, node_cfgs)
try:
# Single solve at the pinned ADS-B altitude — no layer sweep. The
# sweep exists to DISCOVER an unknown altitude; here identity already
@@ -379,7 +423,7 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non
"known_no_converge",
s_in,
result if isinstance(result, dict) else None,
- extra={"known_lane": True, "label": "no_converge", "published": False},
+ extra={"known_lane": True, "label": "no_converge", "published": False, **epoch_meta},
)
return
@@ -419,7 +463,7 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non
raw_lat=raw_lat,
raw_lon=raw_lon,
displacement_km=err_km,
- extra={"known_lane": True, "label": label, "published": published},
+ extra={"known_lane": True, "label": label, "published": published, **epoch_meta},
)
@@ -484,6 +528,216 @@ def run_known_lane_pass(solve_fn, node_cfgs: dict | None = None, mode: str | Non
return attempts
+def _build_follow_solver_input(key: str, claims: dict[str, dict]) -> dict | None:
+ """Shape one followed dark track's claims into a solver input.
+
+ _build_solver_input's shape, with the dark pseudo-state (see
+ services/dark_follow.py) supplying what the ADS-B fix supplies there: the
+ initial guess is the KF prediction dead-reckoned to the newest claim's
+ epoch, and the velocity seed is the KF's learned velocity. Two fields the
+ ADS-B lane has no use for carry the rest of the point of this lane:
+
+ anchor_key — the followed key itself. multinode_key_decision's anchor
+ branch honours it (it is mn-dark-*, it is live, and the solve started
+ from this track's own prediction so it lands inside the branch's flat
+ _MN_ASSOC_MAX_DIST_KM check by construction), so the solve keys onto the
+ SAME track it was predicted from. That is continuity by construction
+ rather than by the proximity scan happening to pick the right neighbour
+ — the failure the measurements behind this lane are of.
+
+ track_ids — deliberately empty. A follow input is detection-level: its
+ correspondence came from the prediction, not from a tracker pairing, so
+ there is no provenance to record. The empty set is also what admits it
+ past _claim_resolve_slot (an input with no track provenance has nothing
+ to be a duplicate of) and what keeps it out of supersession, which must
+ not pop a neighbour on a track id this solve never used.
+
+ ``lane``/``guess_source``/``follow_key`` ride through to the history record
+ (solver._record_solve_history stamps all three) so the lane is separable in
+ /api/test/solver-stats without inferring it from the key.
+
+ A CONSEQUENCE WORTH KNOWING: with no cv_epochs on the input, an n=2 follow
+ solve cannot pass the n=2 confirmation gate and is always withheld, so the
+ lane publishes at n>=3 only. Left that way on purpose — at n=2 the solver
+ fits five unknowns to four measurements and returns a zero residual for a
+ WRONG claim exactly as it does for a right one, which is the same blindness
+ the gate exists for and is not made safer by the claim having come from a
+ prediction. A follow target already needs n>=3 to be followed at all; this
+ just means it needs three claiming nodes as well.
+ """
+ newest = max(claims.values(), key=lambda c: int(c["ts_ms"]))
+ newest_ts_ms = int(newest["ts_ms"])
+ fix = newest.get("follow_fix")
+ if not isinstance(fix, dict):
+ return None
+ lat, lon = fix.get("lat"), fix.get("lon")
+ if lat is None or lon is None:
+ return None
+
+ vel_east = _num(fix.get("vel_east"))
+ vel_north = _num(fix.get("vel_north"))
+ # Normally zero: the newest claim's prediction was made at its own frame
+ # epoch, which is this epoch. Kept for the same reason the ADS-B path
+ # keeps it — the claim record is written by another thread and the epoch it
+ # names is the only thing that says when the prediction was true.
+ dt_s = (newest_ts_ms - int(_num(fix.get("fix_ts_ms"), newest_ts_ms))) / 1000.0
+ guess_lat, guess_lon = offset_latlon_m(
+ float(lat),
+ float(lon),
+ east_m=vel_east * dt_s,
+ north_m=vel_north * dt_s,
+ )
+
+ return {
+ "initial_guess": {
+ "lat": guess_lat,
+ "lon": guess_lon,
+ "alt_km": _num(fix.get("alt_km")),
+ },
+ "initial_velocity": {
+ "vel_east_ms": vel_east,
+ "vel_north_ms": vel_north,
+ },
+ "measurements": [
+ {
+ "node_id": nid,
+ "delay_us": float(c["delay_us"]),
+ "doppler_hz": float(c["doppler_hz"]),
+ "snr": _num(c.get("snr")),
+ }
+ for nid, c in sorted(claims.items())
+ ],
+ "n_nodes": len(claims),
+ "timestamp_ms": newest_ts_ms,
+ "anchor_key": key,
+ "track_ids": [],
+ "lane": "dark_follow",
+ "guess_source": "prediction",
+ "follow_key": key,
+ }
+
+
+def _follow_shadow_attempt(key: str, s_in: dict, node_cfgs: dict, solve_fn) -> None:
+ """Solve one follow input without touching the feed, and record it.
+
+ Shadow's whole job is to answer "would this lane have helped?", which needs
+ the solve to actually run — the claim alone says nothing about whether the
+ prediction was right. Classified against the dark displacement cap, the
+ same number the binding path's gate would judge it by, so the shadow record
+ and a binding reject mean the same thing; the verdict also feeds the ghost
+ guard, which would otherwise be inert for the whole soak.
+ """
+ try:
+ result = solve_fn(s_in, node_cfgs)
+ except Exception:
+ logging.exception("Dark-follow shadow solve failed for %s", key)
+ result = None
+
+ ok = False
+ disp_km = None
+ if result and result.get("success"):
+ ig = s_in["initial_guess"]
+ disp_km = haversine_km(float(ig["lat"]), float(ig["lon"]), float(result["lat"]), float(result["lon"]))
+ ok = disp_km <= solver_mod._MAX_DISPLACEMENT_KM_DARK
+ solver_mod._record_solve_history(
+ "dark_follow_shadow",
+ s_in,
+ result if isinstance(result, dict) else None,
+ displacement_km=disp_km,
+ extra={"shadow": True, "published": False, "follow_ok": ok},
+ )
+
+
+def run_dark_follow_pass(solve_fn, node_cfgs: dict | None = None, mode: str | None = None) -> int:
+ """One pass over the dark-follow claims; returns the inputs produced.
+
+ A followed key is solved when it has fresh follow-claims from >= 2 nodes at
+ compatible timestamps (_select_claims, follow=True), a claim newer than its
+ last follow-solve, and its DARK_FOLLOW_INTERVAL_S rate limit has elapsed.
+ In binding mode the input goes onto state.solver_queue and is processed by
+ the normal dark path — the same gate stack, keying, KF smoothing and
+ history records every bottom-up solve gets, which is the point: this lane
+ changes where a solve STARTS and which key it lands on, not what a solve
+ has to survive. In shadow it is solved and recorded here instead, because
+ anything reaching the queue would publish.
+
+ ``mode`` overrides state.DARK_FOLLOW_MODE for this pass, for the same
+ reason run_known_lane_pass takes the override: a solver worker daemon
+ leaked into the test process would otherwise race the test for the per-key
+ rate limit.
+ """
+ if mode is None:
+ mode = dark_follow.mode()
+ elif mode not in ("off", "shadow", "binding"):
+ mode = "off"
+ if mode == "off":
+ return 0
+ claims_by_key = getattr(state, "known_claims", None)
+ if not claims_by_key:
+ return 0
+
+ now_ms = int(time.time() * 1000)
+ now_mono = time.monotonic()
+ inputs = 0
+ for key, dq in list(claims_by_key.items()):
+ if not isinstance(key, str) or not key.startswith("mn-dark-"):
+ continue
+ last_mono = _last_follow_mono.get(key)
+ if last_mono is not None and now_mono - last_mono < dark_follow.DARK_FOLLOW_INTERVAL_S:
+ continue
+ claims = _select_claims(dq, now_ms, follow=True)
+ if not claims:
+ continue
+ if node_cfgs is None:
+ from services.frame_processor import get_node_configs
+
+ node_cfgs = get_node_configs()
+ # A node whose config has gone (disconnected since the claim) cannot be
+ # solved with: the LM needs its geometry. Drop the node rather than
+ # the key — the remaining nodes are still a solve if there are two.
+ claims = {nid: c for nid, c in claims.items() if nid in node_cfgs}
+ if len(claims) < 2:
+ continue
+ newest_ts = max(int(c["ts_ms"]) for c in claims.values())
+ if _last_follow_ts_ms.get(key, -1) >= newest_ts:
+ continue
+ s_in = _build_follow_solver_input(key, claims)
+ if s_in is None:
+ continue
+ # Stamped before the solve, as the ADS-B pass does: a solve that raises
+ # must not be retried against the same claims on every pass forever.
+ _last_follow_mono[key] = now_mono
+ _last_follow_ts_ms[key] = newest_ts
+ cfgs = {nid: node_cfgs[nid] for nid in claims}
+ state.bump_counter("dark_follow_inputs")
+ inputs += 1
+ if mode == "binding":
+ try:
+ state.solver_queue.put_nowait((s_in, cfgs, time.time()))
+ except Exception:
+ state.bump_counter("solver_queue_drops")
+ else:
+ _follow_shadow_attempt(key, s_in, cfgs, solve_fn)
+
+ # Opportunistic TTL sweep of both dedup maps — keys churn for the process
+ # lifetime, the same reason _ATTEMPT_TTL_S exists.
+ cutoff_ms = now_ms - _ATTEMPT_TTL_S * 1000.0
+ for k in [k for k, ts in _last_follow_ts_ms.items() if ts < cutoff_ms]:
+ del _last_follow_ts_ms[k]
+ _last_follow_mono.pop(k, None)
+ return inputs
+
+
+def lanes_armed() -> bool:
+ """True when either lane in this module has something to do.
+
+ The solver worker arms its per-iteration pass call on this ONCE, at thread
+ start (see _run_solver_worker for why an off lane must not cost the idle
+ loop even a mode read).
+ """
+ return _mode() != "off" or dark_follow.mode() != "off"
+
+
def maybe_run_pass(solve_fn, mode: str | None = None) -> None:
"""Interval- and mode-gated pass entry point for the solver worker loop.
@@ -493,11 +747,13 @@ def maybe_run_pass(solve_fn, mode: str | None = None) -> None:
rather than queue, and the interval check lives under the same lock so
two workers cannot both pass it in the same window. ``mode`` is the
same test-only override run_known_lane_pass documents; the worker loop
- always passes nothing and reads the live flag.
+ always passes nothing and reads the live flag. It applies to the ADS-B
+ pass only — the dark-follow pass reads its own flag, so a test that arms
+ one lane explicitly does not silently arm the other.
"""
global _last_pass_ts
try:
- if (mode if mode is not None else _mode()) == "off":
+ if (mode if mode is not None else _mode()) == "off" and dark_follow.mode() == "off":
return
if not _PASS_LOCK.acquire(blocking=False):
return
@@ -507,6 +763,11 @@ def maybe_run_pass(solve_fn, mode: str | None = None) -> None:
return
_last_pass_ts = now
run_known_lane_pass(solve_fn, mode=mode)
+ # Second, and under the same lock and interval: the two passes read
+ # the same registry, and the follow pass's per-key rate limit is
+ # single-writer for exactly the reason the ADS-B pass's dedup map
+ # is. Its own mode flag decides whether it does anything.
+ run_dark_follow_pass(solve_fn)
finally:
_PASS_LOCK.release()
except Exception:
diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py
index 7c6b947c..dcdafcb5 100644
--- a/backend/services/tasks/solver.py
+++ b/backend/services/tasks/solver.py
@@ -11,6 +11,8 @@
from collections import deque
from concurrent.futures.process import BrokenProcessPool
+from retina_analytics.association import _point_in_beam
+
from config.constants import (
ARC_ONLY_ANOMALY_ALLOWLIST,
ASSOC_GRID_STEP_KM,
@@ -19,7 +21,7 @@
N2_TRACK_ASSOCIATION,
)
from core import state
-from services import track_filter
+from services import dark_follow, track_filter
# Beam-coverage geometry, used to reject solver results whose range or (at
# n=2) bearing fall outside a contributing node's detection area. This
@@ -100,6 +102,24 @@ def _pool_call(fn, *args):
return fn(*args)
+def _pool_solve_multistart(s_in, node_cfgs, alt_starts_km):
+ """solve_multinode_multistart via the process pool (inline when none).
+
+ Defined here rather than beside _pool_solve_multinode at the foot of this
+ module for the reason _pool_select_consensus is: it is a default argument
+ value, resolved when the ``def`` executes, so it has to be bound before
+ _solve_best_altitude's signature is reached.
+
+ A module-level function taking only picklable arguments, because the pool
+ is a *spawn* pool — a child imports retina_geolocator and nothing of the
+ backend, so what crosses is this function's qualified name plus the input
+ dicts.
+ """
+ from retina_geolocator.multinode_solver import solve_multinode_multistart
+
+ return _pool_call(solve_multinode_multistart, s_in, node_cfgs, alt_starts_km, True)
+
+
# Altitude layers (km) tried when n_nodes ≥ 3. For an overdetermined system
# (3+ delay equations, 2 unknowns after altitude pinning) only the correct
# altitude layer yields rms_delay ≈ 0; wrong layers give rms > 0, so picking
@@ -279,6 +299,91 @@ def _dark_displacement_cap_km() -> float:
# It lived here while the frame path had its own, looser, unstated one.
+# ── Measurement epoch alignment ──────────────────────────────────────────────
+# The solver's residual model evaluates every measurement against ONE target
+# state: the measurement set is assumed simultaneous. It is not. Each node
+# samples on its own free-running cadence (~0.74-1 Hz on the fleet), so the
+# delays in one solver input were captured at times spread over up to a frame
+# interval, and association hands over each track's newest sample regardless of
+# when that was. A 250 m/s target moves ~250 m per second of skew, which shows
+# up as up to ~1 us of bistatic delay error per second — charged in full to the
+# 3 us rms_delay gate, where it is indistinguishable from a contaminated node
+# and drives the trim to throw away legitimately in-cone nodes.
+#
+# The correction is closed-form and needs nothing the measurement does not
+# already carry. Writing d_tx / d_rx for the TX->target and target->RX ranges,
+# the bistatic delay is (d_tx + d_rx - baseline)/c and the bistatic Doppler is
+# (fc/c)(v_tx + v_rx), where v_tx / v_rx are the target's velocity components
+# along the unit vectors pointing FROM the target TOWARD the TX and the RX
+# (retina_geolocator.multinode_solver._residual_function; the simulator's
+# _bistatic_delay / _bistatic_doppler in retina_simulation.world use the
+# identical convention). Moving toward a site shortens that leg, so
+# d(d_tx)/dt = -v_tx and d(d_rx)/dt = -v_rx, and therefore
+#
+# d(delay_us)/dt = -(v_tx + v_rx) / C_KM_US
+# = -doppler_hz * (C_KM_S / fc_hz) / C_KM_US
+# = -doppler_hz * 1e6 / fc_hz
+#
+# i.e. positive Doppler is a closing target and its delay is DECREASING. The
+# unit test test_epoch_alignment.py checks the sign against a target flown
+# through the simulator's own geometry helpers at two times, rather than
+# against this derivation.
+_DELAY_RATE_HZ_TO_US_PER_S = 1e6
+
+
+def align_measurement_epochs(s_in: dict, node_cfgs: dict) -> tuple[dict, dict]:
+ """Dead-reckon every measurement's delay onto the newest one's epoch.
+
+ Pure: returns a new solver input (shallow copy, fresh measurement dicts)
+ and a metadata dict for the history record; *s_in* is never mutated, so a
+ caller can drop the result and keep the untouched input.
+
+ Alignment is all-or-nothing per input. A partially aligned set is worse
+ than an unaligned one — the residual model has no way to know which
+ measurements share an epoch, so mixing corrected and uncorrected delays
+ just moves the error onto a different node. Any measurement missing t_s
+ or doppler_hz, or whose node has no config to read fc_hz from, therefore
+ skips the whole input and counts solver_epoch_align_skipped.
+
+ Returns (s_in, meta) where meta carries epoch_aligned and, when it ran,
+ epoch_skew_s — the widest gap the correction closed.
+ """
+ meas = s_in.get("measurements") or []
+ if len(meas) < 2:
+ return s_in, {"epoch_aligned": False}
+
+ rates = []
+ for m in meas:
+ t_s = m.get("t_s")
+ doppler = m.get("doppler_hz")
+ cfg = node_cfgs.get(m.get("node_id")) or {}
+ # Same fallback chain the geolocator uses when it builds its NodeSetup,
+ # so a node whose config spells the carrier "FC" aligns on exactly the
+ # frequency the solve will predict against.
+ fc_hz = cfg.get("fc_hz", cfg.get("FC"))
+ if t_s is None or doppler is None or not fc_hz:
+ state.bump_counter("solver_epoch_align_skipped")
+ return s_in, {"epoch_aligned": False}
+ rates.append((float(t_s), -float(doppler) * _DELAY_RATE_HZ_TO_US_PER_S / float(fc_hz)))
+
+ # The newest sample, not the input's timestamp_ms: t0 has to be a time some
+ # measurement was actually taken, or every delay is extrapolated and the
+ # freshest node — the one that needed no correction — acquires an error.
+ t0 = max(t for t, _ in rates)
+ skew_s = t0 - min(t for t, _ in rates)
+
+ aligned = dict(s_in)
+ aligned["measurements"] = [
+ {**m, "delay_us": float(m["delay_us"]) + rate * (t0 - t_s)} for m, (t_s, rate) in zip(meas, rates)
+ ]
+ if "timestamp_ms" in aligned:
+ # The set now describes t0, so everything downstream that ages this
+ # solve (multinode expiry, the dead-reckoning gates, the history
+ # record's measurement_ts_ms) should date it from t0 too.
+ aligned["timestamp_ms"] = int(round(t0 * 1000.0))
+ return aligned, {"epoch_aligned": True, "epoch_skew_s": round(skew_s, 3)}
+
+
def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[float], metric: str) -> dict | None:
"""Try each altitude layer; return the result with lowest value of `metric`.
@@ -320,17 +425,82 @@ def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[floa
return best_result
-def _solve_best_altitude(s_in: dict, node_cfgs: dict, solve_fn) -> dict | None:
- """Altitude sweep for n≥3: pick by minimum rms_delay.
+# Fewest measurements the free mode is used at. Below this altitude is not
+# observable and retina_geolocator pins it anyway; the sweep is left in place
+# so the n=2 path keeps its documented behaviour exactly.
+_FREE_ALT_MIN_NODES = 3
+
+
+def _free_alt_starts(ig_alt_km, layers: list[float]) -> list[float]:
+ """The start altitudes the free mode hands the multi-start helper.
+
+ state.SOLVER_FREE_ALT_STARTS of them, clamped into [1, len(layers)] — read
+ per call, like the mode flag, so a test and a config reload both see what
+ they set. One start is the layer nearest ``ig_alt_km``, which is the
+ altitude spliced into ``layers`` when the input carries a non-layer one of
+ its own (ADS-B), exactly as the sweep treats it. Several are a window
+ centred on that layer, clamped to the ends of the ladder so the count never
+ shrinks there (the top and bottom layers are where a wrong start is least
+ recoverable, not most).
+
+ Freeing z removes the ladder's quantisation but not the LM's locality, and
+ the extra starts are what would stop a solve settling on the wrong side of
+ a bistatic ellipse. On this fleet's geometry they had almost nothing to
+ stop: over 1019 free-mode solves on test, three starts' rms_delay differed
+ by more than 0.1 µs in 13 of them, and the nearest-layer start was more
+ than 0.5 µs worse than the best in 2 — so the default is one start and the
+ other two are bought explicitly, by a deployment whose geometry shows it
+ needs them. See core/state.py for the numbers and the trade.
+ """
+ if not layers:
+ return []
+ n = max(1, min(int(state.SOLVER_FREE_ALT_STARTS), len(layers)))
+ alt = float(ig_alt_km) if ig_alt_km is not None else 7.0
+ nearest = min(range(len(layers)), key=lambda i: abs(layers[i] - alt))
+ lo = max(0, min(nearest - (n - 1) // 2, len(layers) - n))
+ return layers[lo : lo + n]
+
- If the initial_guess already carries an ADS-B altitude (not one of the fixed
- grid layers), include it in the sweep so the correct exact altitude is tried.
+def _solve_best_altitude(
+ s_in: dict,
+ node_cfgs: dict,
+ solve_fn,
+ multistart_fn=_pool_solve_multistart,
+) -> dict | None:
+ """Altitude for n≥3, by whichever rule state.SOLVER_ALT_MODE names.
+
+ sweep (default): solve once per layer, pick by minimum rms_delay. If the
+ initial_guess already carries an ADS-B altitude (not one of the fixed grid
+ layers), include it in the sweep so the correct exact altitude is tried.
+
+ free: one call to the multi-start helper, which solves altitude as a sixth
+ unknown from _free_alt_starts. The sweep cannot do better than half its
+ 2 km layer spacing, and on noise-free replay of this fleet's geometry that
+ quantisation alone left rms_delay at a 1.76 µs median against the 3.0 µs
+ reject gate — spending most of the gate's budget on an altitude the
+ measurements themselves determine, and provoking _trim_and_resolve to drop
+ nodes that were never the problem. Costs one pool round trip per
+ candidate instead of six.
+
+ The mode is read per call rather than captured at import, so a test (and a
+ live config reload) sees the value it set. Read here and not inside
+ _process_solver_item because _trim_and_resolve re-enters through this same
+ function: a trim must re-solve under the mode its first solve used, or the
+ residuals it is comparing are not the same quantity.
"""
ig_alt = s_in.get("initial_guess", {}).get("alt_km")
if ig_alt is not None and ig_alt not in _SOLVER_ALT_LAYERS_KM:
layers = sorted(set(_SOLVER_ALT_LAYERS_KM + [round(float(ig_alt), 3)]))
else:
layers = _SOLVER_ALT_LAYERS_KM
+ n_meas = len({m.get("node_id") for m in (s_in.get("measurements") or [])})
+ if state.SOLVER_ALT_MODE == "free" and n_meas >= _FREE_ALT_MIN_NODES:
+ # No fall back to the sweep when this returns None: a helper that got
+ # no solve out of its starts is reporting the same thing the sweep
+ # reports when every layer fails, and sweeping anyway would cost the
+ # six round trips this mode exists to avoid on exactly the candidates
+ # that are least likely to repay them.
+ return multistart_fn(s_in, node_cfgs, _free_alt_starts(ig_alt, layers))
return _sweep_altitudes(s_in, node_cfgs, solve_fn, layers, "rms_delay")
@@ -386,6 +556,7 @@ def _trim_and_resolve(
node_cfgs: dict,
solve_fn,
result: dict,
+ multistart_fn=_pool_solve_multistart,
) -> tuple[dict, dict, dict | None]:
"""Drop the worst-residual node(s) and re-solve, down to _TRIM_MIN_NODES.
@@ -395,6 +566,11 @@ def _trim_and_resolve(
so re-solving on the survivors after dropping the offending node recovers
a solve the blanket gate would otherwise discard outright.
+ Re-solves through _solve_best_altitude, so it inherits whichever altitude
+ mode is in force — the loop compares this round's rms against the previous
+ round's, and mixing a swept altitude with a free one would make that
+ comparison meaningless.
+
Returns (final_result, final_s_in, trim_meta). trim_meta is None only
when no round ever produced a successful re-solve — i.e. no trimming was
actually performed — never when trimming ran but rms stayed high (that
@@ -439,7 +615,7 @@ def _trim_and_resolve(
s_next = _filter_s_in_to_nodes(s_in, survivors)
try:
- new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn)
+ new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn, multistart_fn)
except Exception:
logging.exception("Solver trim re-solve failed")
break
@@ -736,6 +912,7 @@ def multinode_key_decision(
max_dist_km: float = _MN_ASSOC_MAX_DIST_KM,
max_age_s: float = _MN_ASSOC_MAX_AGE_S,
learned_vel_fn=track_filter.learned_velocity,
+ anchor_dr: bool = False,
) -> tuple[str, str, float | None]:
"""The keying rule itself, clock-free — the multinode-track analogue of
claim_decision. Extracted so the offline bench measures the SHIPPED rule
@@ -756,11 +933,14 @@ def multinode_key_decision(
Caller holds _MN_TRACKS_LOCK — it reads `tracks` and the caller writes
back into it under the same lock. Returns (key, how, dist_km) with
- how in {"adsb", "anchor", "proximity", "minted"}; dist_km is how far this
- solve landed from the entry it was keyed onto (dead-reckoned, for the
- proximity branch) and None where nothing was matched — "adsb" and
- "minted". The caller stamps both onto the solve-history record, which is
- the only way to tell a re-key apart from a fragment after the fact.
+ how in {"adsb", "anchor", "proximity", "shadowed", "minted"}; dist_km is
+ how far this solve landed from the entry it was keyed onto (dead-reckoned,
+ for the proximity and shadowed branches) and None where nothing was
+ matched — "adsb" and "minted". The caller stamps both onto the
+ solve-history record, which is the only way to tell a re-key apart from a
+ fragment after the fact. "shadowed" is the one verdict that is NOT a key:
+ it names the followed key this solve was refused in favour of, and the
+ caller must treat it as a rejection (see _process_solver_item).
Order:
1. ADS-B-tagged solves key on the transponder hex — unconditional, and
@@ -786,6 +966,18 @@ def multinode_key_decision(
does; candidates compete on distance normalised by their own gate,
so a fresh close entry beats an old far one rather than the scan
simply taking whichever is nearer in kilometres.
+
+ KEY OWNERSHIP (DARK_FOLLOW_MODE=binding only). A key the follow
+ lane published on within dark_follow.DARK_FOLLOW_OWN_S is removed
+ from this scan's candidates entirely, and if the nearest such key is
+ within DARK_FOLLOW_SHADOW_KM the solve is refused ("shadowed")
+ instead of keyed at all. The follow lane already supplies every
+ solve an established track needs, so a bottom-up solve arriving at
+ one of its keys is either a duplicate — competing with the anchored
+ solve and dragging the filter — or a different aircraft stealing the
+ key; 21% of proximity joins measured on test were the latter. See
+ dark_follow.DARK_FOLLOW_OWN_S for why a tighter gate cannot separate
+ the two.
4. Mint. This key only needs to be unique at birth; every later solve
associates to it above (by proximity, or by anchor once a claim
forms), so it stays stable.
@@ -799,21 +991,45 @@ def multinode_key_decision(
return f"mn-adsb-{adsb_hex}", "adsb", None
lat, lon = result["lat"], result["lon"]
+ ts_s = result.get("timestamp_ms", 0) / 1000.0
# The anchor branch keeps the FLAT gate. It is not a dead-reckoning
# question: the claim named this entry as the aircraft this solve is of,
# and the distance check exists only to refuse an anchor whose solve
# converged somewhere else entirely. Nothing here is predicting where the
# anchor drifted to, so there is no drift term to allow for.
+ #
+ # ...unless the caller says otherwise (anchor_dr). A dark-follow input
+ # (services/dark_follow.py) breaks that premise by construction: its guess
+ # IS a prediction of where the anchor drifted to, so its solve is compared
+ # against an entry the follow lane already knows to be stale. The numbers
+ # make it more than a nicety — the dark displacement cap is 6.0 km and the
+ # flat anchor gate is 6.0 km, so a solve at the edge of the gate that let
+ # it through is at the edge of the gate that must key it, before any drift
+ # is added; at the follow lane's 20 s staleness limit a 270 m/s target adds
+ # another 5.4 km of it. Without this the anchor would be refused exactly
+ # when the aircraft is moving fastest, and the solve would fall through to
+ # the proximity scan the whole lane exists to stop relying on. Same DR and
+ # same age-scaled gate as that scan, so "near the anchor" means one thing.
if anchor_key and anchor_key.startswith("mn-dark-") and anchor_key in tracks:
anchor = tracks[anchor_key]
a_lat, a_lon = anchor.get("lat"), anchor.get("lon")
if a_lat is not None and a_lon is not None:
+ a_gate_km = max_dist_km
+ a_dt = ts_s - anchor.get("timestamp_ms", 0) / 1000.0
+ if anchor_dr and 0.0 < a_dt <= max_age_s:
+ a_vel_east, a_vel_north = _entry_dr_velocity(anchor_key, anchor, learned_vel_fn)
+ a_lat, a_lon = offset_latlon_m(
+ a_lat,
+ a_lon,
+ east_m=a_vel_east * a_dt,
+ north_m=a_vel_north * a_dt,
+ )
+ a_gate_km = _mn_assoc_gate_km(a_dt, max_dist_km)
a_dist = _haversine_km(lat, lon, a_lat, a_lon)
- if a_dist <= max_dist_km:
+ if a_dist <= a_gate_km:
return anchor_key, "anchor", a_dist
- ts_s = result.get("timestamp_ms", 0) / 1000.0
# Candidates compete on d / gate_km, not on d: an entry solved 2 s ago at
# 5 km is a worse match than one solved 40 s ago at 8 km only if you
# ignore that the second one's position is a 40 s extrapolation. A score
@@ -823,6 +1039,13 @@ def multinode_key_decision(
best_key: str | None = None
best_score = 1.0
best_dist: float | None = None
+ # Key ownership: the nearest key the follow lane is currently answering
+ # for, and how far this solve landed from it. Only collected for a
+ # bottom-up solve in binding mode — an anchored or ADS-B solve names the
+ # aircraft it is of, and neither branch above reaches this scan.
+ shadow_scan = not anchor_key and dark_follow.mode() == "binding"
+ shadow_key: str | None = None
+ shadow_dist: float | None = None
for key, prev in tracks.items():
# Only dark tracks are claimable; an untagged solve must never steal the
@@ -846,10 +1069,25 @@ def multinode_key_decision(
north_m=vel_north_ms * dt,
)
d = _haversine_km(lat, lon, p_lat, p_lon)
+ # A key the follow lane just published on is not joinable bottom-up,
+ # whatever the distance says — see dark_follow.DARK_FOLLOW_OWN_S for
+ # the measurement. It still competes to SHADOW this solve below, so
+ # the scan has to remember the nearest one rather than skipping it.
+ if shadow_scan and dark_follow.recently_followed(key, ts_s, dark_follow.DARK_FOLLOW_OWN_S):
+ if shadow_dist is None or d < shadow_dist:
+ shadow_key, shadow_dist = key, d
+ continue
score = d / _mn_assoc_gate_km(dt, max_dist_km)
if score < best_score:
best_key, best_score, best_dist = key, score, d
+ # Close enough to a followed key that this solve is the same aircraft the
+ # follow lane is already solving: refuse it outright rather than mint a
+ # second key for a target that already has one. Farther away it falls
+ # through to the non-followed candidates and, failing those, mints — the
+ # one thing it may never do is join the followed key.
+ if shadow_key is not None and shadow_dist <= dark_follow.DARK_FOLLOW_SHADOW_KM:
+ return shadow_key, "shadowed", shadow_dist
if best_key is not None:
return best_key, "proximity", best_dist
# No claimant — a genuinely new target.
@@ -984,6 +1222,34 @@ def _supersession_match(
# Sized against the map, not the association cadence: multinode_tracks expire
# at 60 s, so refreshing an aircraft every 12 s leaves four solves' worth of
# margin. 0 disables the suppression entirely.
+#
+# The claim is recorded ON PUBLICATION, not on admission, and from the
+# POST-TRIM survivors. Claiming on admission made a candidate that never
+# reached the map suppress every later candidate sharing any of its track ids
+# for the full window — including other aircraft's, since tracker track ids
+# are shared across the association candidates of different aircraft (74 of
+# 178 ids in a 6 min live window appeared in solves of more than one
+# ground-truth aircraft; the same finding that forced _supersession_match's
+# spatial guard). A rejected candidate, or a contaminated superset that the
+# gates sank, therefore blacked out the clean subsets behind it for 12 s and
+# nothing was refreshed at all. Live that cost ~1 537 skips per 646 dark
+# attempts per 30 min — more candidates suppressed than solved, by a factor
+# of two. The rule this suppression is FOR is "an aircraft already on the map
+# at this width does not need re-solving yet", and only a publication puts an
+# aircraft on the map.
+#
+# Two consequences, both accepted deliberately:
+# * the check no longer claims under the same lock, so two workers can now
+# both solve duplicates of one aircraft that arrived together. The pair
+# costs one extra solve and is resolved downstream by keying and
+# supersession, which already handle exactly this; the alternative is the
+# starvation above.
+# * trimmed nodes' track ids are NOT claimed (_filter_s_in_to_nodes rebuilds
+# track_ids from the surviving track_ids_by_node, so result's
+# source_track_ids are the survivors). A node dropped for a bad residual
+# was probably another aircraft's — claiming its track would suppress that
+# aircraft's own candidate on the strength of a measurement this solve
+# threw away.
_SOLVER_RESOLVE_INTERVAL_S = float(os.getenv("SOLVER_RESOLVE_INTERVAL_S", "12"))
_RECENT_SOLVES: dict[str, tuple[float, int]] = {} # track_id → (solved_at, n_nodes)
_RECENT_SOLVES_LOCK = threading.Lock()
@@ -1001,39 +1267,96 @@ def _sweep_recent_solves(now_s: float) -> None:
del _RECENT_SOLVES[tid]
-def _claim_resolve_slot(s_in, now_s: float) -> bool:
- """False when this candidate re-solves tracks another candidate just took.
+def _resolve_slot_covered(s_in, now_s: float) -> tuple[bool, list[dict]]:
+ """Is every track this candidate carries already ON THE MAP at this width?
+
+ Pure: it reads the claims and mutates nothing, so a candidate that is
+ admitted here and then rejected by the gate stack leaves no trace. The
+ claim is made afterwards by _record_resolve_slot, from the publish path
+ only — see the block comment above for why, and for what the loss of
+ atomic test-and-claim costs.
- Records the claim as a side effect, under one lock with the test, so two
- workers cannot both admit the same aircraft's duplicates. An input with no
- track provenance (detection-level, or an anchored input carrying none) is
- always admitted — there is nothing to match it against.
+ Returns (covered, blocking). ``blocking`` is the claims that covered it,
+ for the skip record; it is empty whenever ``covered`` is False. An input
+ with no track provenance (detection-level, or an anchored input carrying
+ none) is never covered — there is nothing to match it against.
"""
if _SOLVER_RESOLVE_INTERVAL_S <= 0 or not isinstance(s_in, dict):
- return True
+ return False, []
track_ids = s_in.get("track_ids")
if not track_ids:
- return True
+ return False, []
n_nodes = int(s_in.get("n_nodes") or 0)
cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S
+ blocking: list[dict] = []
with _RECENT_SOLVES_LOCK:
- covered = True
for tid in track_ids:
held = _RECENT_SOLVES.get(tid)
if held is None or held[0] <= cutoff or held[1] < n_nodes:
- covered = False
- break
- if covered:
- return False
+ return False, []
+ blocking.append({"track_id": tid, "held_ts": round(held[0], 3), "held_n": held[1]})
+ return True, blocking
+
+
+def _record_resolve_slot(track_ids, n_nodes: int, now_s: float) -> None:
+ """Record that ``track_ids`` are covered by a PUBLISHED solve at n_nodes.
+
+ Called from the publish path alone, with the post-trim survivors
+ (``result["source_track_ids"]``). Nothing else may call it: a claim is a
+ statement that this aircraft is on the map, and a rejected solve puts
+ nothing there.
+ """
+ if _SOLVER_RESOLVE_INTERVAL_S <= 0 or not track_ids:
+ return
+ n_nodes = int(n_nodes or 0)
+ cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S
+ with _RECENT_SOLVES_LOCK:
for tid in track_ids:
held = _RECENT_SOLVES.get(tid)
- # Keep the widest claim of the window: a narrow candidate admitted
- # after a wide one must not lower the bar the next copy is tested
- # against.
+ # Keep the widest claim of the window: a narrow publish after a
+ # wide one must not lower the bar the next copy is tested against.
held_nodes = held[1] if held is not None and held[0] > cutoff else 0
_RECENT_SOLVES[tid] = (now_s, max(n_nodes, held_nodes))
_sweep_recent_solves(now_s)
- return True
+
+
+def _record_resolve_skip(s_in, now_s: float, blocking: list[dict]) -> None:
+ """Count and remember one resolve-slot refusal.
+
+ The counter alone could not answer the question the suppression rule
+ raises — *whose* claim blocked this, and was it even the same aircraft.
+ Live on the test droplet the rule refuses ~1 537 candidates per 646 dark
+ attempts per 30 min, and nothing recorded which claim did it, so a skip
+ that suppressed a genuinely different aircraft (tracker track ids are
+ shared across candidates — see _supersession_match) was indistinguishable
+ from one that suppressed a duplicate. The deque carries the blocking
+ claims and the candidate's own guess position so the two can be told apart
+ after the fact.
+
+ Deliberately NOT a solve-history record: skips outrun real dark records
+ roughly two to one, and writing them into that deque would evict the
+ solves the same investigation needs (see state.solver_resolve_skips_recent).
+ """
+ s = s_in if isinstance(s_in, dict) else {}
+ track_ids = list(s.get("track_ids") or [])
+ dark = _is_dark_solver_input(s)
+ state.bump_counter("solver_resolve_skips")
+ if dark:
+ state.bump_counter("solver_resolve_skips_dark")
+ ig = s.get("initial_guess") or {}
+ state.solver_resolve_skips_recent.append(
+ {
+ "ts_ms": int(now_s * 1000),
+ # No key is minted for a candidate that never solves, so lane is
+ # the same fallback routes.test._record_lane uses for a reject.
+ "lane": "dark" if dark else "adsb",
+ "track_ids": track_ids,
+ "n_nodes": int(s.get("n_nodes") or 0),
+ "blocking": blocking,
+ "guess_lat": round(float(ig["lat"]), 6) if ig.get("lat") else None,
+ "guess_lon": round(float(ig["lon"]), 6) if ig.get("lon") else None,
+ }
+ )
# Which single-node track pair currently owns a published n=2 track, and how
@@ -1384,6 +1707,59 @@ def _is_dark_solver_input(s_in) -> bool:
return not (hx and is_transponder_hex(hx))
+def _stamp_foreign_nodes(rec: dict) -> None:
+ """Stamp which of a dark record's own nodes could not see the aircraft.
+
+ Cluster contamination is the dark lane's largest known defect — a
+ candidate assembled by format_track_pairs_for_solver can carry a node
+ whose track belongs to a *different* aircraft, and the solver then fits a
+ geometry no single aircraft ever occupied. Offline the audit measured it
+ at ~60 % of dark candidates; this makes the same number live.
+
+ The test is the associator's own visibility predicate applied whole
+ (retina_analytics.association._point_in_beam against the registered
+ NodeGeometry), which is the same gate known-lane claiming uses — claiming
+ and the dark lane must mean the same thing by "this node can see there",
+ and a second bespoke rule here would let the two disagree. Two
+ consequences worth knowing: it is a ground-projected bearing/footprint
+ test with no altitude term, and under FOV_MODE=active it is the learned
+ FOV rather than the theoretical wedge. Both are exactly what the rest of
+ the pipeline believes about coverage, which is the point.
+
+ Position is the matched ground-truth point already stamped on the record
+ (gt_lat/gt_lon at the solve epoch), so this costs no extra trail lookup —
+ only one cone test per contributing node. Nodes trimmed out by
+ _trim_and_resolve are included: a node dropped for a bad residual is
+ precisely the contamination this measures, and leaving it out would hide
+ every case trimming already rescued.
+
+ A node with no registered geometry is not judged either way. When that
+ leaves nothing judgeable the record is left unstamped rather than stamped
+ clean, so contamination_pct never counts an abstention as innocence.
+ """
+ lat, lon = rec.get("gt_lat"), rec.get("gt_lon")
+ if lat is None or lon is None:
+ return
+ node_ids = list(rec.get("contributing_node_ids") or [])
+ node_ids += [nid for nid in (rec.get("trimmed_node_ids") or []) if nid not in node_ids]
+ if not node_ids:
+ return
+ geometries = state.node_associator.node_geometries
+ judged = 0
+ foreign: list[str] = []
+ for nid in node_ids:
+ geo = geometries.get(nid)
+ if geo is None:
+ continue
+ judged += 1
+ if not _point_in_beam(lat, lon, geo):
+ foreign.append(nid)
+ if not judged:
+ return
+ rec["foreign_node_ids"] = foreign
+ rec["contaminated"] = bool(foreign)
+
+
def _record_dark_accuracy_sample(rec: dict) -> None:
"""Offer one published DARK solve to the rolling accuracy store.
@@ -1434,6 +1810,7 @@ def _record_solve_history(
chi2_per_dof: float | None = None,
key_how: str | None = None,
key_dist_km: float | None = None,
+ follow_key: str | None = None,
superseded_keys: list[str] | None = None,
superseded_blocked: int | None = None,
extra: dict | None = None,
@@ -1450,7 +1827,16 @@ def _record_solve_history(
solve — which branch produced solve_key, and how far the solve landed from
the entry it was keyed onto. Only the publish path has run the keying
rule, so both are None on every reject (the key is minted after the
- gates, which is also why solver_hex is None there).
+ gates, which is also why solver_hex is None there) — with one exception:
+ a ``shadowed_by_follow`` reject IS the keying rule's verdict, and carries
+ key_how/key_dist_km plus ``follow_key`` naming the followed key it was
+ refused in favour of.
+
+ ``follow_key`` overrides the input's own follow_key for the record only.
+ The ghost guard below is deliberately NOT fed from it: the guard judges
+ solves the FOLLOW lane produced, and a shadowed record is a bottom-up
+ solve that merely names a followed key — feeding it there would let the
+ bottom-up lane's refusals drop the very track that refused them.
``superseded_keys``/``superseded_blocked`` are the other side of that
decision: which existing entries this publish popped as the same aircraft
@@ -1460,7 +1846,14 @@ def _record_solve_history(
``extra`` merges caller-supplied fields (trim metadata, beam-rejection
diagnostics) into the record. Applied before the GT stamp so it can
- never clobber gt_hex/gt_error_km/gt_lat/gt_lon.
+ never clobber gt_hex/gt_error_km/gt_lat/gt_lon — and so the trimmed node
+ ids it carries are in hand for the contamination stamp below.
+
+ ``foreign_node_ids``/``contaminated`` are stamped on DARK records that
+ matched ground truth: which of this candidate's own nodes could not see
+ the aircraft it was matched to (see _stamp_foreign_nodes). Absent on
+ every other record, which is what /api/test/solver-stats' contamination
+ block counts as "not judged" rather than as clean.
"""
r = result if isinstance(result, dict) else {}
s = s_in if isinstance(s_in, dict) else {}
@@ -1487,6 +1880,16 @@ def _record_solve_history(
# windowed fragmentation breakdown can see what fraction of ALL
# attempts (not just successful ones) were anchor-carrying.
"anchor_key": s.get("anchor_key"),
+ # Lane provenance, carried by the solver input rather than inferred
+ # from the key: a dark-follow solve (services/dark_follow.py) lands on
+ # an mn-dark-* key and would otherwise be indistinguishable from the
+ # bottom-up solves whose funnel it is not part of. guess_source says
+ # what the initial guess WAS — "prediction" for a followed track,
+ # absent for the association grid centroid every other dark input
+ # carries — and follow_key names the track that predicted it.
+ "lane": s.get("lane"),
+ "guess_source": s.get("guess_source"),
+ "follow_key": follow_key or s.get("follow_key"),
"raw_lat": round(float(raw_lat), 6) if raw_lat is not None else None,
"raw_lon": round(float(raw_lon), 6) if raw_lon is not None else None,
"lat": round(float(r["lat"]), 6) if outcome == "published" else None,
@@ -1551,6 +1954,30 @@ def _record_solve_history(
}
if extra:
rec.update(extra)
+ # The INPUT's follow_key, not the record's: a shadowed_by_follow reject
+ # names a followed key it was refused in favour of, and that key's guard
+ # must not hear about a solve the follow lane never made.
+ _follow_key = s.get("follow_key")
+ if _follow_key:
+ # The dark-follow ghost guard (services/dark_follow.py) needs a verdict
+ # for every follow-solve, and this is the one place all of them pass
+ # through — published, every rejected_* gate, unconverged, and the
+ # shadow pass's own record. Following a track is a feedback loop (the
+ # solve keeps the key alive, the key keeps claiming detections), so a
+ # key that stops earning its solves has to be droppable from OUTSIDE
+ # that loop. A shadow record carries its own verdict in follow_ok:
+ # it never reached the gates, so "did it publish" says nothing.
+ dark_follow.record_outcome(_follow_key, bool(rec.get("follow_ok", outcome == "published")))
+ if outcome == "published":
+ state.bump_counter("dark_follow_published")
+ # ...and the lane now owns the key it published on, for
+ # DARK_FOLLOW_OWN_S. Stamped with the MEASUREMENT epoch, because
+ # the reader (multinode_key_decision) compares it against another
+ # solve's measurement epoch and is deliberately clock-free. The
+ # key published on, not the anchor: on the rare anchor fallback
+ # the lane's solve went somewhere else, and that is the entry it
+ # is now refreshing.
+ dark_follow.note_follow_publish(solve_key or _follow_key, rec["measurement_ts_ms"] / 1000.0)
if raw_lat is not None and raw_lon is not None:
meas_ts_s = (rec["measurement_ts_ms"] or now_ms) / 1000.0
rec.update(_gt_for_record(rec["adsb_hex"], float(raw_lat), float(raw_lon), meas_ts_s))
@@ -1574,6 +2001,11 @@ def _record_solve_history(
rec["vel_err_ms"] = round(math.hypot(ve - gt_ve, vn - gt_vn), 1)
else:
rec["vel_err_ms"] = None
+ # Live cluster-contamination metric, dark lane only and only where ground
+ # truth actually matched — without a truth position there is nothing to
+ # ask "could this node see it?" about. See _stamp_foreign_nodes.
+ if _dark and rec.get("gt_hex"):
+ _stamp_foreign_nodes(rec)
if rec["outcome"] == "published" and _dark and rec.get("gt_error_km") is not None:
_record_dark_accuracy_sample(rec)
# Route by lane: the known lane's per-hex-per-pass volume would otherwise
@@ -1693,7 +2125,12 @@ def fov_gate_verdict(fov, n_nodes: int, brg: float, dist_km: float, range_rule_p
return range_rule_pass or fov_pass
-def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus) -> dict | None:
+def _process_solver_item(
+ item: tuple,
+ solve_fn,
+ select_fn=_pool_select_consensus,
+ multistart_fn=_pool_solve_multistart,
+) -> dict | None:
"""Process a single solver queue entry. Returns the solver result (or None).
Extracted from the worker loop so the success/failure/latency bookkeeping
@@ -1704,6 +2141,10 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
initial_guess and _CONSENSUS_MODE != "off" — n=2 (mirror-disambiguation
is the displacement/beam gates' job, not consensus's) and detection-level
inputs (no initial_guess to pin an altitude with) never call it.
+
+ multistart_fn is the free-altitude solve (_pool_solve_multistart by
+ default; tests substitute a stub), reached only when
+ state.SOLVER_ALT_MODE is "free" — see _solve_best_altitude.
"""
s_in, node_cfgs = item[0], item[1]
enqueued_at: float | None = item[2] if len(item) > 2 else None
@@ -1724,10 +2165,19 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
# here rather than at enqueue: the frame path must not carry solver state,
# and a copy that queued before its twin was solved can only be recognised
# once it reaches a worker.
- if not _claim_resolve_slot(s_in, time.time()):
- state.bump_counter("solver_resolve_skips")
+ _now_s = time.time()
+ _covered, _blocking = _resolve_slot_covered(s_in, _now_s)
+ if _covered:
+ _record_resolve_skip(s_in, _now_s, _blocking)
return None
n_nodes = s_in.get("n_nodes", 0) if isinstance(s_in, dict) else 0
+ # Before anything reads a delay: the nodes did not sample simultaneously,
+ # and every gate below (rms_delay first among them) assumes they did. Runs
+ # ahead of consensus and the altitude sweep so both judge the same aligned
+ # numbers the published solve is fitted to.
+ epoch_meta: dict = {"epoch_aligned": False}
+ if state.SOLVER_EPOCH_ALIGN and isinstance(s_in, dict):
+ s_in, epoch_meta = align_measurement_epochs(s_in, node_cfgs)
consensus_meta: dict | None = None
try:
if "initial_guess" not in s_in:
@@ -1736,7 +2186,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
if _CONSENSUS_MODE != "off":
s_in, consensus_meta = _consensus_select(s_in, node_cfgs, select_fn)
n_nodes = s_in.get("n_nodes", n_nodes)
- result = _solve_best_altitude(s_in, node_cfgs, solve_fn)
+ result = _solve_best_altitude(s_in, node_cfgs, solve_fn, multistart_fn)
else:
result = _solve_best_altitude_n2(s_in, node_cfgs, solve_fn)
except Exception:
@@ -1762,7 +2212,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
and (result.get("rms_delay") or 0) > _SOLVER_RMS_DELAY_MAX_US
and result.get("per_node_delay_res_us")
):
- result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result)
+ result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result, multistart_fn)
n_nodes = result.get("n_nodes", n_nodes)
# Built once and threaded through every history record below
@@ -1771,6 +2221,24 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
_extra: dict | None = dict(trim_meta) if trim_meta else {}
if consensus_meta is not None:
_extra["consensus_meta"] = consensus_meta
+ # Always stamped, aligned or not: "this solve was not aligned" is the
+ # fact /api/test/mlat-history needs to separate a residual the
+ # correction could not have helped from one it was applied to.
+ _extra.update(epoch_meta)
+ # How this solve got its altitude, and — in free mode — what each
+ # start altitude fitted to. Stamped on every record, published or
+ # rejected, and in BOTH modes (the sweep's solves report
+ # altitude_mode "pinned"), because the only way to judge SOLVER_ALT_MODE
+ # live is to compare the two lanes' rms_delay and gt_error_km over the
+ # same history buffer. The per-start list is what says whether the
+ # three starts were worth keeping or one would have done.
+ if result.get("altitude_mode"):
+ _extra["altitude_mode"] = result["altitude_mode"]
+ if result.get("rms_by_start") is not None:
+ _extra["alt_starts_km"] = result.get("alt_starts_km")
+ _extra["alt_start_rms_us"] = [None if v is None else round(float(v), 3) for v in result["rms_by_start"]]
+ if result.get("z_saturated"):
+ _extra["z_saturated"] = True
_extra = _extra or None
rms_delay = result.get("rms_delay", 0) or 0
@@ -2179,118 +2647,162 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
# _MN_POS_HISTORY_LOCK (inside the smoother) is never taken in
# reverse anywhere.
_anchor_key = s_in.get("anchor_key") if isinstance(s_in, dict) else None
- key, _key_how, _key_dist_km = multinode_key_decision(state.multinode_tracks, result, _adsb_hex, _anchor_key)
- # Dark-lane key births vs re-keys. The fragmentation question is
- # "how often does one aircraft get a second key", and the only
- # place that is decided is right here — solver_successes counts
- # solves, distinct_keys counts survivors, neither counts the
- # decision. Dark only: the ADS-B lane keys off the transponder
- # hex unconditionally and has no decision to observe. Anchor
- # hits are deliberately in neither counter; solver_anchor_hits
- # already carries them, and double-counting them here would make
- # minted + proximity stop summing to the dark decisions this
- # gate actually made.
- if key.startswith("mn-dark-"):
- if _key_how == "minted":
- state.bump_counter("solver_key_minted_dark")
- elif _key_how == "proximity":
- state.bump_counter("solver_key_proximity_dark")
- if _anchor_key:
- # Only an anchored solver input (top-down claiming, active
- # mode) ever sets s_in["anchor_key"] — this whole block is
- # inert off/shadow, by construction, with no mode read here.
- state.bump_counter("solver_anchored_published")
- state.bump_counter("solver_anchor_hits" if _key_how == "anchor" else "solver_anchor_fallbacks")
- # Raw solve position, before smoothing — the history record keeps
- # both so display-side drift can be separated from solver error.
- _raw_lat, _raw_lon = result["lat"], result["lon"]
- # Multi-epoch averaging cuts single-frame noise by ~√K. Originally
- # n=2-with-ADS-B only — production showed dark targets (where MLAT
- # is the only position source) were the one population left raw.
- # Smoothing now runs through the env-gated KF in
- # services/track_filter.py, with this module's EWMA kept as the
- # TRACK_SMOOTHER=ewma fallback.
- result = track_filter.smooth_solve(result, key, _adsb_hex, ewma_fn=_ewma_smooth_track)
- prev = state.multinode_tracks.get(key)
- if prev:
- # Latch: a tracker flag raised on an earlier solve holds for
- # the multinode track's lifetime (≤60 s expiry) even if the
- # contributing track has since despawned or gone quiet.
- result["is_anomalous"] = bool(result.get("is_anomalous")) or bool(prev.get("is_anomalous"))
- result["anomaly_types"] = sorted(
- set(result.get("anomaly_types", [])) | set(prev.get("anomaly_types", []))
- )
- # Source-track identity: the single-node track ids this solve was
- # built from. Used below for supersession and carried into the
- # history record so a bad map marker can be traced to its inputs.
- result["source_track_ids"] = sorted(s_in.get("track_ids") or []) if isinstance(s_in, dict) else []
-
- # Supersession: an earlier entry that is THIS aircraft, under a
- # key the proximity match (multinode_key_decision) missed, is
- # replaced now rather than left rendering beside the new one for
- # up to 60 s. solve_count carries forward so the re-solved
- # aircraft does not fall back under the n=2 gate below.
- #
- # A shared source track id is the cheap filter, not the rule. The
- # premise this block used to carry — "one aircraft is one set of
- # source tracks" — is false: single-node tracker tracks are shared
- # between the association candidates of DIFFERENT aircraft (74 of
- # 178 track ids in a 6 min live window appeared in published solves
- # of more than one ground-truth aircraft), so popping on the shared
- # id alone destroyed a live neighbour's key 36 times in 44
- # supersessions — 41 of them beyond the association gate, 43 under
- # 15 s old — and the victim's next solve minted a fresh key (dark
- # keys churning at 7.4/min with a 7 s median lifetime). Now
- # _supersession_match has to agree: the old entry dead-reckons
- # into the gate, or its inputs are a subset of this solve's.
- # Replayed over the same solves that cuts mints 47 -> 22 and
- # cross-aircraft pops 36 -> 7. Refusals are counted
- # (mn_superseded_blocked), not
- # silent — the shared-id signal is mostly contamination and the
- # panel has to be able to see that.
- #
- # Unchanged by anchor honoring: `old_key == key: continue` below
- # already protects an anchor from superseding itself, and a
- # proximity-minted fragment built from exactly the anchor's source
- # tracks merging INTO the anchor (old_key != key, key ==
- # anchor_key) is the identical-inputs branch (b) of the predicate —
- # exactly the fragmentation-collapse this whole feature exists for.
- max_superseded_count = 0
- _superseded_keys: list[str] = []
- _superseded_blocked = 0
- if result["source_track_ids"]:
- new_ids = set(result["source_track_ids"])
- _ts_ms = result.get("timestamp_ms") or 0
- for old_key, old_r in list(state.multinode_tracks.items()):
- if old_key == key:
- continue
- if not new_ids.intersection(old_r.get("source_track_ids") or ()):
- continue
- matched, _ = _supersession_match(old_key, old_r, new_ids, _raw_lat, _raw_lon, _ts_ms)
- if not matched:
- _superseded_blocked += 1
- state.bump_counter("mn_superseded_blocked")
- continue
- state.multinode_tracks.pop(old_key, None)
- with state.anomaly_lock:
- state.anomaly_hexes.discard(multinode_hex_from_key(old_key))
- with _MN_POS_HISTORY_LOCK:
- _MN_POS_HISTORY.pop(old_key, None)
- track_filter.drop_key(old_key)
- max_superseded_count = max(max_superseded_count, old_r.get("solve_count", 0))
- _superseded_keys.append(old_key)
- state.bump_counter("mn_superseded")
-
- result["solve_count"] = max(prev.get("solve_count", 0) if prev else 0, max_superseded_count) + 1
- state.multinode_tracks[key] = result
- if trim_meta:
- state.bump_counter("solver_trimmed")
+ key, _key_how, _key_dist_km = multinode_key_decision(
+ state.multinode_tracks,
+ result,
+ _adsb_hex,
+ _anchor_key,
+ # Only the follow lane's anchor is a stale position by
+ # construction — see the anchor branch for why that changes
+ # the distance check it must be judged by.
+ anchor_dr=bool(isinstance(s_in, dict) and s_in.get("follow_key")),
+ )
+ # Key ownership (DARK_FOLLOW_MODE=binding). A bottom-up solve
+ # that landed on a key the follow lane is answering for is not
+ # keyed at all: it is either a duplicate of the anchored solve
+ # already refreshing that key or a different aircraft about to
+ # steal it, and both drag the entry and its filter. The verdict
+ # is taken here, under the same lock as the decision, so nothing
+ # about the entry can change between deciding and refusing; the
+ # record and the counter are emitted outside it, as every other
+ # outcome's are.
+ _shadow_key = key if _key_how == "shadowed" else None
+ if _shadow_key is None:
+ # Dark-lane key births vs re-keys. The fragmentation question is
+ # "how often does one aircraft get a second key", and the only
+ # place that is decided is right here — solver_successes counts
+ # solves, distinct_keys counts survivors, neither counts the
+ # decision. Dark only: the ADS-B lane keys off the transponder
+ # hex unconditionally and has no decision to observe. Anchor
+ # hits are deliberately in neither counter; solver_anchor_hits
+ # already carries them, and double-counting them here would make
+ # minted + proximity stop summing to the dark decisions this
+ # gate actually made.
+ if key.startswith("mn-dark-"):
+ if _key_how == "minted":
+ state.bump_counter("solver_key_minted_dark")
+ elif _key_how == "proximity":
+ state.bump_counter("solver_key_proximity_dark")
+ if _anchor_key:
+ # s_in["anchor_key"] is set by exactly two producers: top-down
+ # claiming in active mode, and a dark-follow input in binding
+ # mode (services/dark_follow.py). Both are off by default, so
+ # this block stays inert by construction with no mode read
+ # here. The counters do not split the two, deliberately: they
+ # measure the same thing either way — how often an anchor
+ # named the track the solve actually landed on — and
+ # follow_key on the history record separates them after the
+ # fact for anyone who needs it.
+ state.bump_counter("solver_anchored_published")
+ state.bump_counter("solver_anchor_hits" if _key_how == "anchor" else "solver_anchor_fallbacks")
+ # Raw solve position, before smoothing — the history record keeps
+ # both so display-side drift can be separated from solver error.
+ _raw_lat, _raw_lon = result["lat"], result["lon"]
+ # Multi-epoch averaging cuts single-frame noise by ~√K. Originally
+ # n=2-with-ADS-B only — production showed dark targets (where MLAT
+ # is the only position source) were the one population left raw.
+ # Smoothing now runs through the env-gated KF in
+ # services/track_filter.py, with this module's EWMA kept as the
+ # TRACK_SMOOTHER=ewma fallback.
+ result = track_filter.smooth_solve(result, key, _adsb_hex, ewma_fn=_ewma_smooth_track)
+ prev = state.multinode_tracks.get(key)
+ if prev:
+ # Latch: a tracker flag raised on an earlier solve holds for
+ # the multinode track's lifetime (≤60 s expiry) even if the
+ # contributing track has since despawned or gone quiet.
+ result["is_anomalous"] = bool(result.get("is_anomalous")) or bool(prev.get("is_anomalous"))
+ result["anomaly_types"] = sorted(
+ set(result.get("anomaly_types", [])) | set(prev.get("anomaly_types", []))
+ )
+ # Source-track identity: the single-node track ids this solve was
+ # built from. Used below for supersession and carried into the
+ # history record so a bad map marker can be traced to its inputs.
+ result["source_track_ids"] = sorted(s_in.get("track_ids") or []) if isinstance(s_in, dict) else []
+
+ # Supersession: an earlier entry that is THIS aircraft, under a
+ # key the proximity match (multinode_key_decision) missed, is
+ # replaced now rather than left rendering beside the new one for
+ # up to 60 s. solve_count carries forward so the re-solved
+ # aircraft does not fall back under the n=2 gate below.
+ #
+ # A shared source track id is the cheap filter, not the rule. The
+ # premise this block used to carry — "one aircraft is one set of
+ # source tracks" — is false: single-node tracker tracks are shared
+ # between the association candidates of DIFFERENT aircraft (74 of
+ # 178 track ids in a 6 min live window appeared in published solves
+ # of more than one ground-truth aircraft), so popping on the shared
+ # id alone destroyed a live neighbour's key 36 times in 44
+ # supersessions — 41 of them beyond the association gate, 43 under
+ # 15 s old — and the victim's next solve minted a fresh key (dark
+ # keys churning at 7.4/min with a 7 s median lifetime). Now
+ # _supersession_match has to agree: the old entry dead-reckons
+ # into the gate, or its inputs are a subset of this solve's.
+ # Replayed over the same solves that cuts mints 47 -> 22 and
+ # cross-aircraft pops 36 -> 7. Refusals are counted
+ # (mn_superseded_blocked), not
+ # silent — the shared-id signal is mostly contamination and the
+ # panel has to be able to see that.
+ #
+ # Unchanged by anchor honoring: `old_key == key: continue` below
+ # already protects an anchor from superseding itself, and a
+ # proximity-minted fragment built from exactly the anchor's source
+ # tracks merging INTO the anchor (old_key != key, key ==
+ # anchor_key) is the identical-inputs branch (b) of the predicate —
+ # exactly the fragmentation-collapse this whole feature exists for.
+ max_superseded_count = 0
+ _superseded_keys: list[str] = []
+ _superseded_blocked = 0
+ if result["source_track_ids"]:
+ new_ids = set(result["source_track_ids"])
+ _ts_ms = result.get("timestamp_ms") or 0
+ for old_key, old_r in list(state.multinode_tracks.items()):
+ if old_key == key:
+ continue
+ if not new_ids.intersection(old_r.get("source_track_ids") or ()):
+ continue
+ matched, _ = _supersession_match(old_key, old_r, new_ids, _raw_lat, _raw_lon, _ts_ms)
+ if not matched:
+ _superseded_blocked += 1
+ state.bump_counter("mn_superseded_blocked")
+ continue
+ state.multinode_tracks.pop(old_key, None)
+ with state.anomaly_lock:
+ state.anomaly_hexes.discard(multinode_hex_from_key(old_key))
+ with _MN_POS_HISTORY_LOCK:
+ _MN_POS_HISTORY.pop(old_key, None)
+ track_filter.drop_key(old_key)
+ max_superseded_count = max(max_superseded_count, old_r.get("solve_count", 0))
+ _superseded_keys.append(old_key)
+ state.bump_counter("mn_superseded")
+
+ result["solve_count"] = max(prev.get("solve_count", 0) if prev else 0, max_superseded_count) + 1
+ state.multinode_tracks[key] = result
+ if trim_meta:
+ state.bump_counter("solver_trimmed")
+ if _shadow_key is not None:
+ state.bump_counter("dark_bottomup_shadowed")
+ _record_solve_history(
+ "shadowed_by_follow",
+ s_in,
+ result,
+ follow_key=_shadow_key,
+ key_how=_key_how,
+ key_dist_km=_key_dist_km,
+ extra=_extra,
+ )
+ return result
# Append a snapshot to the track-archive buffer for Parquet persistence.
# solve_ts_ms records when the solve completed (server wallclock) so
# analysts can measure end-to-end latency vs. result["timestamp_ms"].
archive_record = dict(result)
archive_record["solve_ts_ms"] = int(time.time() * 1000)
state.track_archive_buffer.append(archive_record)
+ # The re-solve claim, taken here and nowhere else: this aircraft is now
+ # on the map at this width, which is the only thing that makes a
+ # duplicate not worth solving. Survivors only — source_track_ids is
+ # rebuilt from the post-trim node set. Outside _MN_TRACKS_LOCK on
+ # purpose, so _RECENT_SOLVES_LOCK is never nested inside it.
+ _record_resolve_slot(result.get("source_track_ids"), result.get("n_nodes"), time.time())
_record_solve_history(
"published",
s_in,
@@ -2370,11 +2882,12 @@ def _run_solver_worker():
# coverage collector lock hard enough to stall test_mlat_history's
# trail-race stress test (~50 daemons caught inside the mode check in a
# single py-spy snapshot).
- known_lane_armed = known_lane._mode() != "off"
+ known_lane_armed = known_lane.lanes_armed()
while True:
_solver_worker_iteration()
if known_lane_armed:
- # Known-lane pass (identity-first claims → per-hex solves).
+ # Known-lane pass (identity-first claims → per-hex solves), plus
+ # the dark-follow pass behind the same lock and interval.
# Ridden on the worker loop rather than its own thread so the
# solve compute stays on the threads that already own the solver
# locks and pool; interval- and concurrency-gated inside, and it
diff --git a/backend/services/tcp_handler.py b/backend/services/tcp_handler.py
index bbe82197..6b21be77 100644
--- a/backend/services/tcp_handler.py
+++ b/backend/services/tcp_handler.py
@@ -529,6 +529,12 @@ def _enqueue_detection(msg: dict, node_id: str | None):
if node_id:
last = _per_node_last_enqueue.get(node_id, 0.0)
if (now_m - last) < _NODE_MIN_INTERVAL_S:
+ # Counted, not silent: this is the only place a node's detections
+ # are discarded on purpose, and until now nothing said how many.
+ # state.frames_dropped is the queue-saturation counter and reads
+ # zero throughout, so "the tracker sees every frame this node
+ # sent" looked true from every published metric.
+ state.bump_counter("node_frames_rate_limited")
return # position already updated; skip expensive queue work
_per_node_last_enqueue[node_id] = now_m
diff --git a/backend/services/track_filter.py b/backend/services/track_filter.py
index 22accba3..aa705ea5 100644
--- a/backend/services/track_filter.py
+++ b/backend/services/track_filter.py
@@ -270,12 +270,21 @@ def learned_velocity(track_key: str) -> tuple[float, float, float, float] | None
filter. _KF_LOCK is a leaf lock (see module docstring), so taking it here
keeps the established solver.py -> track_filter order; callers must not
hold it already.
+
+ The sqrt is clamped for the same reason _smooth_kf's kf_pos_sigma_m one
+ is: this is a read-only accessor on a hot display path with two callers
+ that each lose real work when it throws — solver.py's
+ multinode_key_decision (drops that solve) and aircraft_feed's
+ multinode_to_aircraft (drops the whole broadcast) — so a pathological
+ filter state must degrade to "sigma 0", never to a ValueError. An
+ accessor is the wrong place to discover a covariance is sick; the
+ invariant is enforced upstream in _measurement_R and _kf_correct.
"""
with _KF_LOCK:
entry = _KF_TRACKS.get(track_key)
if entry is None:
return None
- vel_sigma = math.sqrt(0.5 * (entry.P[1, 1] + entry.P[3, 3]))
+ vel_sigma = math.sqrt(max(0.0, 0.5 * (entry.P[1, 1] + entry.P[3, 3])))
return float(entry.x[1]), float(entry.x[3]), float(vel_sigma), float(entry.last_ts_s)
@@ -391,9 +400,13 @@ def _measurement_R(result: dict) -> np.ndarray:
not one scaled version of the other, so they add rather than one
replacing the other:
- cov present and sane: R = (_KF_R_INFLATE**2) * cov_m2 + base
+ cov present and PSD: R = (_KF_R_INFLATE**2) * cov_m2 + base
cov absent/degenerate: R = base (cov=0 limit)
+ "Degenerate" now includes a cov that is not positive-semidefinite, which
+ an ill-conditioned solve really does produce — see the determinant check
+ in the body for why that has to be rejected rather than passed through.
+
where base = diag(_KF_DEFAULT_POS_SIGMA_M**2, _KF_DEFAULT_POS_SIGMA_M**2)
is added UNCONDITIONALLY — the no-cov fallback is not a separate branch,
it is exactly this same formula evaluated at cov_m2 = 0. This is what
@@ -413,8 +426,37 @@ def _measurement_R(result: dict) -> np.ndarray:
cov = result.get("cov_en_km2")
if cov is not None:
arr = np.asarray(cov, dtype=float) * 1e6 # km^2 -> m^2
- if arr.shape == (2, 2) and np.all(np.isfinite(arr)) and arr[0, 0] > 0 and arr[1, 1] > 0:
- r = (_KF_R_INFLATE**2) * arr + base # independent noise sources -> variances add
+ if arr.shape == (2, 2) and np.all(np.isfinite(arr)):
+ arr = (arr + arr.T) / 2.0
+ # PSD, not just positive-diagonal. A 2x2 symmetric matrix is PSD
+ # iff both diagonals are >= 0 AND the determinant is >= 0; the
+ # determinant is the half that was missing, and it is not a
+ # theoretical gap. cov_en_km2 is the top-left 2x2 block of
+ # s2 * inv(JtJ) for the solver's 5-state fit, and the solver
+ # falls back to pinv only on an outright LinAlgError — an
+ # ill-conditioned-but-not-singular JtJ (near-parallel baselines,
+ # the same degenerate tail that puts the formal sigma's p99 at
+ # 3.8e6 km) inverts to numerical garbage that is INDEFINITE while
+ # still having both diagonals positive, so it passed both this
+ # check and the solver's own.
+ #
+ # An indefinite R is not survivable downstream. _kf_correct's
+ # Joseph form preserves positive-semidefiniteness for any gain,
+ # but only GIVEN PSD P and R — its K R K^T term inherits R's
+ # negative eigenvalue directly — and _init_entry seeds P's
+ # position block from this matrix, so a sick R poisons the filter
+ # at birth as well as on every update. Once P's velocity
+ # diagonals go negative, learned_velocity's sqrt raises: on the
+ # test droplet that was 91 tracebacks in 40 minutes, each one
+ # costing either a solve or an entire feed broadcast.
+ #
+ # Rejecting is the honest response rather than repairing by
+ # eigenvalue clipping: a covariance this degenerate carries no
+ # trustworthy relative weighting to preserve, and "R = base" is
+ # already this function's documented answer for a degenerate cov.
+ det = arr[0, 0] * arr[1, 1] - arr[0, 1] * arr[1, 0]
+ if arr[0, 0] > 0 and arr[1, 1] > 0 and det >= 0:
+ r = (_KF_R_INFLATE**2) * arr + base # independent noise sources -> variances add
s = math.sqrt(0.5 * (r[0, 0] + r[1, 1]))
if s < _KF_MIN_POS_SIGMA_M:
@@ -510,6 +552,10 @@ def _smooth_kf(result: dict, track_key: str, adsb_hex: str | None) -> dict:
f, q = _f_q(dt)
x_pred = f @ entry.x
p_pred = f @ entry.P @ f.T + q
+ # Symmetrize here too, not only in _kf_correct: F P F^T is symmetric
+ # in exact arithmetic but drifts by roundoff, and this filter composes
+ # predict and update thousands of times over a track's life.
+ p_pred = (p_pred + p_pred.T) / 2.0
z_e, z_n = _enu_offset_m(entry.ref_lat, entry.ref_lon, r_lat, r_lon)
z = np.array([z_e, z_n])
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index 9aa250c7..45d8f82a 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -155,6 +155,7 @@ def _reset_module_state():
from services import (
aircraft_feed,
alerting,
+ dark_follow,
feed_helpers,
frame_processor,
known_claiming,
@@ -177,6 +178,7 @@ def _reset_module_state():
alerting,
tcp_handler,
known_claiming,
+ dark_follow,
node_bias,
publication,
periodic,
diff --git a/backend/tests/test_adsb_seed_backend.py b/backend/tests/test_adsb_seed_backend.py
index cba3bb36..77b3fc86 100644
--- a/backend/tests/test_adsb_seed_backend.py
+++ b/backend/tests/test_adsb_seed_backend.py
@@ -605,3 +605,32 @@ def test_associator_gets_the_state_world_resolver(self):
must consult the same resolver claiming and the auto-tag filter use,
or one consumer accepts what another rejects."""
assert state.node_associator.node_world_provider is state.node_world
+
+ def test_a_sim_and_a_real_node_over_one_footprint_get_no_overlap_zone(self):
+ """The same resolver, one level down: bottom-up pairing must not build
+ a grid across worlds either. Registering a synthetic node and a
+ hardware node on overlapping coverage used to leave a zone whose only
+ possible pairing was a simulated echo against a real one — which is how
+ real node ids reached the synthetic fleet's dark solves."""
+ _a = state.node_associator
+ try:
+ _a.register_node("synth-GVL-9001", dict(_NODE_CFG))
+ _a.register_node("hw-9001", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36))
+
+ assert _a.overlap_zones == {}
+ assert _a._neighbors.get("synth-GVL-9001", set()) == set()
+ assert _a.assoc_world_skipped_pairs == 1
+ finally:
+ state._reset_for_tests()
+
+ def test_two_synthetic_nodes_over_one_footprint_still_pair(self):
+ """The gate is the world difference, not the registration."""
+ _a = state.node_associator
+ try:
+ _a.register_node("synth-GVL-9001", dict(_NODE_CFG))
+ _a.register_node("synth-GVL-9002", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36))
+
+ assert _a.overlap_zones
+ assert _a.assoc_world_skipped_pairs == 0
+ finally:
+ state._reset_for_tests()
diff --git a/backend/tests/test_analytics_routes.py b/backend/tests/test_analytics_routes.py
index 22a821e0..2b912d03 100644
--- a/backend/tests/test_analytics_routes.py
+++ b/backend/tests/test_analytics_routes.py
@@ -129,6 +129,47 @@ def test_status_returns_expected_fields(self, client):
assert "overlap_zones" in body
assert "overlaps" in body
+ def test_track_pairs_block_reports_the_live_counters(self, client):
+ """superseded and cluster_splits belong to the LIVE block, not the
+ inline-only one.
+
+ They used to be structurally zero in production because the only
+ exclusivity stage ran on a chi2 nothing computes with cv_fit=None.
+ The deferred path now prunes on implied-velocity conflict and splits
+ clusters that hold two tracks of one node, so both counters move on a
+ live fleet and reading them as "inline only" would be wrong.
+ """
+ _a = state.node_associator
+ _a.track_pairs_superseded += 5
+ _a.cluster_splits += 3
+ try:
+ body = client.get("/api/radar/association/status").json()
+ assert body["track_pairs"].keys() == {
+ "gated",
+ "unfitted",
+ "deferred",
+ "superseded",
+ "cluster_splits",
+ }
+ assert body["track_pairs"]["superseded"] == 5
+ assert body["track_pairs"]["cluster_splits"] == 3
+ assert body["track_pairs_inline_only"].keys() == {"accepted", "rejected"}
+ finally:
+ _a.track_pairs_superseded -= 5
+ _a.cluster_splits -= 3
+
+ def test_status_reports_world_skipped_pairs(self, client):
+ """The world gate on overlap zones is otherwise invisible: a fleet
+ whose sim/real pairs are being refused looks exactly like a fleet whose
+ pairs never overlapped, and only this counter separates them."""
+ _a = state.node_associator
+ _a.assoc_world_skipped_pairs += 7
+ try:
+ body = client.get("/api/radar/association/status").json()
+ assert body["assoc_world_skipped_pairs"] == 7
+ finally:
+ state._reset_for_tests()
+
def test_status_includes_claiming_block(self, client):
"""Top-down claiming (ASSOC_CLAIM_MODE) since boot — off by default
in tests, so this pins the shape rather than any particular mode."""
diff --git a/backend/tests/test_dark_follow.py b/backend/tests/test_dark_follow.py
new file mode 100644
index 00000000..5d1b5d68
--- /dev/null
+++ b/backend/tests/test_dark_follow.py
@@ -0,0 +1,845 @@
+"""Dark track following (DARK_FOLLOW_MODE) — services/dark_follow.py,
+known_claiming's path 3, known_lane's follow pass, and the solver's anchor
+dead-reckoning.
+
+The lane is the known lane's shape applied to aircraft with no transponder:
+an established mn-dark-* track's Kalman state stands in for a dead-reckoned
+ADS-B fix, and the solve it produces carries the followed key as its anchor so
+it lands back on the same track. Pinned here:
+
+- pseudo-state eligibility (age, solve count, node count, filter state, and
+ the velocity-sigma ceiling that also drops the key);
+- claiming: a matching detection is claimed and a non-matching one is not, an
+ ADS-B state always beats a dark pseudo-state for the same detection, and
+ ``off`` claims nothing;
+- the follow pass: one queue item per key with the anchor, the prediction as
+ initial guess and no track provenance; the per-key rate limit; shadow
+ records without publishing and binding reaches the queue;
+- binding mode removing the claimed detections from the frame the dark lane
+ sees (frame_processor);
+- the ghost guard: two rejected follow-solves drop the key for the cooldown,
+ including when the verdicts arrive through _record_solve_history;
+- key ownership in binding mode: a bottom-up solve may not join a key the lane
+ just published on, is refused outright inside DARK_FOLLOW_SHADOW_KM, and
+ those refusals are invisible to the guard.
+
+Style follows test_known_claiming.py (registered associator geometry, frames
+built around a real predicted observation) and test_solver_anchor.py.
+"""
+
+import time
+
+import pytest
+from retina_analytics.association import predict_observation
+
+from config.constants import FT_TO_M
+from core import state
+from pipeline.passive_radar import DEFAULT_NODE_CONFIG, PassiveRadarPipeline
+from services import dark_follow, track_filter
+from services import known_claiming as kc
+from services.frame_processor import process_one_frame
+from services.geo import offset_latlon_m
+from services.tasks import known_lane
+from services.tasks import solver as solver_mod
+
+_NODE_CFG = {
+ "rx_lat": 34.85,
+ "rx_lon": -82.40,
+ "rx_alt_ft": 1000,
+ "tx_lat": 34.9412,
+ "tx_lon": -82.4103,
+ "tx_alt_ft": 2000,
+ "fc_hz": 183e6,
+ "beam_width_deg": 90,
+ "max_range_km": 60,
+ "beam_azimuth_deg": 45.0,
+}
+
+_NODE_ID = "test-dark-follow"
+_KEY = "mn-dark-followed"
+
+# Well inside _NODE_CFG's beam — the same corner of Greenville the claiming
+# tests use, so the visibility gate is never the thing under test.
+_LAT, _LON, _ALT_M = 34.88, -82.35, 7000.0
+
+
+@pytest.fixture(autouse=True)
+def _lane_off_unless_armed(monkeypatch):
+ """Leave the live flag off between tests.
+
+ state.DARK_FOLLOW_MODE defaults to "shadow", and every solver worker daemon
+ a TestClient lifespan has leaked into this process polls it — arming it
+ globally would let one race these tests for the per-key rate limit, the
+ same hazard test_known_lane.py documents for KNOWN_LANE_MODE. Tests that
+ need the lane arm it themselves (``_install``) or pass ``mode`` explicitly.
+ """
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "off")
+
+
+def _register(node_id=_NODE_ID):
+ state.node_associator.register_node(node_id, _NODE_CFG)
+ return state.node_associator.node_geometries[node_id]
+
+
+def _frame(ts_ms, delays, dopplers, adsb=None):
+ f = {
+ "timestamp": ts_ms,
+ "delay": list(delays),
+ "doppler": list(dopplers),
+ "snr": [20.0] * len(delays),
+ }
+ if adsb is not None:
+ f["adsb"] = adsb
+ return f
+
+
+def _track(ts_ms, **overrides) -> dict:
+ """A live, followable mn-dark-* entry in state.multinode_tracks."""
+ rec = {
+ "lat": _LAT,
+ "lon": _LON,
+ "alt_m": _ALT_M,
+ "vel_east": 0.0,
+ "vel_north": 0.0,
+ "timestamp_ms": ts_ms,
+ "n_nodes": 3,
+ "solve_count": 5,
+ "contributing_node_ids": [_NODE_ID],
+ }
+ rec.update(overrides)
+ return rec
+
+
+def _kf(monkeypatch, vel_east=0.0, vel_north=0.0, vel_sigma=5.0, keys=(_KEY,)):
+ """Fake filter state for ``keys`` — the real KF is fed by the publish path,
+ which none of these tests go through."""
+ lookup = {k: (vel_east, vel_north, vel_sigma, 0.0) for k in keys}
+ monkeypatch.setattr(track_filter, "learned_velocity", lookup.get)
+
+
+def _install(monkeypatch, ts_ms, mode="shadow", key=_KEY, kf=True, **overrides):
+ """Register the node, install one followable track, arm the lane."""
+ geo = _register()
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", mode)
+ state.multinode_tracks[key] = _track(ts_ms, **overrides)
+ if kf:
+ _kf(monkeypatch, keys=(key,))
+ else:
+ monkeypatch.setattr(track_filter, "learned_velocity", lambda _k: None)
+ # The target list is TTL-cached; a test that rewrites multinode_tracks has
+ # to invalidate it or it reads the previous assertion's world.
+ dark_follow._reset_for_tests()
+ return geo
+
+
+def _pred(geo, lat=_LAT, lon=_LON, alt_m=_ALT_M, ve=0.0, vn=0.0):
+ return predict_observation(geo, lat, lon, alt_m / 1000.0, ve, vn)
+
+
+def _follow_claim(node_id, ts_ms, delay_us, doppler_hz, lat=_LAT, lon=_LON):
+ return {
+ "node_id": node_id,
+ "delay_us": delay_us,
+ "doppler_hz": doppler_hz,
+ "pred_delay_us": delay_us,
+ "pred_doppler_hz": doppler_hz,
+ "ts_ms": ts_ms,
+ "dark_follow": True,
+ "follow_fix": {
+ "lat": lat,
+ "lon": lon,
+ "alt_km": _ALT_M / 1000.0,
+ "vel_east": 0.0,
+ "vel_north": 0.0,
+ "fix_ts_ms": ts_ms,
+ },
+ "contested": False,
+ }
+
+
+def _install_follow_claims(node_ids, ts_ms, key=_KEY):
+ from collections import deque
+
+ dq = state.known_claims.setdefault(key, deque(maxlen=state.KNOWN_CLAIMS_PER_HEX_MAX))
+ for i, nid in enumerate(node_ids):
+ dq.append(_follow_claim(nid, ts_ms, 100.0 + i, 10.0 + i))
+ return dq
+
+
+def _reject_result():
+ """A minimal converged solver result, enough for a history record."""
+ return {
+ "success": True,
+ "lat": _LAT,
+ "lon": _LON,
+ "alt_m": _ALT_M,
+ "n_nodes": 3,
+ "timestamp_ms": int(time.time() * 1000),
+ "contributing_node_ids": ["n1", "n2", "n3"],
+ }
+
+
+def _drain_queue():
+ items = []
+ while True:
+ try:
+ items.append(state.solver_queue.get_nowait())
+ except Exception:
+ return items
+
+
+class TestPseudoStates:
+ """Which dark tracks may be followed at all."""
+
+ def test_an_established_track_is_a_target(self, monkeypatch):
+ ts = int(time.time() * 1000) - 2000
+ _install(monkeypatch, ts)
+
+ (t,) = dark_follow.follow_targets()
+
+ assert t["key"] == _KEY
+ assert t["lat"] == _LAT
+ assert t["pos_sigma_m"] == dark_follow._DEFAULT_POS_SIGMA_M
+ assert state.dark_follow_targets == 1
+
+ def test_a_stale_track_is_not(self, monkeypatch):
+ ts = int((time.time() - dark_follow.DARK_FOLLOW_MAX_AGE_S - 5) * 1000)
+ _install(monkeypatch, ts)
+
+ assert dark_follow.follow_targets() == []
+
+ def test_too_few_solves_is_not(self, monkeypatch):
+ ts = int(time.time() * 1000) - 2000
+ _install(monkeypatch, ts, solve_count=dark_follow.DARK_FOLLOW_MIN_SOLVES - 1)
+
+ assert dark_follow.follow_targets() == []
+
+ def test_too_few_nodes_is_not(self, monkeypatch):
+ ts = int(time.time() * 1000) - 2000
+ _install(monkeypatch, ts, n_nodes=2)
+
+ assert dark_follow.follow_targets() == []
+
+ def test_no_filter_state_is_not(self, monkeypatch):
+ """The velocity and its sigma ARE the prediction; without them there is
+ nothing to dead-reckon with and no honest way to widen a gate."""
+ ts = int(time.time() * 1000) - 2000
+ _install(monkeypatch, ts, kf=False)
+
+ assert dark_follow.follow_targets() == []
+
+ def test_an_adsb_key_is_never_followed(self, monkeypatch):
+ ts = int(time.time() * 1000) - 2000
+ _install(monkeypatch, ts, key="mn-adsb-abc123")
+
+ assert dark_follow.follow_targets() == []
+
+ def test_a_noisy_velocity_drops_the_key(self, monkeypatch):
+ ts = int(time.time() * 1000) - 2000
+ _install(monkeypatch, ts)
+ _kf(monkeypatch, vel_sigma=dark_follow.DARK_FOLLOW_MAX_VEL_SIGMA_MS + 1.0)
+ dark_follow._reset_for_tests()
+
+ assert dark_follow.follow_targets() == []
+ assert state.dark_follow_dropped == 1
+
+ def test_off_mode_has_no_targets(self, monkeypatch):
+ ts = int(time.time() * 1000) - 2000
+ _install(monkeypatch, ts, mode="off")
+
+ assert dark_follow.follow_targets() == []
+
+
+class TestClaiming:
+ """Path 3 of the claiming stage: leftover detections vs pseudo-states."""
+
+ def test_a_matching_detection_is_claimed_and_a_stray_is_not(self, monkeypatch):
+ ts = int(time.time() * 1000)
+ geo = _install(monkeypatch, ts - 2000)
+ pd, pf = _pred(geo)
+ followed: set[int] = set()
+
+ claimed = kc.claim_known_targets(
+ _NODE_ID,
+ _frame(ts, [pd, pd + 500.0], [pf, pf + 500.0]),
+ follow_claimed=followed,
+ )
+
+ assert claimed == set()
+ assert followed == {0}
+ assert state.dark_follow_claims == 1
+ (c,) = list(state.known_claims[_KEY])
+ assert c["dark_follow"] is True
+ assert c["node_id"] == _NODE_ID
+ assert c["delay_us"] == pytest.approx(pd)
+ # follow_fix, never adsb_fix: the feed's single-node ADS-B section and
+ # the node-trust residuals both key on adsb_fix, and a follow claim has
+ # no transponder fix to offer them.
+ assert "adsb_fix" not in c
+ assert c["follow_fix"]["lat"] == pytest.approx(_LAT)
+
+ def test_adsb_wins_a_contested_detection(self, monkeypatch):
+ """One detection both a cached transponder fix and a followed track
+ explain. The ADS-B paths run first and path 3 only ever sees what they
+ left, so the aircraft with an identity keeps it."""
+ ts = int(time.time() * 1000)
+ geo = _install(monkeypatch, ts - 2000)
+ state.adsb_aircraft["abc123"] = {
+ "hex": "abc123",
+ "lat": _LAT,
+ "lon": _LON,
+ "alt_baro": _ALT_M / FT_TO_M,
+ "gs": 0,
+ "track": 0,
+ "last_seen_ms": ts,
+ }
+ pd, pf = _pred(geo)
+ followed: set[int] = set()
+
+ claimed = kc.claim_known_targets(_NODE_ID, _frame(ts, [pd], [pf]), follow_claimed=followed)
+
+ assert claimed == {0}
+ assert followed == set()
+ assert _KEY not in state.known_claims
+ assert state.dark_follow_claims == 0
+
+ def test_off_mode_claims_nothing(self, monkeypatch):
+ ts = int(time.time() * 1000)
+ geo = _install(monkeypatch, ts - 2000, mode="off")
+ pd, pf = _pred(geo)
+ followed: set[int] = set()
+
+ kc.claim_known_targets(_NODE_ID, _frame(ts, [pd], [pf]), follow_claimed=followed)
+
+ assert followed == set()
+ assert state.known_claims == {}
+
+ def test_a_caller_that_cannot_take_the_split_gets_no_path_3(self, monkeypatch):
+ """Omitting follow_claimed disables path 3: the two lanes have
+ independent binding modes, so a caller that cannot separate them must
+ not be handed a set it would strip wholesale."""
+ ts = int(time.time() * 1000)
+ geo = _install(monkeypatch, ts - 2000)
+ pd, pf = _pred(geo)
+
+ assert kc.claim_known_targets(_NODE_ID, _frame(ts, [pd], [pf])) == set()
+ assert state.known_claims == {}
+
+ def test_a_stale_track_claims_nothing(self, monkeypatch):
+ ts = int(time.time() * 1000)
+ geo = _register()
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "shadow")
+ state.multinode_tracks[_KEY] = _track(int((time.time() - 120) * 1000))
+ _kf(monkeypatch)
+ dark_follow._reset_for_tests()
+ pd, pf = _pred(geo)
+ followed: set[int] = set()
+
+ kc.claim_known_targets(_NODE_ID, _frame(ts, [pd], [pf]), follow_claimed=followed)
+
+ assert followed == set()
+
+
+class TestFollowPass:
+ """known_lane's second pass: claims → solver input."""
+
+ _CFGS = {"n1": _NODE_CFG, "n2": _NODE_CFG, "n3": _NODE_CFG}
+
+ @pytest.fixture(autouse=True)
+ def _private_queue(self, monkeypatch):
+ """A queue of this test's own: a leaked solver worker daemon drains
+ state.solver_queue, and would take the item under assertion."""
+ import queue
+
+ monkeypatch.setattr(state, "solver_queue", queue.Queue(maxsize=200))
+
+ def test_three_nodes_produce_one_anchored_queue_item(self):
+ ts = int(time.time() * 1000)
+ _install_follow_claims(["n1", "n2", "n3"], ts)
+
+ assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 1
+
+ (item,) = _drain_queue()
+ s_in, cfgs, _enqueued = item
+ assert s_in["anchor_key"] == _KEY
+ assert s_in["follow_key"] == _KEY
+ assert s_in["lane"] == "dark_follow"
+ assert s_in["guess_source"] == "prediction"
+ assert s_in["track_ids"] == []
+ assert s_in["n_nodes"] == 3
+ assert s_in["initial_guess"]["lat"] == pytest.approx(_LAT)
+ assert s_in["initial_guess"]["lon"] == pytest.approx(_LON)
+ assert s_in["initial_guess"]["alt_km"] == pytest.approx(_ALT_M / 1000.0)
+ assert set(cfgs) == {"n1", "n2", "n3"}
+ assert state.dark_follow_inputs == 1
+
+ def test_a_single_node_is_not_enough(self):
+ _install_follow_claims(["n1"], int(time.time() * 1000))
+
+ assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 0
+ assert _drain_queue() == []
+
+ def test_the_rate_limit_holds_between_passes(self):
+ ts = int(time.time() * 1000)
+ _install_follow_claims(["n1", "n2"], ts)
+ assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 1
+
+ # A newer claim set, but inside DARK_FOLLOW_INTERVAL_S.
+ _install_follow_claims(["n1", "n2"], ts + 500)
+ assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 0
+ assert len(_drain_queue()) == 1
+
+ def test_no_newer_claim_is_no_new_solve(self, monkeypatch):
+ monkeypatch.setattr(dark_follow, "DARK_FOLLOW_INTERVAL_S", 0.0)
+ _install_follow_claims(["n1", "n2"], int(time.time() * 1000))
+ assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 1
+
+ assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 0
+
+ def test_off_mode_does_nothing(self):
+ _install_follow_claims(["n1", "n2"], int(time.time() * 1000))
+
+ assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="off") == 0
+ assert _drain_queue() == []
+
+ def test_shadow_records_and_never_publishes(self):
+ ts = int(time.time() * 1000)
+ _install_follow_claims(["n1", "n2"], ts)
+
+ def solve(s_in, cfgs):
+ ig = s_in["initial_guess"]
+ return {
+ "success": True,
+ "lat": ig["lat"],
+ "lon": ig["lon"],
+ "alt_m": ig["alt_km"] * 1000.0,
+ "timestamp_ms": s_in["timestamp_ms"],
+ "n_nodes": s_in["n_nodes"],
+ "contributing_node_ids": ["n1", "n2"],
+ }
+
+ assert known_lane.run_dark_follow_pass(solve, self._CFGS, mode="shadow") == 1
+
+ assert _drain_queue() == []
+ assert state.multinode_tracks == {}
+ assert state.dark_follow_published == 0
+ (rec,) = [r for r in state.mlat_solve_history if r.get("follow_key")]
+ assert rec["outcome"] == "dark_follow_shadow"
+ assert rec["lane"] == "dark_follow"
+ assert rec["guess_source"] == "prediction"
+ assert rec["published"] is False
+ assert rec["displacement_km"] == pytest.approx(0.0, abs=1e-3)
+
+ def test_shadow_classifies_a_displaced_solve_as_not_ok(self):
+ """The shadow verdict is the dark displacement cap — the same number
+ the binding path's gate would judge the solve by — so the guard is not
+ inert for the whole soak."""
+ _install_follow_claims(["n1", "n2"], int(time.time() * 1000))
+
+ def solve(s_in, cfgs):
+ ig = s_in["initial_guess"]
+ return {
+ "success": True,
+ "lat": ig["lat"] + 0.5, # ~55 km
+ "lon": ig["lon"],
+ "timestamp_ms": s_in["timestamp_ms"],
+ "n_nodes": s_in["n_nodes"],
+ "contributing_node_ids": ["n1", "n2"],
+ }
+
+ known_lane.run_dark_follow_pass(solve, self._CFGS, mode="shadow")
+
+ (rec,) = [r for r in state.mlat_solve_history if r.get("follow_key")]
+ assert rec["follow_ok"] is False
+
+ def test_a_node_with_no_config_is_dropped_not_the_key(self):
+ ts = int(time.time() * 1000)
+ _install_follow_claims(["n1", "n2", "gone"], ts)
+
+ assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 1
+
+ (item,) = _drain_queue()
+ assert set(item[1]) == {"n1", "n2"}
+ assert item[0]["n_nodes"] == 2
+
+ def test_the_adsb_pass_ignores_follow_claims(self):
+ """Both passes read state.known_claims; neither may solve the other's
+ entries."""
+ _install_follow_claims(["n1", "n2"], int(time.time() * 1000))
+
+ assert known_lane.run_known_lane_pass(lambda s, c: None, self._CFGS, mode="shadow") == 0
+
+
+class TestModesInProcessOneFrame:
+ """Binding removes the followed detections from the frame the dark lane
+ processes; shadow leaves the frame whole."""
+
+ def _run(self, monkeypatch, mode):
+ ts = int(time.time() * 1000)
+ geo = _install(monkeypatch, ts - 2000, mode=mode)
+ monkeypatch.setattr(state, "KNOWN_LANE_MODE", "binding")
+ pd, pf = _pred(geo)
+ frame = _frame(ts, [pd, pd + 500.0], [pf, pf + 500.0])
+
+ default = PassiveRadarPipeline(DEFAULT_NODE_CONFIG)
+ seen = []
+ monkeypatch.setattr(default, "process_frame", lambda f: seen.append(f))
+ process_one_frame(_NODE_ID, frame, default)
+ assert len(seen) == 1
+ return frame, seen[0]
+
+ def test_binding_strips_the_followed_detection(self, monkeypatch):
+ frame, processed = self._run(monkeypatch, "binding")
+
+ assert len(processed["delay"]) == 1
+ assert processed["delay"][0] == frame["delay"][1]
+ # The original frame is untouched — the archive and the ADS-B cache
+ # extraction still see everything the node sent.
+ assert len(frame["delay"]) == 2
+
+ def test_shadow_leaves_the_frame_whole(self, monkeypatch):
+ _frame_in, processed = self._run(monkeypatch, "shadow")
+
+ assert len(processed["delay"]) == 2
+ assert state.dark_follow_claims == 1
+
+
+class TestGhostGuard:
+ """Following is a feedback loop; the guard is what makes it droppable."""
+
+ def _armed(self, monkeypatch):
+ ts = int(time.time() * 1000) - 2000
+ _install(monkeypatch, ts)
+ assert len(dark_follow.follow_targets()) == 1
+
+ def test_two_rejected_solves_drop_the_key(self, monkeypatch):
+ self._armed(monkeypatch)
+
+ dark_follow.record_outcome(_KEY, False)
+ dark_follow._expire_targets_for_tests()
+ assert len(dark_follow.follow_targets()) == 1
+
+ dark_follow.record_outcome(_KEY, False)
+ dark_follow._expire_targets_for_tests()
+ assert dark_follow.follow_targets() == []
+ assert state.dark_follow_dropped == 1
+
+ def test_a_good_solve_clears_the_streak(self, monkeypatch):
+ self._armed(monkeypatch)
+
+ dark_follow.record_outcome(_KEY, False)
+ dark_follow.record_outcome(_KEY, True)
+ dark_follow.record_outcome(_KEY, False)
+ dark_follow._expire_targets_for_tests()
+
+ assert len(dark_follow.follow_targets()) == 1
+ assert state.dark_follow_dropped == 0
+
+ def test_the_cooldown_expires(self, monkeypatch):
+ self._armed(monkeypatch)
+ monkeypatch.setattr(dark_follow, "DARK_FOLLOW_COOLDOWN_S", 0.0)
+
+ dark_follow.drop_target(_KEY, "test")
+ dark_follow._expire_targets_for_tests()
+
+ assert len(dark_follow.follow_targets()) == 1
+
+ def test_rejected_history_records_feed_the_guard(self, monkeypatch):
+ """The verdicts arrive through _record_solve_history in binding mode —
+ the one place every outcome of a follow-solve passes through."""
+ self._armed(monkeypatch)
+ s_in = {"follow_key": _KEY, "lane": "dark_follow", "n_nodes": 3}
+
+ for _ in range(2):
+ solver_mod._record_solve_history("rejected_rms_delay", s_in, _reject_result())
+
+ dark_follow._expire_targets_for_tests()
+ assert dark_follow.follow_targets() == []
+ assert state.dark_follow_dropped == 1
+
+ def test_a_published_record_counts_and_clears(self, monkeypatch):
+ self._armed(monkeypatch)
+ s_in = {"follow_key": _KEY, "lane": "dark_follow", "n_nodes": 3}
+ solver_mod._record_solve_history("rejected_rms_delay", s_in, _reject_result())
+ solver_mod._record_solve_history("published", s_in, _reject_result(), solve_key=_KEY)
+
+ assert state.dark_follow_published == 1
+ dark_follow._expire_targets_for_tests()
+ assert len(dark_follow.follow_targets()) == 1
+
+
+class TestAnchorDeadReckoning:
+ """A follow input's anchor is stale by construction — its guess IS the
+ prediction of where the anchor drifted to — so the anchor distance check
+ has to dead-reckon before it measures."""
+
+ _TS_MS = 1_000_000
+
+ def _tracks(self, dt_s, speed_ms):
+ return {
+ _KEY: {
+ "lat": _LAT,
+ "lon": _LON,
+ "vel_east": 0.0,
+ "vel_north": speed_ms,
+ "timestamp_ms": self._TS_MS - int(dt_s * 1000),
+ "n_nodes": 3,
+ "solve_count": 5,
+ }
+ }
+
+ def _result(self, north_km):
+ lat, lon = offset_latlon_m(_LAT, _LON, east_m=0.0, north_m=north_km * 1000.0)
+ return {"lat": lat, "lon": lon, "timestamp_ms": self._TS_MS}
+
+ def test_the_flat_gate_refuses_a_fast_anchor(self):
+ """15 s of coasting at 270 m/s is 4.05 km of travel; a solve 2 km past
+ that is 6.05 km from where the entry was last STORED — outside the flat
+ 6 km gate, purely because the aircraft moved."""
+ key, how, _d = solver_mod.multinode_key_decision(
+ self._tracks(15.0, 270.0),
+ self._result(6.05),
+ None,
+ _KEY,
+ learned_vel_fn=lambda _k: None,
+ )
+ assert how != "anchor"
+
+ def test_dead_reckoning_honours_it(self):
+ key, how, dist = solver_mod.multinode_key_decision(
+ self._tracks(15.0, 270.0),
+ self._result(6.05),
+ None,
+ _KEY,
+ learned_vel_fn=lambda _k: None,
+ anchor_dr=True,
+ )
+ assert (key, how) == (_KEY, "anchor")
+ assert dist == pytest.approx(2.0, abs=0.1)
+
+ def test_dead_reckoning_still_refuses_a_far_solve(self):
+ """The check's job is unchanged: an anchor whose solve converged
+ somewhere else entirely is not honoured just because it was named."""
+ key, how, _d = solver_mod.multinode_key_decision(
+ self._tracks(15.0, 270.0),
+ self._result(30.0),
+ None,
+ _KEY,
+ learned_vel_fn=lambda _k: None,
+ anchor_dr=True,
+ )
+ assert how != "anchor"
+
+
+class TestKeyOwnership:
+ """A key the follow lane just published on is not the bottom-up lane's to
+ join.
+
+ Binding mode only, and the reason is measured rather than aesthetic: 21% of
+ bottom-up proximity joins on test landed on a key belonging to a DIFFERENT
+ aircraft, and same-aircraft re-key distances (p50 1.5 km) overlap the
+ wrong-aircraft ones entirely, so no tighter spatial gate separates them.
+ See dark_follow.DARK_FOLLOW_OWN_S.
+ """
+
+ _TS_MS = 2_000_000
+ _TS_S = _TS_MS / 1000.0
+
+ def setup_method(self):
+ dark_follow._reset_for_tests()
+
+ def teardown_method(self):
+ dark_follow._reset_for_tests()
+
+ def _tracks(self, dt_s=1.0):
+ """One live dark entry at the reference position, last solved dt_s ago
+ and not moving — so the distance below is exactly the offset."""
+ return {
+ _KEY: {
+ "lat": _LAT,
+ "lon": _LON,
+ "vel_east": 0.0,
+ "vel_north": 0.0,
+ "timestamp_ms": self._TS_MS - int(dt_s * 1000),
+ "n_nodes": 3,
+ "solve_count": 5,
+ }
+ }
+
+ def _result(self, north_km):
+ lat, lon = offset_latlon_m(_LAT, _LON, east_m=0.0, north_m=north_km * 1000.0)
+ return {"lat": lat, "lon": lon, "timestamp_ms": self._TS_MS}
+
+ def _decide(self, north_km, anchor_key=None, dt_s=1.0):
+ return solver_mod.multinode_key_decision(
+ self._tracks(dt_s),
+ self._result(north_km),
+ None,
+ anchor_key,
+ learned_vel_fn=lambda _k: None,
+ )
+
+ def test_a_solve_next_to_a_freshly_followed_key_is_shadowed(self, monkeypatch):
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding")
+ dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0)
+ key, how, dist = self._decide(1.0)
+ assert (key, how) == (_KEY, "shadowed")
+ assert dist == pytest.approx(1.0, abs=0.05)
+
+ def test_a_solve_further_out_mints_rather_than_joining(self, monkeypatch):
+ """4 km is well inside the 6 km proximity gate — without ownership
+ this solve joins the followed key, which is the bug. It gets its own
+ key instead: too far to be the same aircraft, and never the followed
+ one's."""
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding")
+ dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0)
+ key, how, _dist = self._decide(4.0)
+ assert how == "minted"
+ assert key != _KEY
+
+ def test_ownership_expires(self, monkeypatch):
+ """Past DARK_FOLLOW_OWN_S the lane has stopped answering for the key
+ (three missed follow-solve intervals), so the bottom-up lane may have
+ it back."""
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding")
+ dark_follow.note_follow_publish(_KEY, self._TS_S - 20.0)
+ key, how, _dist = self._decide(1.0)
+ assert (key, how) == (_KEY, "proximity")
+
+ @pytest.mark.parametrize("mode", ["shadow", "off"])
+ def test_the_inert_modes_key_exactly_as_before(self, monkeypatch, mode):
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", mode)
+ dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0)
+ key, how, _dist = self._decide(1.0)
+ assert (key, how) == (_KEY, "proximity")
+
+ def test_the_follow_lanes_own_solve_still_lands_on_its_key(self, monkeypatch):
+ """Ownership is a rule about BOTTOM-UP solves. The follow lane's own
+ solves are anchored and return from the anchor branch, which never
+ reaches the proximity scan — otherwise the lane would shadow itself
+ off the map."""
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding")
+ dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0)
+ key, how, dist = self._decide(1.0, anchor_key=_KEY)
+ assert (key, how) == (_KEY, "anchor")
+ assert dist == pytest.approx(1.0, abs=0.05)
+
+ def test_an_n2_solve_cannot_join_a_followed_n3_key(self, monkeypatch):
+ """The population the ownership rule is aimed at: an n=2 bottom-up
+ solve, whose own position error is ~2.4 km median, arriving at a key
+ the follow lane is refreshing from n>=3 measurements."""
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding")
+ dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0)
+ result = self._result(1.5)
+ result["n_nodes"] = 2
+ key, how, _dist = solver_mod.multinode_key_decision(
+ self._tracks(),
+ result,
+ None,
+ None,
+ learned_vel_fn=lambda _k: None,
+ )
+ assert (key, how) == (_KEY, "shadowed")
+
+
+class TestShadowedSolveIsARejection:
+ """What the solver worker does with a "shadowed" verdict: record it, count
+ it, and touch nothing else."""
+
+ def setup_method(self):
+ state._reset_for_tests()
+ solver_mod._reset_for_tests()
+ dark_follow._reset_for_tests()
+
+ def teardown_method(self):
+ solver_mod._reset_for_tests()
+ dark_follow._reset_for_tests()
+
+ def _entry(self, ts_ms):
+ return {
+ "success": True,
+ "lat": _LAT,
+ "lon": _LON,
+ "alt_m": _ALT_M,
+ "vel_east": 0.0,
+ "vel_north": 0.0,
+ "n_nodes": 3,
+ "solve_count": 5,
+ "timestamp_ms": ts_ms - 1000,
+ "contributing_node_ids": [_NODE_ID],
+ }
+
+ def _solve_fn(self, ts_ms, north_km):
+ lat, lon = offset_latlon_m(_LAT, _LON, east_m=0.0, north_m=north_km * 1000.0)
+
+ def fn(_s_in, _cfgs):
+ return {
+ "success": True,
+ "lat": lat,
+ "lon": lon,
+ "alt_m": _ALT_M,
+ "vel_east": 0.0,
+ "vel_north": 0.0,
+ "rms_delay": 1.0,
+ "rms_doppler": 5.0,
+ "n_nodes": 3,
+ "n_measurements": 3,
+ "timestamp_ms": ts_ms,
+ "contributing_node_ids": ["n1", "n2", "n3"],
+ }
+
+ return fn
+
+ def _run(self, monkeypatch, north_km=1.0, follow_age_s=2.0):
+ monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding")
+ ts_ms = int(time.time() * 1000)
+ state.multinode_tracks[_KEY] = self._entry(ts_ms)
+ dark_follow.note_follow_publish(_KEY, ts_ms / 1000.0 - follow_age_s)
+ solver_mod._process_solver_item(
+ ({"n_nodes": 3}, {}, time.time()),
+ self._solve_fn(ts_ms, north_km),
+ )
+ return ts_ms
+
+ def test_the_solve_is_recorded_counted_and_not_published(self, monkeypatch):
+ ts_ms = self._run(monkeypatch)
+ assert state.dark_bottomup_shadowed == 1
+ # The followed entry is byte-for-byte what it was: no new position, no
+ # solve_count bump, no smoothing, and no second key minted beside it.
+ assert list(state.multinode_tracks) == [_KEY]
+ assert state.multinode_tracks[_KEY] == self._entry(ts_ms)
+ assert len(state.mlat_solve_history) == 1
+ rec = state.mlat_solve_history[0]
+ assert rec["outcome"] == "shadowed_by_follow"
+ assert rec["follow_key"] == _KEY
+ assert rec["key_how"] == "shadowed"
+ assert rec["key_dist_km"] == pytest.approx(1.0, abs=0.05)
+ # A reject has no key of its own, shadowed or otherwise.
+ assert rec["solve_key"] is None
+ assert rec["lat"] is None
+
+ def test_a_solve_the_lane_does_not_own_still_publishes(self, monkeypatch):
+ """The same solve with the ownership window expired — the control that
+ says the assertions above are about ownership and not about the
+ harness."""
+ self._run(monkeypatch, follow_age_s=30.0)
+ assert state.dark_bottomup_shadowed == 0
+ assert state.mlat_solve_history[0]["outcome"] == "published"
+
+ def test_refusals_never_reach_the_follow_ghost_guard(self, monkeypatch):
+ """A shadowed record names a followed key but was not produced BY the
+ follow lane, so the guard must not hear about it — otherwise the
+ bottom-up lane's refusals would drop the very track that refused
+ them, twice in a row being enough."""
+ _kf(monkeypatch)
+ ts_ms = self._run(monkeypatch)
+ state.multinode_tracks[_KEY] = self._entry(ts_ms)
+ dark_follow.note_follow_publish(_KEY, ts_ms / 1000.0 - 2.0)
+ solver_mod._process_solver_item(
+ ({"n_nodes": 3}, {}, time.time()),
+ self._solve_fn(ts_ms, 1.0),
+ )
+ assert state.dark_bottomup_shadowed == 2
+ # Two rejects in a row is exactly what drops a followed key. It is
+ # still a target, so the guard never saw them.
+ assert [t["key"] for t in dark_follow.follow_targets()] == [_KEY]
diff --git a/backend/tests/test_epoch_alignment.py b/backend/tests/test_epoch_alignment.py
new file mode 100644
index 00000000..a805dc22
--- /dev/null
+++ b/backend/tests/test_epoch_alignment.py
@@ -0,0 +1,274 @@
+"""Measurement epoch alignment (SOLVER_EPOCH_ALIGN) — solver.align_measurement_epochs.
+
+The solver's residual model evaluates every measurement against ONE target
+state, so a solver input is implicitly a claim that its measurements were
+simultaneous. Nodes sample on independent free-running cadences, so the claim
+is false by up to a frame interval, and the resulting delay error is charged to
+the rms_delay gate. These tests pin the correction, and — more importantly —
+pin its SIGN against the simulator's own geometry rather than against the
+derivation the helper's comment gives, since a sign error there would silently
+double the very error the correction exists to remove.
+"""
+
+import pytest
+from retina_simulation.world import _bistatic_delay, _bistatic_doppler
+
+from core import state
+from services.tasks import solver as solver_mod
+from services.tasks.solver import align_measurement_epochs
+
+_FC_HZ = 183e6
+
+# One node's ENU geometry, km. Only fc_hz is read by the helper; the rest is
+# here because the simulator's delay/Doppler helpers need a real bistatic
+# triangle to produce numbers whose sign means anything.
+_TX_ENU = (-20.0, 5.0, 0.05)
+_RX_ENU = (0.0, 0.0, 0.3)
+
+_NODE_CFGS = {
+ "node-a": {"fc_hz": _FC_HZ},
+ "node-b": {"fc_hz": _FC_HZ},
+ "node-c": {"FC": _FC_HZ}, # the alternate spelling the geolocator accepts
+}
+
+
+def _s_in(measurements, **over):
+ base = {
+ "initial_guess": {"lat": 34.85, "lon": -82.4, "alt_km": 9.0},
+ "measurements": measurements,
+ "n_nodes": len({m["node_id"] for m in measurements}),
+ "timestamp_ms": 1_700_000_000_000,
+ }
+ base.update(over)
+ return base
+
+
+def _m(node_id, delay_us, doppler_hz, t_s, snr=15.0):
+ return {
+ "node_id": node_id,
+ "delay_us": delay_us,
+ "doppler_hz": doppler_hz,
+ "snr": snr,
+ "t_s": t_s,
+ }
+
+
+@pytest.fixture(autouse=True)
+def _zero_counter():
+ state.solver_epoch_align_skipped = 0
+ yield
+
+
+class TestPureHelper:
+ def test_newest_measurement_is_the_epoch_and_is_untouched(self):
+ """t0 is the newest SAMPLE time, not the input's timestamp_ms: the
+ freshest node needed no correction and must not acquire one."""
+ s_in = _s_in(
+ [
+ _m("node-a", 40.0, 100.0, 1000.0),
+ _m("node-b", 50.0, -80.0, 1001.5),
+ _m("node-c", 60.0, 0.0, 1002.0),
+ ]
+ )
+ out, meta = align_measurement_epochs(s_in, _NODE_CFGS)
+ by_id = {m["node_id"]: m for m in out["measurements"]}
+ assert by_id["node-c"]["delay_us"] == 60.0
+ assert meta["epoch_aligned"] is True
+ assert meta["epoch_skew_s"] == pytest.approx(2.0)
+
+ def test_each_delay_moves_by_its_own_doppler_rate(self):
+ """d(delay_us)/dt = -doppler_hz * 1e6 / fc_hz, applied over that
+ measurement's own gap to t0."""
+ s_in = _s_in(
+ [
+ _m("node-a", 40.0, 100.0, 1000.0),
+ _m("node-b", 50.0, -80.0, 1001.5),
+ _m("node-c", 60.0, 0.0, 1002.0),
+ ]
+ )
+ out, _ = align_measurement_epochs(s_in, _NODE_CFGS)
+ by_id = {m["node_id"]: m for m in out["measurements"]}
+ assert by_id["node-a"]["delay_us"] == pytest.approx(40.0 + (-100.0 * 1e6 / _FC_HZ) * 2.0)
+ assert by_id["node-b"]["delay_us"] == pytest.approx(50.0 + (80.0 * 1e6 / _FC_HZ) * 0.5)
+
+ def test_zero_doppler_measurement_is_unchanged(self):
+ """A tangential target's bistatic range is stationary, so no amount of
+ skew moves its delay — the rate is the only thing that can."""
+ s_in = _s_in([_m("node-a", 40.0, 0.0, 1000.0), _m("node-b", 50.0, 20.0, 1004.0)])
+ out, _ = align_measurement_epochs(s_in, _NODE_CFGS)
+ assert out["measurements"][0]["delay_us"] == 40.0
+
+ def test_input_is_not_mutated(self):
+ """Pure: a caller must be able to drop the result and keep the
+ original, which is exactly what the flag-off path does."""
+ meas = [_m("node-a", 40.0, 100.0, 1000.0), _m("node-b", 50.0, -80.0, 1002.0)]
+ s_in = _s_in(meas)
+ out, _ = align_measurement_epochs(s_in, _NODE_CFGS)
+ assert s_in["measurements"][0]["delay_us"] == 40.0
+ assert s_in["measurements"] is not out["measurements"]
+ assert s_in["timestamp_ms"] == 1_700_000_000_000
+
+ def test_timestamp_ms_is_restamped_to_the_epoch(self):
+ s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), _m("node-b", 50.0, -80.0, 1002.25)])
+ out, _ = align_measurement_epochs(s_in, _NODE_CFGS)
+ assert out["timestamp_ms"] == 1_002_250
+
+ def test_fc_spelled_FC_is_accepted(self):
+ """Same fallback chain the geolocator uses to build its NodeSetup, so
+ a node aligns on exactly the carrier its solve predicts against."""
+ s_in = _s_in([_m("node-c", 40.0, 100.0, 1000.0), _m("node-b", 50.0, 0.0, 1001.0)])
+ out, meta = align_measurement_epochs(s_in, _NODE_CFGS)
+ assert meta["epoch_aligned"] is True
+ assert out["measurements"][0]["delay_us"] == pytest.approx(40.0 - 100.0 * 1e6 / _FC_HZ)
+
+ def test_single_measurement_input_is_a_no_op(self):
+ s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0)])
+ out, meta = align_measurement_epochs(s_in, _NODE_CFGS)
+ assert out is s_in
+ assert meta == {"epoch_aligned": False}
+ assert state.solver_epoch_align_skipped == 0
+
+
+class TestSkipPath:
+ @pytest.mark.parametrize(
+ "broken",
+ [
+ {"t_s": None},
+ {"doppler_hz": None},
+ ],
+ )
+ def test_missing_field_skips_the_whole_input(self, broken):
+ """All-or-nothing: a partially aligned set has no marker saying which
+ measurements share an epoch, so it just relocates the error."""
+ good = _m("node-a", 40.0, 100.0, 1000.0)
+ bad = {**_m("node-b", 50.0, -80.0, 1002.0), **broken}
+ s_in = _s_in([good, bad])
+ out, meta = align_measurement_epochs(s_in, _NODE_CFGS)
+ assert out is s_in
+ assert meta == {"epoch_aligned": False}
+ assert state.solver_epoch_align_skipped == 1
+
+ def test_missing_t_s_key_entirely_skips(self):
+ """The pre-upgrade measurement shape: no t_s key at all."""
+ untimed = {"node_id": "node-b", "delay_us": 50.0, "doppler_hz": -80.0, "snr": 9.0}
+ s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), untimed])
+ out, meta = align_measurement_epochs(s_in, _NODE_CFGS)
+ assert out is s_in
+ assert meta["epoch_aligned"] is False
+ assert state.solver_epoch_align_skipped == 1
+
+ def test_unknown_node_config_skips(self):
+ s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), _m("node-zzz", 50.0, -80.0, 1002.0)])
+ out, meta = align_measurement_epochs(s_in, {"node-a": {"fc_hz": _FC_HZ}})
+ assert out is s_in
+ assert meta["epoch_aligned"] is False
+ assert state.solver_epoch_align_skipped == 1
+
+
+class TestSignAgainstSimulatorGeometry:
+ """The sign check, run against the simulator's own delay/Doppler model.
+
+ A target is flown in a straight line and sampled at two times using
+ _bistatic_delay / _bistatic_doppler. The older sample plus the correction
+ must land on the newer sample's true delay — which is a statement about the
+ sign of the Doppler-to-delay-rate conversion that no amount of algebra in a
+ comment can substitute for.
+ """
+
+ _POS0 = (10.0, 15.0, 9.0) # km ENU
+ _DT_S = 2.0
+
+ @staticmethod
+ def _truth(vel_kms, dt_s):
+ pos0 = TestSignAgainstSimulatorGeometry._POS0
+ pos1 = tuple(pos0[i] + vel_kms[i] * dt_s for i in range(3))
+ return (
+ _bistatic_delay(pos0, _TX_ENU, _RX_ENU),
+ _bistatic_delay(pos1, _TX_ENU, _RX_ENU),
+ _bistatic_doppler(pos0, vel_kms, _TX_ENU, _RX_ENU, _FC_HZ),
+ )
+
+ @pytest.mark.parametrize(
+ "vel_kms",
+ [
+ (-0.20, -0.15, 0.0), # inbound: bistatic range shrinking
+ (0.20, 0.15, 0.0), # outbound: bistatic range growing
+ (0.05, -0.24, 0.01), # mostly crossing, with a climb
+ ],
+ )
+ def test_alignment_moves_the_stale_delay_toward_the_truth(self, vel_kms):
+ delay0, delay1, doppler0 = self._truth(vel_kms, self._DT_S)
+
+ # node-a sampled _DT_S seconds ago; node-b is the newest sample and
+ # therefore defines t0. node-b's own numbers are irrelevant to the
+ # assertion — it is only here to set the epoch.
+ s_in = _s_in(
+ [
+ _m("node-a", delay0, doppler0, 1000.0),
+ _m("node-b", 77.0, 0.0, 1000.0 + self._DT_S),
+ ]
+ )
+ out, meta = align_measurement_epochs(s_in, _NODE_CFGS)
+ assert meta["epoch_aligned"] is True
+ aligned = out["measurements"][0]["delay_us"]
+
+ err_before = abs(delay0 - delay1)
+ err_after = abs(aligned - delay1)
+ # A wrong sign would double err_before rather than shrink it, so the
+ # margin here is the sign test. Over 2 s of straight-line flight the
+ # first-order term dominates; the residual is the trajectory's
+ # curvature in bistatic range, not a modelling disagreement.
+ assert err_after < err_before * 0.2
+ assert err_before > 0.05 # the case would prove nothing otherwise
+
+ def test_correction_and_truth_share_a_sign(self):
+ """Stated directly, so a failure says 'the sign is wrong' rather than
+ 'the error did not shrink enough'."""
+ vel_kms = (-0.20, -0.15, 0.0)
+ delay0, delay1, doppler0 = self._truth(vel_kms, self._DT_S)
+ s_in = _s_in(
+ [
+ _m("node-a", delay0, doppler0, 1000.0),
+ _m("node-b", 77.0, 0.0, 1000.0 + self._DT_S),
+ ]
+ )
+ out, _ = align_measurement_epochs(s_in, _NODE_CFGS)
+ correction = out["measurements"][0]["delay_us"] - delay0
+ assert correction * (delay1 - delay0) > 0
+
+
+class TestProcessSolverItemWiring:
+ """The flag, and that the aligned numbers are what the solve actually sees."""
+
+ @staticmethod
+ def _item():
+ s_in = _s_in(
+ [
+ _m("node-a", 40.0, 300.0, 1000.0),
+ _m("node-b", 50.0, 0.0, 1004.0),
+ ]
+ )
+ return (s_in, _NODE_CFGS, None)
+
+ def test_flag_off_leaves_the_input_untouched(self, monkeypatch):
+ monkeypatch.setattr(state, "SOLVER_EPOCH_ALIGN", False)
+ seen = {}
+
+ def _solve(s_in, node_cfgs):
+ seen["delays"] = [m["delay_us"] for m in s_in["measurements"]]
+ return None
+
+ solver_mod._process_solver_item(self._item(), _solve)
+ assert seen["delays"] == [40.0, 50.0]
+
+ def test_flag_on_hands_the_solver_aligned_delays(self, monkeypatch):
+ monkeypatch.setattr(state, "SOLVER_EPOCH_ALIGN", True)
+ seen = {}
+
+ def _solve(s_in, node_cfgs):
+ seen["delays"] = [m["delay_us"] for m in s_in["measurements"]]
+ return None
+
+ solver_mod._process_solver_item(self._item(), _solve)
+ assert seen["delays"][0] == pytest.approx(40.0 + (-300.0 * 1e6 / _FC_HZ) * 4.0)
+ assert seen["delays"][1] == 50.0
diff --git a/backend/tests/test_feed_multinode.py b/backend/tests/test_feed_multinode.py
index 9fb4ab28..1b1d1d74 100644
--- a/backend/tests/test_feed_multinode.py
+++ b/backend/tests/test_feed_multinode.py
@@ -1,9 +1,11 @@
"""Feed-side handling of multinode solver tracks.
Covers the dead-reckoning of mn-* entries in build_combined_aircraft_json:
-position is advanced with the solved velocity, but only up to a 30 s horizon —
+position is advanced with the solved velocity, but only up to MN_DR_CAP_S —
past that a velocity error dominates any solve accuracy, so an old solve holds
-its last dead-reckoned point until the 60 s entry expiry.
+its last dead-reckoned point until the entry expires. Expiry is lane-aware:
+MN_DARK_EXPIRY_S for a dark (mn-dark-*) entry, the historic 60 s for an
+ADS-B-assisted (mn-adsb-*) one, whose transponder hex anchors it.
"""
import math
@@ -16,6 +18,7 @@
os.environ.setdefault("RETINA_ENV", "test")
os.environ.setdefault("RADAR_API_KEY", "test-key-abc123")
+from config.constants import MN_DARK_EXPIRY_S, MN_DR_CAP_S # noqa: E402
from core import state # noqa: E402
from services import track_filter # noqa: E402
from services.geo import offset_latlon_m # noqa: E402
@@ -68,15 +71,72 @@ def test_fresh_entry_is_dead_reckoned_fully(self):
exp_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * 10.0)
assert ac["lat"] == pytest.approx(exp_lat, abs=2e-4)
- def test_dr_horizon_is_capped_at_30s(self):
- # 45 s old (younger than the 60 s expiry): advanced 30 s worth of
- # motion, not 45.
- state.multinode_tracks["mn-dark-x"] = _mn_entry(age_s=45.0, vel_north=100.0)
+ def test_dr_horizon_is_capped_at_mn_dr_cap_s(self):
+ # 20 s old (still inside the 30 s dark expiry, so this exercises the
+ # DR cap and not staleness): advanced MN_DR_CAP_S worth of motion,
+ # not 20 s worth.
+ state.multinode_tracks["mn-dark-x"] = _mn_entry(age_s=20.0, vel_north=100.0)
ac = self._build_mn()
- capped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * 30.0)
- uncapped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * 45.0)
+ capped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * MN_DR_CAP_S)
+ uncapped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * 20.0)
assert ac["lat"] == pytest.approx(capped_lat, abs=2e-4)
- assert abs(ac["lat"] - uncapped_lat) > 5e-3
+ # The 5 s the cap withheld is 500 m of northing, ~0.0045 deg of lat.
+ assert abs(ac["lat"] - uncapped_lat) > 3e-3
+
+ def test_assisted_entry_past_the_cap_holds_its_dead_reckoned_point(self):
+ # The assisted lane lives 60 s, so it reaches ages the dark lane never
+ # does — and past MN_DR_CAP_S it must hold, not keep extrapolating.
+ state.multinode_tracks["mn-adsb-abc123"] = _mn_entry(age_s=45.0, vel_north=100.0)
+ ac = self._build_mn()
+ capped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * MN_DR_CAP_S)
+ assert ac["lat"] == pytest.approx(capped_lat, abs=2e-4)
+
+
+class TestMultinodeLaneExpiry:
+ """Dark entries expire at MN_DARK_EXPIRY_S, assisted ones at 60 s.
+
+ A dark solve now lands every 1-3 s while the aircraft is tracked, so an
+ entry that has not re-solved in 30 s is a lost track rather than a cadence
+ gap; holding it drew an icon a measured 3.99 km (median) from any
+ aircraft. An mn-adsb-* entry is anchored to a transponder hex, so the
+ same gap is the ADS-B feed breathing and it keeps the historic 60 s.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _clean_state(self):
+ state.multinode_tracks.clear()
+ state.track_histories.clear()
+ yield
+ state.multinode_tracks.clear()
+ state.track_histories.clear()
+
+ def _mn_hexes(self):
+ from services.frame_processor import build_combined_aircraft_json
+
+ pipeline = types.SimpleNamespace(geolocated_tracks={}, config={})
+ result = build_combined_aircraft_json(pipeline)
+ return [a["hex"] for a in result["aircraft"] if a.get("multinode")]
+
+ def test_dark_entry_inside_the_dark_expiry_is_rendered(self):
+ state.multinode_tracks["mn-dark-x"] = _mn_entry(age_s=MN_DARK_EXPIRY_S - 5.0, vel_north=100.0)
+ assert len(self._mn_hexes()) == 1
+ assert "mn-dark-x" in state.multinode_tracks
+
+ def test_dark_entry_past_the_dark_expiry_is_dropped(self):
+ state.multinode_tracks["mn-dark-x"] = _mn_entry(age_s=MN_DARK_EXPIRY_S + 1.0, vel_north=100.0)
+ assert self._mn_hexes() == []
+ # Expiry, not a display gate: the entry leaves the store entirely.
+ assert "mn-dark-x" not in state.multinode_tracks
+
+ def test_assisted_entry_at_the_same_age_survives(self):
+ state.multinode_tracks["mn-adsb-abc123"] = _mn_entry(age_s=MN_DARK_EXPIRY_S + 1.0, vel_north=100.0)
+ assert len(self._mn_hexes()) == 1
+ assert "mn-adsb-abc123" in state.multinode_tracks
+
+ def test_assisted_entry_still_expires_at_60s(self):
+ state.multinode_tracks["mn-adsb-abc123"] = _mn_entry(age_s=61.0, vel_north=100.0)
+ assert self._mn_hexes() == []
+ assert "mn-adsb-abc123" not in state.multinode_tracks
class TestMultinodeDeadReckonSource:
@@ -485,3 +545,98 @@ def test_kf_state_supplies_the_velocity_sigma(self):
ac = self._build_mn()
assert ac["pos_sigma_vel_ms"] == pytest.approx(round(lv[2], 1), abs=0.05)
assert ac["pos_sigma_vel_ms"] != pytest.approx(25.0)
+
+
+class TestMultinodeEntryFailureIsolation:
+ """One bad multinode key must cost one aircraft, not the whole feed.
+
+ The 2026-09-05 droplet failure went through here: a track whose filter
+ covariance had gone non-PSD made learned_velocity raise, and because the
+ per-entry work sat inline in build_combined_aircraft_json's loop the
+ exception propagated out of the flush task ("Aircraft flush failed") and
+ dropped the ENTIRE tick's payload -- every other aircraft with it, 91
+ times in 40 minutes. track_filter now stops that covariance ever
+ forming; this is the second line of defence, which has to hold for any
+ future per-entry bug, not just that one.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _clean_state(self):
+ from services import aircraft_feed
+
+ state.multinode_tracks.clear()
+ state.track_histories.clear()
+ track_filter.reset()
+ aircraft_feed._reset_for_tests()
+ yield
+ state.multinode_tracks.clear()
+ state.track_histories.clear()
+ track_filter.reset()
+ aircraft_feed._reset_for_tests()
+
+ def _build(self):
+ from services.frame_processor import build_combined_aircraft_json
+
+ pipeline = types.SimpleNamespace(geolocated_tracks={}, config={})
+ return build_combined_aircraft_json(pipeline)
+
+ def test_one_raising_entry_does_not_abort_the_build(self, monkeypatch, caplog):
+ from services import aircraft_feed
+
+ good_a, bad, good_b = "mn-dark-good-a", "mn-dark-bad", "mn-dark-good-b"
+ for i, key in enumerate((good_a, bad, good_b)):
+ # Spread them out: co-located entries are collapsed by
+ # dedup_aircraft, which would hide the very thing under test.
+ entry = _mn_entry(age_s=5.0, vel_north=100.0)
+ entry["lat"] = LAT + 0.1 * i
+ state.multinode_tracks[key] = entry
+
+ real_learned_velocity = track_filter.learned_velocity
+
+ def _boom(track_key):
+ # Exactly the failure the droplet saw, raised from exactly the
+ # function it was raised from.
+ if track_key == bad:
+ raise ValueError("math domain error")
+ return real_learned_velocity(track_key)
+
+ monkeypatch.setattr(aircraft_feed.track_filter, "learned_velocity", _boom)
+
+ with caplog.at_level("ERROR"):
+ result = self._build()
+
+ mn_hexes = {a["hex"] for a in result["aircraft"] if a.get("multinode")}
+ from services.id_utils import multinode_hex_from_key
+
+ # The two healthy aircraft are still served ...
+ assert multinode_hex_from_key(good_a) in mn_hexes
+ assert multinode_hex_from_key(good_b) in mn_hexes
+ # ... and only the sick one is missing.
+ assert multinode_hex_from_key(bad) not in mn_hexes
+ # Logged once, naming the key, so this is diagnosable rather than silent.
+ assert sum("Multinode feed entry failed" in r.message for r in caplog.records) == 1
+ assert bad in caplog.text
+
+ def test_repeated_failures_are_rate_limited_to_one_log_line(self, monkeypatch, caplog):
+ from services import aircraft_feed
+
+ bad = "mn-dark-bad"
+ state.multinode_tracks[bad] = _mn_entry(age_s=5.0, vel_north=100.0)
+
+ def _boom(track_key):
+ raise ValueError("math domain error")
+
+ monkeypatch.setattr(aircraft_feed.track_filter, "learned_velocity", _boom)
+
+ with caplog.at_level("ERROR"):
+ for _ in range(20):
+ # Re-stamp: the entry would otherwise age past the 60 s expiry
+ # only after many more ticks, but keeping it fresh makes the
+ # 20 failures unambiguous.
+ state.multinode_tracks[bad] = _mn_entry(age_s=5.0, vel_north=100.0)
+ self._build()
+
+ # 20 failures inside one _MN_ENTRY_FAIL_LOG_INTERVAL_S window ->
+ # exactly one line, not 20.
+ assert sum("Multinode feed entry failed" in r.message for r in caplog.records) == 1
+ assert aircraft_feed._mn_entry_fail_count == 20
diff --git a/backend/tests/test_frame_processor.py b/backend/tests/test_frame_processor.py
index c5d31e92..56a1409a 100644
--- a/backend/tests/test_frame_processor.py
+++ b/backend/tests/test_frame_processor.py
@@ -6,15 +6,19 @@
import queue
import time
+import types
import pytest
+from retina_tracker.track import TrackState
from config.constants import GT_DISPLAY_STALE_S
from core import state
from pipeline.passive_radar import DEFAULT_NODE_CONFIG, PassiveRadarPipeline
+from services import frame_processor
from services.frame_processor import (
append_track_history,
build_combined_aircraft_json,
+ confirmed_track_views,
dedup_aircraft,
flush_all_archive_buffers,
get_node_configs,
@@ -277,6 +281,95 @@ def test_claiming_anchored_inputs_reach_the_solver_queue(self, monkeypatch):
# ── Multinode result conversion ──────────────────────────────────────────────
+class TestConfirmedTrackViewsStaleness:
+ """TRACK_MAX_STALE_S: a coasting track stops being offered to association
+ once its newest REAL detection has aged out.
+
+ The freshness signal is the newest entry get_recent_detections returns,
+ which by construction is an ASSOCIATED sample (mark_missed appends None to
+ history["measurements"] and the reverse scan skips those) — so these fakes
+ hand back only real detections, exactly as the tracker does, and the coast
+ is expressed as a gap between that newest sample and the frame time.
+ """
+
+ @staticmethod
+ def _track(newest_ts_ms: int, status=TrackState.COASTING, track_id="trk-stale"):
+ hist = [
+ {"timestamp": newest_ts_ms - 1000, "delay": 40.0, "doppler": 5.0, "snr": 12.0, "adsb": None},
+ {"timestamp": newest_ts_ms, "delay": 41.0, "doppler": 5.0, "snr": 12.0, "adsb": None},
+ ]
+ return types.SimpleNamespace(
+ id=track_id,
+ state_status=status,
+ adsb_hex=None,
+ get_recent_detections=lambda n: hist[-n:],
+ )
+
+ def _tracker(self, *tracks):
+ return types.SimpleNamespace(tracks=list(tracks))
+
+ def test_five_second_old_coasting_track_is_excluded_at_three(self, monkeypatch):
+ monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0)
+ state.tracks_stale_skipped = 0
+ now_ms = 1_000_000
+ tracker = self._tracker(self._track(now_ms - 5000))
+ assert confirmed_track_views(tracker, now_ts_ms=now_ms) == []
+ assert state.tracks_stale_skipped == 1
+
+ def test_same_track_is_included_at_ten(self, monkeypatch):
+ monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 10.0)
+ state.tracks_stale_skipped = 0
+ now_ms = 1_000_000
+ tracker = self._tracker(self._track(now_ms - 5000))
+ views = confirmed_track_views(tracker, now_ts_ms=now_ms)
+ assert [v["track_id"] for v in views] == ["trk-stale"]
+ assert state.tracks_stale_skipped == 0
+
+ def test_zero_disables_the_filter(self, monkeypatch):
+ monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 0.0)
+ state.tracks_stale_skipped = 0
+ now_ms = 1_000_000
+ tracker = self._tracker(self._track(now_ms - 600_000))
+ assert len(confirmed_track_views(tracker, now_ts_ms=now_ms)) == 1
+ assert state.tracks_stale_skipped == 0
+
+ def test_no_frame_time_disables_the_filter(self, monkeypatch):
+ """The bench and the ADS-B-seeding tests call without a frame time;
+ wall clock is not a substitute, so those callers stay unfiltered."""
+ monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0)
+ state.tracks_stale_skipped = 0
+ tracker = self._tracker(self._track(0))
+ assert len(confirmed_track_views(tracker)) == 1
+ assert state.tracks_stale_skipped == 0
+
+ def test_fresh_track_survives_beside_a_stale_one(self, monkeypatch):
+ """Staleness is per track, not per tracker — the node keeps
+ contributing whatever it can still actually see."""
+ monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0)
+ state.tracks_stale_skipped = 0
+ now_ms = 1_000_000
+ tracker = self._tracker(
+ self._track(now_ms - 5000, track_id="gone"),
+ self._track(now_ms - 500, track_id="here"),
+ )
+ views = confirmed_track_views(tracker, now_ts_ms=now_ms)
+ assert [v["track_id"] for v in views] == ["here"]
+ assert state.tracks_stale_skipped == 1
+
+ def test_tentative_is_still_excluded_regardless_of_freshness(self, monkeypatch):
+ """The TENTATIVE filter is unchanged and independent: a brand-new
+ TENTATIVE track's newest detection is as fresh as it gets, and it must
+ still not reach association."""
+ monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0)
+ state.tracks_stale_skipped = 0
+ now_ms = 1_000_000
+ tracker = self._tracker(self._track(now_ms, status=TrackState.TENTATIVE))
+ assert confirmed_track_views(tracker, now_ts_ms=now_ms) == []
+ # Skipped as TENTATIVE, not as stale — the counter must stay clean so
+ # it only ever means "an aircraft left this node's cone".
+ assert state.tracks_stale_skipped == 0
+
+
class TestMultinodeToAircraft:
def test_basic_conversion(self):
r = {
diff --git a/backend/tests/test_mlat_history.py b/backend/tests/test_mlat_history.py
index e03d9da4..30e8495b 100644
--- a/backend/tests/test_mlat_history.py
+++ b/backend/tests/test_mlat_history.py
@@ -777,3 +777,293 @@ def test_known_lane_records_are_not_sampled_by_this_path(self):
extra={"known_lane": True, "label": "truth_match", "published": True},
)
assert not state.accuracy_samples
+
+
+def _register_geo(node_id, beam_azimuth_deg, rx_lat=LAT, rx_lon=LON, max_range_km=50.0):
+ """Register one node geometry with the associator, aimed as given.
+
+ The contamination stamp asks the associator's own visibility predicate,
+ so a test node has to exist there rather than in a config dict.
+ """
+ from retina_analytics.association import NodeGeometry
+
+ geo = NodeGeometry(
+ node_id=node_id,
+ rx_lat=rx_lat,
+ rx_lon=rx_lon,
+ rx_alt_km=0.0,
+ tx_lat=rx_lat + 0.5,
+ tx_lon=rx_lon + 0.5,
+ tx_alt_km=0.3,
+ beam_azimuth_deg=beam_azimuth_deg,
+ beam_width_deg=41.0,
+ max_range_km=max_range_km,
+ )
+ state.node_associator.node_geometries[node_id] = geo
+ return geo
+
+
+class TestForeignNodeStamp:
+ """A dark record matched to ground truth says which of its own nodes
+ could not have seen that aircraft.
+
+ Cluster contamination — a solver candidate assembled from tracks of two
+ different aircraft — is the dark lane's largest known defect, and until
+ now it was measurable only offline. The verdict is the associator's own
+ visibility predicate, the same one known-lane claiming gates on.
+ """
+
+ def setup_method(self):
+ state._reset_for_tests()
+ solver_mod._reset_for_tests()
+
+ def teardown_method(self):
+ solver_mod._reset_for_tests()
+
+ def _run(self, contributing=("n_in", "n_out"), **extra):
+ return solver_mod._process_solver_item(
+ (dict(_CONFIRMED_N2), {}, time.time()),
+ _solve_fn(contributing_node_ids=list(contributing), **extra),
+ )
+
+ def test_a_node_aimed_away_is_named_foreign(self):
+ # Ground truth sits due north of both nodes; n_in is aimed at it and
+ # n_out at the opposite bearing.
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=180.0)
+ self._run()
+ rec = state.mlat_solve_history[0]
+ assert rec["gt_hex"] == "abc123"
+ assert rec["foreign_node_ids"] == ["n_out"]
+ assert rec["contaminated"] is True
+
+ def test_all_nodes_in_cone_is_not_contaminated(self):
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=10.0)
+ self._run()
+ rec = state.mlat_solve_history[0]
+ assert rec["foreign_node_ids"] == []
+ assert rec["contaminated"] is False
+
+ def test_a_node_out_of_range_is_foreign(self):
+ """Range, not only bearing: the predicate applies whole."""
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=0.0, max_range_km=1.0)
+ self._run()
+ assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_out"]
+
+ def test_trimmed_nodes_are_judged_too(self):
+ """A node dropped by _trim_and_resolve is exactly the contamination
+ this measures — excluding it would hide every case trimming already
+ rescued."""
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_trimmed", beam_azimuth_deg=180.0)
+ solver_mod._record_solve_history(
+ "published",
+ dict(_CONFIRMED_N2),
+ {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2, "contributing_node_ids": ["n_in"]},
+ solve_key="mn-dark-1",
+ raw_lat=LAT,
+ raw_lon=LON,
+ extra={"trimmed_node_ids": ["n_trimmed"], "trim_rounds": 1},
+ )
+ assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_trimmed"]
+
+ def test_no_ground_truth_means_no_stamp(self):
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=180.0)
+ self._run()
+ rec = state.mlat_solve_history[0]
+ assert "foreign_node_ids" not in rec
+ assert "contaminated" not in rec
+
+ def test_unregistered_nodes_are_not_stamped_clean(self):
+ """Nothing judgeable is an abstention, not innocence."""
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ self._run()
+ rec = state.mlat_solve_history[0]
+ assert rec["gt_hex"] == "abc123"
+ assert "foreign_node_ids" not in rec
+
+ def test_an_adsb_record_is_not_stamped(self):
+ """Dark lane only — the tagged lane's identity is not in doubt."""
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=180.0)
+ s_in = dict(_CONFIRMED_N2, adsb_hex="abc123")
+ solver_mod._process_solver_item(
+ (s_in, {}, time.time()),
+ _solve_fn(contributing_node_ids=["n_in", "n_out"]),
+ )
+ assert "foreign_node_ids" not in state.mlat_solve_history[0]
+
+
+class TestLaneFilterAndPerLaneCap:
+ """?lane= and ?limit= on /api/test/mlat-history.
+
+ The flat records[:1000] cap made the response a race between lanes: the
+ known lane writes ~16x the dark lane's volume, so a 30 min request held
+ only the newest ~6 min of dark records and the rest of the window read as
+ a quiet period. The cap is now per lane.
+ """
+
+ def setup_method(self):
+ state._reset_for_tests()
+ solver_mod._reset_for_tests()
+
+ def teardown_method(self):
+ solver_mod._reset_for_tests()
+
+ def _client(self):
+ from main import app
+
+ return TestClient(app)
+
+ def _dark(self, n=1):
+ for _ in range(n):
+ solver_mod._record_solve_history(
+ "published",
+ {"n_nodes": 3},
+ {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3},
+ solve_key="mn-dark-1",
+ raw_lat=LAT,
+ raw_lon=LON,
+ )
+
+ def _known(self, n=1):
+ for _ in range(n):
+ solver_mod._record_solve_history(
+ "known_truth_match",
+ {"n_nodes": 2, "adsb_hex": "abc123", "initial_guess": {"lat": LAT, "lon": LON}},
+ {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2},
+ extra={"known_lane": True, "label": "truth_match", "published": False},
+ )
+
+ def _adsb(self, n=1):
+ for _ in range(n):
+ solver_mod._record_solve_history(
+ "published",
+ {"n_nodes": 3, "adsb_hex": "abc123"},
+ {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3},
+ solve_key="mn-adsb-abc123",
+ raw_lat=LAT,
+ raw_lon=LON,
+ )
+
+ def test_default_lane_is_all_and_counts_every_lane(self):
+ self._dark()
+ self._known()
+ self._adsb()
+ data = self._client().get("/api/test/mlat-history?all=1").json()
+ assert data["lane"] == "all"
+ assert data["lane_counts"] == {"dark": 1, "known": 1, "adsb": 1, "dark_follow": 0}
+ assert data["n_records"] == 3
+
+ def test_lane_dark_returns_only_dark_records(self):
+ self._dark(2)
+ self._known(3)
+ self._adsb(1)
+ data = self._client().get("/api/test/mlat-history?all=1&lane=dark").json()
+ assert data["n_records"] == 2
+ assert data["lane_counts"] == {"dark": 2, "known": 0, "adsb": 0, "dark_follow": 0}
+ assert all(r["solve_key"] == "mn-dark-1" for r in data["records"])
+
+ def test_lane_known_returns_only_known_records(self):
+ self._dark(2)
+ self._known(3)
+ data = self._client().get("/api/test/mlat-history?all=1&lane=known").json()
+ assert data["n_records"] == 3
+ assert all(r["known_lane"] for r in data["records"])
+
+ def test_unknown_lane_is_rejected(self):
+ assert self._client().get("/api/test/mlat-history?all=1&lane=bogus").status_code == 400
+
+ def test_known_volume_cannot_evict_dark_records_from_the_response(self):
+ """The bug the per-lane cap fixes, at 1/500 scale."""
+ self._dark(2)
+ self._known(20)
+ data = self._client().get("/api/test/mlat-history?all=1&limit=2").json()
+ # 2 dark + 2 known survive the cap; the flat cap would have returned
+ # the 2 newest records overall, both known.
+ lanes = [("known" if r.get("known_lane") else "dark") for r in data["records"]]
+ assert sorted(lanes) == ["dark", "dark", "known", "known"]
+ # n_records / lane_counts stay pre-cap so truncation is legible.
+ assert data["n_records"] == 22
+ assert data["lane_counts"] == {"dark": 2, "known": 20, "adsb": 0, "dark_follow": 0}
+
+ def test_limit_is_clamped_to_the_maximum(self):
+ self._dark(3)
+ data = self._client().get("/api/test/mlat-history?all=1&limit=99999").json()
+ assert len(data["records"]) == 3
+
+ def test_hex_lookup_reports_the_lane_block_too(self):
+ self._dark()
+ rec = state.mlat_solve_history[0]
+ data = self._client().get(f"/api/test/mlat-history?hex={rec['solver_hex']}").json()
+ assert data["lane"] == "all"
+ assert data["lane_counts"]["dark"] == 1
+
+
+class TestResolveSkipDump:
+ """?kind=resolve_skips dumps the solver's skip deque.
+
+ A skip is not a solve outcome and must not be written into the
+ solve-history deques: on the live fleet skips outrun dark records roughly
+ two to one and would evict exactly the records an investigation needs.
+ """
+
+ def setup_method(self):
+ state._reset_for_tests()
+ solver_mod._reset_for_tests()
+
+ def teardown_method(self):
+ solver_mod._reset_for_tests()
+
+ def _client(self):
+ from main import app
+
+ return TestClient(app)
+
+ def _skip(self, track_ids=("a1", "b1"), n_nodes=3, **s_in):
+ now = time.time()
+ s = dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids), **s_in)
+ solver_mod._record_resolve_slot(list(track_ids), n_nodes, now)
+ covered, blocking = solver_mod._resolve_slot_covered(dict(s), now)
+ assert covered is True
+ solver_mod._record_resolve_skip(dict(s), now, blocking)
+
+ def test_skip_records_the_blocking_claim(self):
+ self._skip()
+ data = self._client().get("/api/test/mlat-history?kind=resolve_skips").json()
+ assert data["kind"] == "resolve_skips"
+ assert data["n_records"] == 1
+ rec = data["records"][0]
+ assert rec["lane"] == "dark"
+ assert rec["track_ids"] == ["a1", "b1"]
+ assert rec["n_nodes"] == 3
+ assert {b["track_id"] for b in rec["blocking"]} == {"a1", "b1"}
+ assert all(b["held_n"] == 3 for b in rec["blocking"])
+
+ def test_skips_do_not_land_in_the_solve_history(self):
+ self._skip()
+ assert not state.mlat_solve_history
+ assert not state.mlat_solve_history_known
+
+ def test_lane_filter_applies_to_skips(self):
+ self._skip(track_ids=("a1", "b1"))
+ self._skip(track_ids=("a2", "b2"), adsb_hex="abc123")
+ assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=dark").json()["n_records"] == 1
+ assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=adsb").json()["n_records"] == 1
+ assert self._client().get("/api/test/mlat-history?kind=resolve_skips").json()["lane_counts"] == {
+ "dark": 1,
+ "known": 0,
+ "adsb": 1,
+ "dark_follow": 0,
+ }
+
+ def test_unknown_kind_is_rejected(self):
+ assert self._client().get("/api/test/mlat-history?kind=bogus").status_code == 400
diff --git a/backend/tests/test_mn_lifetime.py b/backend/tests/test_mn_lifetime.py
index 5b648ca1..d3f8bd49 100644
--- a/backend/tests/test_mn_lifetime.py
+++ b/backend/tests/test_mn_lifetime.py
@@ -2,8 +2,8 @@
Two problems in how state.multinode_tracks renders as mn-* aircraft:
-- a track solved exactly once dead-reckons for the full 60 s entry lifetime
- (30 s DR cap) even though nothing has confirmed it is a real aircraft and
+- a track solved exactly once dead-reckons for the full entry lifetime
+ (MN_DR_CAP_S DR cap) even though nothing has confirmed it is a real aircraft and
not a mirror-point/wrong-frame ghost. solver.py now stamps a solve_count
onto every published entry, and aircraft_feed.py withholds rendering until
a 2-node track has MN_N2_MIN_SOLVES solves, and caps an n>=3 one-shot's
@@ -425,14 +425,18 @@ def test_n3_one_shot_within_ttl_is_rendered(self):
assert len(self._mn_aircraft()) == 1
def test_n3_one_shot_past_ttl_is_not_rendered_but_not_expired(self):
- # Past MN_ONESHOT_TTL_S (15 s) but well inside the 60 s entry expiry,
- # so this exercises the display gate and not staleness.
+ # Past MN_ONESHOT_TTL_S (15 s) but inside the dark lane's
+ # MN_DARK_EXPIRY_S (30 s), so this exercises the display gate and not
+ # staleness.
state.multinode_tracks["k"] = _mn_entry(n_nodes=3, age_s=20.0, solve_count=1)
assert self._mn_aircraft() == []
assert "k" in state.multinode_tracks
def test_n3_confirmed_old_entry_is_rendered(self):
- state.multinode_tracks["k"] = _mn_entry(n_nodes=3, age_s=40.0, solve_count=2)
+ # 25 s: old enough to be past every display gate, young enough to sit
+ # inside the dark lane's MN_DARK_EXPIRY_S — the key here is bare "k",
+ # which is not mn-adsb-* and so expires on the dark budget.
+ state.multinode_tracks["k"] = _mn_entry(n_nodes=3, age_s=25.0, solve_count=2)
assert len(self._mn_aircraft()) == 1
def test_n2_gate_disabled_via_min_solves_1(self, monkeypatch):
diff --git a/backend/tests/test_solve_uncertainty.py b/backend/tests/test_solve_uncertainty.py
index 1d7628a2..c4b335c0 100644
--- a/backend/tests/test_solve_uncertainty.py
+++ b/backend/tests/test_solve_uncertainty.py
@@ -121,16 +121,19 @@ def test_growth_adds_in_quadrature(self):
def test_zero_age_is_the_solve_sigma(self):
assert su.grown_sigma_m(650.0, 25.0, 0.0) == pytest.approx(650.0)
- def test_growth_is_capped_at_60s(self):
- at_60 = su.grown_sigma_m(180.0, 25.0, 60.0)
- assert su.grown_sigma_m(180.0, 25.0, 600.0) == pytest.approx(at_60)
- assert at_60 == pytest.approx(math.sqrt(180.0**2 + 1500.0**2))
+ def test_growth_is_capped_at_the_growth_horizon(self):
+ # The horizon mirrors the frontend's UNCERTAINTY_DR_CAP_S (30 s),
+ # which in turn mirrors MN_DARK_EXPIRY_S.
+ assert su._GROWTH_MAX_AGE_S == 30.0
+ at_cap = su.grown_sigma_m(180.0, 25.0, su._GROWTH_MAX_AGE_S)
+ assert su.grown_sigma_m(180.0, 25.0, 600.0) == pytest.approx(at_cap)
+ assert at_cap == pytest.approx(math.sqrt(180.0**2 + 750.0**2))
def test_negative_age_does_not_shrink_or_grow(self):
assert su.grown_sigma_m(650.0, 25.0, -5.0) == pytest.approx(650.0)
def test_growth_is_monotonic_up_to_the_cap(self):
- seq = [su.grown_sigma_m(210.0, 30.0, t) for t in (0.0, 5.0, 20.0, 59.0)]
+ seq = [su.grown_sigma_m(210.0, 30.0, t) for t in (0.0, 5.0, 20.0, 29.0)]
assert seq == sorted(seq)
assert seq[0] < seq[-1]
diff --git a/backend/tests/test_solver_alt_mode.py b/backend/tests/test_solver_alt_mode.py
new file mode 100644
index 00000000..2c9eec02
--- /dev/null
+++ b/backend/tests/test_solver_alt_mode.py
@@ -0,0 +1,344 @@
+"""SOLVER_ALT_MODE: how the n>=3 solve gets its altitude.
+
+sweep (the default) calls the LM once per fixed altitude layer and keeps the
+lowest rms_delay — six process-pool round trips, and an altitude quantised to
+a ladder 2 km wide, which puts up to 1 km of error into the residual the
+reject gate reads. free makes ONE call to the geolocator's multi-start
+helper, which solves altitude as a sixth unknown from SOLVER_FREE_ALT_STARTS
+start layers — one by default, the layer nearest the association guess.
+
+These tests are about the routing, not the physics: the geolocator's own
+suite (tests/test_free_altitude.py there) measures what the free solve
+actually fits. What matters here is that the default is byte-identical to
+the sweep, that free spends one call and not six, that trimming re-solves
+under the same mode, and that both modes leave enough on the history record
+to be compared live.
+"""
+
+import time
+
+import pytest
+
+from core import state
+from services import frame_processor
+from services.tasks import solver as solver_mod
+
+LAT, LON = 35.0, -82.0
+
+
+def _s_in(node_ids, alt_km=9.0, **overrides):
+ s_in = {
+ "initial_guess": {"lat": LAT, "lon": LON, "alt_km": alt_km},
+ "measurements": [{"node_id": nid, "delay_us": 10.0, "doppler_hz": 1.0, "snr": 15.0} for nid in node_ids],
+ "n_nodes": len(node_ids),
+ "timestamp_ms": int(time.time() * 1000),
+ }
+ s_in.update(overrides)
+ return s_in
+
+
+def _stub_result(node_ids, rms_delay=0.5, **overrides):
+ result = {
+ "success": True,
+ "lat": LAT,
+ "lon": LON,
+ "alt_m": 9000.0,
+ "timestamp_ms": int(time.time() * 1000),
+ "vel_east": 0.0,
+ "vel_north": 0.0,
+ "rms_delay": rms_delay,
+ "rms_doppler": 5.0,
+ "n_nodes": len(node_ids),
+ "n_measurements": len(node_ids),
+ "contributing_node_ids": list(node_ids),
+ }
+ result.update(overrides)
+ return result
+
+
+class _Recorder:
+ """A solve_fn / multistart_fn that records every call it is given."""
+
+ def __init__(self, result_for):
+ self.calls: list[tuple] = []
+ self._result_for = result_for
+
+ def __call__(self, s_in, node_cfgs, *rest):
+ self.calls.append((s_in, node_cfgs, rest))
+ nodes = tuple(m["node_id"] for m in s_in["measurements"])
+ return self._result_for(nodes, s_in, *rest)
+
+
+class _AltModeBase:
+ def setup_method(self):
+ state._reset_for_tests()
+ solver_mod._reset_for_tests()
+
+ def teardown_method(self):
+ solver_mod._reset_for_tests()
+
+
+class TestFreeAltStarts:
+ """The starts handed to the multi-start helper: SOLVER_FREE_ALT_STARTS of
+ them, one by default."""
+
+ @pytest.mark.parametrize(
+ "alt_km,expected",
+ [
+ (9.0, [9.0]),
+ (7.0, [7.0]),
+ (8.2, [9.0]),
+ (1.5, [1.5]),
+ # Off the ends of the ladder: still the nearest layer, not nothing.
+ (0.4, [1.5]),
+ (40.0, [11.0]),
+ ],
+ )
+ def test_one_start_at_the_nearest_layer_by_default(self, alt_km, expected):
+ assert state.SOLVER_FREE_ALT_STARTS == 1
+ assert solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) == expected
+
+ def test_an_adsb_altitude_in_the_ladder_is_the_start(self):
+ """_solve_best_altitude splices a non-layer altitude (ADS-B) into the
+ layers, and the starts are taken over that spliced list — so the single
+ default start is that exact altitude, which is what the sweep would
+ have pinned too."""
+ layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4]))
+ assert solver_mod._free_alt_starts(8.4, layers) == [8.4]
+
+ @pytest.mark.parametrize(
+ "alt_km,expected",
+ [
+ (9.0, [7.0, 9.0, 11.0]),
+ (7.0, [5.0, 7.0, 9.0]),
+ (8.2, [7.0, 9.0, 11.0]),
+ # Clamped at the ends: the ladder's first and last layers still get
+ # three starts, not one or two.
+ (1.5, [1.5, 3.0, 5.0]),
+ (0.4, [1.5, 3.0, 5.0]),
+ (11.0, [7.0, 9.0, 11.0]),
+ (40.0, [7.0, 9.0, 11.0]),
+ ],
+ )
+ def test_three_starts_are_the_window_around_the_nearest_layer(self, alt_km, expected, monkeypatch):
+ """The pre-default behaviour, still reachable by configuration."""
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3)
+ starts = solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM)
+ assert starts == expected
+ assert len(starts) == 3
+
+ def test_three_starts_window_the_spliced_adsb_altitude(self, monkeypatch):
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3)
+ layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4]))
+ assert solver_mod._free_alt_starts(8.4, layers) == [7.0, 8.4, 9.0]
+
+ @pytest.mark.parametrize("configured", [0, -3])
+ def test_fewer_than_one_start_still_starts_somewhere(self, configured, monkeypatch):
+ """A count below one would leave the LM no start at all, so it clamps
+ rather than raises: a mis-set env degrades to a working solve."""
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", configured)
+ assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == [9.0]
+
+ def test_more_starts_than_layers_is_every_layer(self, monkeypatch):
+ """The other clamp: a count past the end of the ladder would slice
+ short of it, quietly dropping starts that were asked for."""
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 99)
+ assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == solver_mod._SOLVER_ALT_LAYERS_KM
+
+ def test_no_layers_gives_no_starts(self):
+ assert solver_mod._free_alt_starts(9.0, []) == []
+
+
+class TestSweepIsTheDefault(_AltModeBase):
+ def test_sweep_calls_the_lm_once_per_layer_and_never_the_multistart(self):
+ nodes = ["n1", "n2", "n3"]
+ solve = _Recorder(lambda n, s, *r: _stub_result(n))
+ multistart = _Recorder(lambda n, s, *r: pytest.fail("multistart called in sweep mode"))
+
+ result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart)
+
+ assert result is not None and result["success"]
+ assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM)
+ assert [c[0]["initial_guess"]["alt_km"] for c in solve.calls] == solver_mod._SOLVER_ALT_LAYERS_KM
+ assert multistart.calls == []
+ assert state.SOLVER_ALT_MODE == "sweep"
+
+
+class TestFreeMode(_AltModeBase):
+ def setup_method(self):
+ super().setup_method()
+ self._saved_mode = state.SOLVER_ALT_MODE
+ state.SOLVER_ALT_MODE = "free"
+
+ def teardown_method(self):
+ state.SOLVER_ALT_MODE = self._saved_mode
+ super().teardown_method()
+
+ def test_one_multistart_call_with_one_start(self):
+ nodes = ["n1", "n2", "n3"]
+ solve = _Recorder(lambda n, s, *r: pytest.fail("sweep ran in free mode"))
+ multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4]))
+
+ result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart)
+
+ assert result is not None and result["success"]
+ assert solve.calls == []
+ assert len(multistart.calls) == 1
+ (_, _, rest) = multistart.calls[0]
+ assert rest == ([9.0],)
+
+ def test_an_adsb_guess_altitude_is_the_start(self):
+ """A non-layer initial_guess altitude is spliced into the ladder and
+ becomes the start itself — the free-mode analogue of the sweep's extra
+ layer, and the one exact altitude the candidate has."""
+ nodes = ["n1", "n2", "n3"]
+ multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free"))
+
+ solver_mod._solve_best_altitude(_s_in(nodes, alt_km=8.437), {}, lambda s, c: None, multistart)
+
+ assert multistart.calls[0][2] == ([8.437],)
+
+ def test_the_start_count_is_configurable(self, monkeypatch):
+ """SOLVER_FREE_ALT_STARTS buys back the neighbour window for a geometry
+ whose single start lands on the wrong side of an ellipse."""
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3)
+ multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free"))
+
+ solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, lambda s, c: None, multistart)
+
+ assert len(multistart.calls) == 1
+ assert multistart.calls[0][2] == ([7.0, 9.0, 11.0],)
+
+ def test_n2_keeps_the_sweep(self):
+ """Altitude is unobservable at n=2 — the free path is not entered even
+ with the mode on."""
+ nodes = ["n1", "n2"]
+ solve = _Recorder(lambda n, s, *r: _stub_result(n))
+ multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken at n=2"))
+
+ result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart)
+
+ assert result is not None
+ assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM)
+ assert multistart.calls == []
+
+ def test_a_failed_multistart_is_a_failed_solve(self):
+ """No silent fall back to the sweep: three starts producing nothing is
+ the same verdict as every layer producing nothing."""
+ solve = _Recorder(lambda n, s, *r: pytest.fail("swept after a failed multistart"))
+ multistart = _Recorder(lambda n, s, *r: None)
+
+ assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart) is None
+
+ def test_history_carries_the_mode_and_the_per_start_residuals(self):
+ nodes = ["n1", "n2", "n3"]
+ multistart = _Recorder(
+ lambda n, s, *r: _stub_result(
+ n, altitude_mode="free", rms_by_start=[1.2345, None, 0.4321], alt_starts_km=[5.0, 7.0, 9.0]
+ )
+ )
+ solver_mod._process_solver_item(
+ (_s_in(nodes), {}, time.time()),
+ lambda s, c: pytest.fail("sweep ran in free mode"),
+ multistart_fn=multistart,
+ )
+
+ rec = state.mlat_solve_history[-1]
+ assert rec["outcome"] == "published"
+ assert rec["altitude_mode"] == "free"
+ assert rec["alt_starts_km"] == [5.0, 7.0, 9.0]
+ assert rec["alt_start_rms_us"] == [1.234, None, 0.432]
+
+ def test_a_single_start_still_records_its_residual(self):
+ """The comparison channel does not depend on there being several
+ starts: one start records a one-element list, not a bare number or
+ nothing at all."""
+ nodes = ["n1", "n2", "n3"]
+ multistart = _Recorder(
+ lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4321], alt_starts_km=[9.0])
+ )
+ solver_mod._process_solver_item(
+ (_s_in(nodes), {}, time.time()),
+ lambda s, c: pytest.fail("sweep ran in free mode"),
+ multistart_fn=multistart,
+ )
+
+ rec = state.mlat_solve_history[-1]
+ assert rec["alt_starts_km"] == [9.0]
+ assert rec["alt_start_rms_us"] == [0.432]
+
+ def test_z_saturation_reaches_the_history(self):
+ nodes = ["n1", "n2", "n3"]
+ multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", z_saturated=True))
+ solver_mod._process_solver_item((_s_in(nodes), {}, time.time()), lambda s, c: None, multistart_fn=multistart)
+ assert state.mlat_solve_history[-1]["z_saturated"] is True
+
+ def test_trimming_re_solves_through_the_multistart(self):
+ """A trim round must use the mode its first solve used, or the rms it
+ compares against the previous round is a different quantity."""
+ full = ["n1", "n2", "n3", "n4", "bad"]
+ trimmed = ["n1", "n2", "n3", "n4"]
+
+ def _result(nodes, s_in, *rest):
+ if "bad" in nodes:
+ return _stub_result(
+ nodes,
+ rms_delay=8.0,
+ altitude_mode="free",
+ per_node_delay_res_us={n: (12.0 if n == "bad" else 0.5) for n in nodes},
+ )
+ return _stub_result(
+ nodes,
+ rms_delay=0.8,
+ altitude_mode="free",
+ per_node_delay_res_us={n: 0.3 for n in nodes},
+ )
+
+ multistart = _Recorder(_result)
+ result = solver_mod._process_solver_item(
+ (_s_in(full), {}, time.time()),
+ lambda s, c: pytest.fail("sweep ran during a free-mode trim"),
+ multistart_fn=multistart,
+ )
+
+ assert result is not None and result["n_nodes"] == 4
+ assert sorted(m["node_id"] for m in multistart.calls[-1][0]["measurements"]) == trimmed
+ rec = state.mlat_solve_history[-1]
+ assert rec["outcome"] == "published"
+ assert rec["trimmed_node_ids"] == ["bad"]
+ assert rec["altitude_mode"] == "free"
+
+ def test_an_unrecognised_mode_would_sweep(self):
+ """The flag degrades to the inert mode, like its siblings — asserted on
+ the resolution rule rather than by re-importing core.state."""
+ state.SOLVER_ALT_MODE = "definitely-not-a-mode"
+ solve = _Recorder(lambda n, s, *r: _stub_result(n))
+ multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken for a bad mode"))
+ assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart)
+ assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM)
+
+
+class TestConfigsForSolverInput:
+ """Only the configs a candidate can reach are queued with it.
+
+ The pool is a spawn pool, so whatever is queued is pickled and shipped on
+ every solve — 58 fleet configs against a candidate's 2-8 measurements.
+ """
+
+ _FLEET = {f"n{i}": {"rx_lat": 35.0 + i, "rx_lon": -82.0} for i in range(8)}
+
+ def test_restricted_to_the_measurement_nodes(self):
+ s_in = _s_in(["n1", "n3", "n5"])
+ cfgs = frame_processor.configs_for_solver_input(self._FLEET, s_in)
+ assert sorted(cfgs) == ["n1", "n3", "n5"]
+ assert cfgs["n3"] is self._FLEET["n3"]
+
+ def test_unknown_measurement_nodes_are_simply_absent(self):
+ """A measurement from a node with no config is the case
+ solve_multinode already handles by skipping it — not an error here."""
+ cfgs = frame_processor.configs_for_solver_input(self._FLEET, _s_in(["n1", "ghost"]))
+ assert sorted(cfgs) == ["n1"]
+
+ def test_no_measurements_gives_nothing(self):
+ assert frame_processor.configs_for_solver_input(self._FLEET, {"measurements": []}) == {}
+ assert frame_processor.configs_for_solver_input(self._FLEET, {}) == {}
diff --git a/backend/tests/test_solver_stats.py b/backend/tests/test_solver_stats.py
index 67bc599c..9ceeeac0 100644
--- a/backend/tests/test_solver_stats.py
+++ b/backend/tests/test_solver_stats.py
@@ -37,6 +37,7 @@ def _rec(
adsb_hex=None,
known_lane=False,
displacement_km=None,
+ lane=None,
):
return {
"ts_ms": int((time.time() - age_s) * 1000),
@@ -48,6 +49,7 @@ def _rec(
"adsb_hex": adsb_hex,
"known_lane": known_lane,
"displacement_km": displacement_km,
+ "lane": lane,
}
@@ -257,6 +259,16 @@ def test_consensus_and_counters_reflect_state(self):
state.solver_consensus_fallback = 9
state.solver_consensus_shadow = 10
state.solver_vel_untrusted_published = 11
+ state.dark_follow_targets = 13
+ state.dark_follow_claims = 14
+ state.dark_follow_inputs = 15
+ state.dark_follow_published = 16
+ state.dark_follow_dropped = 17
+ state.dark_bottomup_shadowed = 18
+ state.tracks_stale_skipped = 13
+ state.solver_epoch_align_skipped = 14
+ state.solver_resolve_skips_dark = 9
+ state.node_frames_rate_limited = 13
out = _solver_window_stats(10.0)
assert out["counters"] == {
"successes": 5,
@@ -265,9 +277,19 @@ def test_consensus_and_counters_reflect_state(self):
"solver_trimmed": 3,
"stale_drops": 4,
"resolve_skips": 12,
+ "tracks_stale_skipped": 13,
+ "epoch_align_skipped": 14,
+ "resolve_skips_dark": 9,
"queue_drops": 6,
+ "node_frames_rate_limited": 13,
"worker_errors": 0,
"vel_untrusted_published": 11,
+ "dark_follow_targets": 13,
+ "dark_follow_claims": 14,
+ "dark_follow_inputs": 15,
+ "dark_follow_published": 16,
+ "dark_follow_dropped": 17,
+ "dark_bottomup_shadowed": 18,
}
assert out["consensus"]["selected"] == 7
assert out["consensus"]["filtered"] == 8
@@ -520,7 +542,7 @@ def test_empty_state_no_division_errors(self):
assert out["position_error_km"] == {"median": None, "p90": None, "n": 0}
assert out["ghosts"]["live_tracks"] == 0
assert out["ghosts"]["precision_pct"] is None
- assert out["lane_split"] == {"dark": 0, "adsb": 0, "known": 0}
+ assert out["lane_split"] == {"dark": 0, "adsb": 0, "known": 0, "dark_follow": 0}
assert out["window_effective_minutes"] == 0.0
@@ -618,7 +640,7 @@ def test_known_records_leave_the_funnel_and_land_in_lane_split(self):
_push(_rec("known_truth_match", known_lane=True, displacement_km=0.5))
_push(_rec("known_ghost", known_lane=True, displacement_km=8.0))
out = _solver_window_stats(10.0)
- assert out["lane_split"] == {"dark": 2, "adsb": 0, "known": 4}
+ assert out["lane_split"] == {"dark": 2, "adsb": 0, "known": 4, "dark_follow": 0}
assert out["attempts"] == 2
assert out["published"]["total"] == 1
assert out["rejects"] == {"total": 1, "by_reason": {"displacement": 1}}
@@ -626,12 +648,24 @@ def test_known_records_leave_the_funnel_and_land_in_lane_split(self):
assert "known_truth_match" not in out["rejects"]["by_reason"]
assert "known_ghost" not in out["rejects"]["by_reason"]
+ def test_follow_records_leave_the_dark_funnel(self):
+ """A dark-follow solve (services/dark_follow.py) is keyed mn-dark-* by
+ design — it is the same aircraft, reached top-down — so only the
+ ``lane`` stamp keeps it out of a bottom-up funnel it is not part of."""
+ _push(_rec("published", solve_key="mn-dark-1"))
+ _push(_rec("published", solve_key="mn-dark-2", lane="dark_follow"))
+ _push(_rec("rejected_displacement", lane="dark_follow"))
+ out = _solver_window_stats(10.0)
+ assert out["lane_split"] == {"dark": 1, "adsb": 0, "known": 0, "dark_follow": 2}
+ assert out["attempts"] == 1
+ assert out["rejects"]["total"] == 0
+
def test_adsb_lane_records_are_counted_but_not_funnelled(self):
_push(_rec("published", solve_key="mn-adsb-a1b2c3"))
_push(_rec("rejected_displacement", adsb_hex="a1b2c3"))
_push(_rec("published", solve_key="mn-dark-1"))
out = _solver_window_stats(10.0)
- assert out["lane_split"] == {"dark": 1, "adsb": 2, "known": 0}
+ assert out["lane_split"] == {"dark": 1, "adsb": 2, "known": 0, "dark_follow": 0}
assert out["attempts"] == 1
assert out["rejects"]["total"] == 0
@@ -788,3 +822,118 @@ def get(self, *a, **kw):
state.multinode_tracks["mn-dark-1"] = {"lat": 35.0, "lon": -82.0}
state.adsb_aircraft["real1"] = _MutatingFix({"lat": 35.009, "lon": -82.0, "last_seen_ms": now_ms})
assert _solver_window_stats(10.0)["ghosts"]["ghost_tracks"] == 0
+
+
+def _skip_rec(lane="dark", age_s=0.0, track_ids=("a1",), n_nodes=3):
+ return {
+ "ts_ms": int((time.time() - age_s) * 1000),
+ "lane": lane,
+ "track_ids": list(track_ids),
+ "n_nodes": n_nodes,
+ "blocking": [{"track_id": track_ids[0], "held_ts": time.time() - age_s, "held_n": n_nodes}],
+ "guess_lat": None,
+ "guess_lon": None,
+ }
+
+
+class TestResolveSkipBlock:
+ """Resolve-slot skips are windowed from their own deque, not from the
+ since-boot counter, so they can be read against the attempts in the same
+ window — the ratio the claim-on-publish fix is judged on."""
+
+ def setup_method(self):
+ state._reset_for_tests()
+
+ def test_totals_split_by_lane(self):
+ for _ in range(3):
+ state.solver_resolve_skips_recent.append(_skip_rec("dark"))
+ state.solver_resolve_skips_recent.append(_skip_rec("adsb"))
+ out = _solver_window_stats(10.0)["resolve_skips"]
+ assert out["total"] == 4
+ assert out["dark"] == 3
+
+ def test_window_excludes_old_skips(self):
+ state.solver_resolve_skips_recent.append(_skip_rec(age_s=20 * 60))
+ state.solver_resolve_skips_recent.append(_skip_rec(age_s=1))
+ assert _solver_window_stats(10.0)["resolve_skips"]["total"] == 1
+
+ def test_attempts_ratio_is_skips_over_dark_attempts(self):
+ for _ in range(4):
+ state.solver_resolve_skips_recent.append(_skip_rec())
+ state.mlat_solve_history.append(_rec("published"))
+ state.mlat_solve_history.append(_rec("rejected_beam"))
+ out = _solver_window_stats(10.0)
+ assert out["attempts"] == 2
+ assert out["resolve_skips"]["attempts_ratio"] == 2.0
+
+ def test_attempts_ratio_is_none_without_attempts(self):
+ state.solver_resolve_skips_recent.append(_skip_rec())
+ assert _solver_window_stats(10.0)["resolve_skips"]["attempts_ratio"] is None
+
+ def test_window_effective_minutes_exposes_a_truncated_deque(self):
+ """The deque is 500 entries against ~50 skips/min live, so a long
+ window IS truncated here even when the solve stores cover it."""
+ state.solver_resolve_skips_recent.append(_skip_rec(age_s=6 * 60))
+ out = _solver_window_stats(30.0)["resolve_skips"]
+ assert 5.9 <= out["window_effective_minutes"] <= 6.1
+
+ def test_a_skip_is_not_an_attempt_or_a_reject(self):
+ """Skips must not leak into the funnel — they never reached a solve."""
+ for _ in range(5):
+ state.solver_resolve_skips_recent.append(_skip_rec())
+ out = _solver_window_stats(10.0)
+ assert out["attempts"] == 0
+ assert out["rejects"]["total"] == 0
+
+
+def _gt_rec(foreign=(), **kw):
+ """A dark record carrying the contamination stamp."""
+ rec = _rec("published", **kw)
+ rec["gt_hex"] = "abc123"
+ rec["foreign_node_ids"] = list(foreign)
+ rec["contaminated"] = bool(foreign)
+ return rec
+
+
+class TestContaminationBlock:
+ """Live cluster contamination: of the dark records that matched ground
+ truth, how many carried a node that could not see the aircraft."""
+
+ def setup_method(self):
+ state._reset_for_tests()
+
+ def test_pct_and_mean_over_judged_records(self):
+ state.mlat_solve_history.append(_gt_rec(foreign=["n1"]))
+ state.mlat_solve_history.append(_gt_rec(foreign=["n1", "n2"]))
+ state.mlat_solve_history.append(_gt_rec(foreign=[]))
+ state.mlat_solve_history.append(_gt_rec(foreign=[]))
+ out = _solver_window_stats(10.0)["contamination"]
+ assert out["records_with_gt"] == 4
+ assert out["contaminated"] == 2
+ assert out["pct"] == 50.0
+ assert out["foreign_nodes_per_record"] == 0.75
+
+ def test_unstamped_records_are_out_of_the_denominator(self):
+ """No GT match, or no judgeable node geometry, is an abstention — not
+ a clean record."""
+ state.mlat_solve_history.append(_gt_rec(foreign=["n1"]))
+ state.mlat_solve_history.append(_rec("published"))
+ out = _solver_window_stats(10.0)["contamination"]
+ assert out["records_with_gt"] == 1
+ assert out["pct"] == 100.0
+
+ def test_empty_window_abstains_rather_than_reporting_zero(self):
+ out = _solver_window_stats(10.0)["contamination"]
+ assert out == {
+ "records_with_gt": 0,
+ "contaminated": 0,
+ "pct": None,
+ "foreign_nodes_per_record": None,
+ }
+
+ def test_known_lane_records_are_not_counted(self):
+ """Dark lane only, like every other block in the funnel."""
+ rec = _gt_rec(foreign=["n1"])
+ rec["known_lane"] = True
+ state.mlat_solve_history_known.append(rec)
+ assert _solver_window_stats(10.0)["contamination"]["records_with_gt"] == 0
diff --git a/backend/tests/test_solver_trimming.py b/backend/tests/test_solver_trimming.py
index 93504a5a..0f529848 100644
--- a/backend/tests/test_solver_trimming.py
+++ b/backend/tests/test_solver_trimming.py
@@ -827,3 +827,76 @@ def solve_fn(_s_in, _cfgs):
assert state.fov_shadow_agree == 1
assert state.fov_shadow_would_pass == 0
assert state.fov_shadow_would_reject == 0
+
+
+class TestTrimmedTracksAreNotClaimed(_TrimmingTestBase):
+ """A trimmed node's tracks must not take a re-solve claim.
+
+ The claim says "this aircraft is on the map at this width". A node
+ dropped for a bad residual contributed nothing to the published position
+ and its track was probably a different aircraft's — claiming it would
+ suppress that aircraft's own candidate on the strength of a measurement
+ this solve threw away.
+ """
+
+ _FULL = ["n1", "n2", "n3", "n4", "bad"]
+ _TRIM = ["n1", "n2", "n3", "n4"]
+
+ def test_the_dropped_nodes_track_is_left_unclaimed(self):
+ table = {
+ frozenset(self._FULL): _stub_result(
+ self._FULL,
+ rms_delay=8.0,
+ per_node={"n1": 0.5, "n2": 0.5, "n3": 0.5, "n4": 0.5, "bad": 12.0},
+ ),
+ frozenset(self._TRIM): _stub_result(
+ self._TRIM,
+ rms_delay=0.8,
+ per_node={"n1": 0.3, "n2": 0.3, "n3": 0.3, "n4": 0.3},
+ ),
+ }
+ s_in = _s_in(
+ self._FULL,
+ track_ids=["t1", "t2", "t3", "t4", "tbad"],
+ track_ids_by_node={
+ "n1": ["t1"],
+ "n2": ["t2"],
+ "n3": ["t3"],
+ "n4": ["t4"],
+ "bad": ["tbad"],
+ },
+ )
+ result = self._run(s_in, _stub_solve_fn(table))
+ assert result is not None and result["success"]
+ assert result["source_track_ids"] == ["t1", "t2", "t3", "t4"]
+ assert set(solver_mod._RECENT_SOLVES) == {"t1", "t2", "t3", "t4"}
+
+ def test_a_candidate_built_on_the_dropped_track_still_runs(self):
+ """The other half of the same claim: whoever "tbad" really belongs to
+ keeps its slot."""
+ table = {
+ frozenset(self._FULL): _stub_result(
+ self._FULL,
+ rms_delay=8.0,
+ per_node={"n1": 0.5, "n2": 0.5, "n3": 0.5, "n4": 0.5, "bad": 12.0},
+ ),
+ frozenset(self._TRIM): _stub_result(
+ self._TRIM,
+ rms_delay=0.8,
+ per_node={"n1": 0.3, "n2": 0.3, "n3": 0.3, "n4": 0.3},
+ ),
+ }
+ s_in = _s_in(
+ self._FULL,
+ track_ids=["t1", "t2", "t3", "t4", "tbad"],
+ track_ids_by_node={
+ "n1": ["t1"],
+ "n2": ["t2"],
+ "n3": ["t3"],
+ "n4": ["t4"],
+ "bad": ["tbad"],
+ },
+ )
+ self._run(s_in, _stub_solve_fn(table))
+ neighbour = {"n_nodes": 2, "track_ids": ["tbad", "tother"]}
+ assert solver_mod._resolve_slot_covered(neighbour, time.time())[0] is False
diff --git a/backend/tests/test_solver_worker.py b/backend/tests/test_solver_worker.py
index 6f5ea7b3..6e3a2035 100644
--- a/backend/tests/test_solver_worker.py
+++ b/backend/tests/test_solver_worker.py
@@ -38,6 +38,7 @@ def _reset_state():
state.n2_unconfirmed = 0
state.solver_stale_drops = 0
state.solver_resolve_skips = 0
+ state.solver_resolve_skips_dark = 0
state.multinode_tracks.clear()
state.task_last_success.clear()
@@ -334,56 +335,153 @@ class TestResolveSuppression:
emits its own candidate for it inside one association window. Solving all
of them starves aircraft that have no solve at all — the queue ages out
behind work whose result is superseded the moment it lands.
+
+ The claim that suppresses a duplicate is taken on PUBLICATION
+ (_record_resolve_slot), not on admission: the rule is "this aircraft is
+ already on the map at this width", and only a publish puts it there.
+ _resolve_slot_covered is the pure test run before the solve.
"""
def _s_in(self, track_ids, n_nodes=2):
return dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids))
- def test_a_second_copy_of_the_same_tracks_is_skipped(self):
+ def _covered(self, track_ids, n_nodes=2, now=None):
+ return solver_mod._resolve_slot_covered(self._s_in(track_ids, n_nodes), now or time.time())[0]
+
+ def _publish(self, track_ids, n_nodes=2, now=None):
+ solver_mod._record_resolve_slot(list(track_ids), n_nodes, now or time.time())
+
+ def test_a_second_copy_of_a_published_candidate_is_skipped(self):
now = time.time()
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is False
+ assert self._covered(["a1", "b1"], now=now) is False
+ self._publish(["a1", "b1"], now=now)
+ assert self._covered(["a1", "b1"], now=now) is True
- def test_a_candidate_carrying_an_unsolved_track_runs(self):
+ def test_the_check_alone_claims_nothing(self):
+ """The whole point of the split: a candidate that is admitted and then
+ rejected by the gate stack must leave no trace."""
+ now = time.time()
+ assert self._covered(["a1", "b1"], now=now) is False
+ assert self._covered(["a1", "b1"], now=now) is False
+
+ def test_a_candidate_carrying_an_unpublished_track_runs(self):
"""An aircraft entering coverage must never be suppressed."""
now = time.time()
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b2"]), now) is True
+ self._publish(["a1", "b1"], now=now)
+ assert self._covered(["a1", "b2"], now=now) is False
def test_a_wider_view_of_the_same_tracks_runs(self):
now = time.time()
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=2), now) is True
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True
+ self._publish(["a1", "b1"], n_nodes=2, now=now)
+ assert self._covered(["a1", "b1"], n_nodes=5, now=now) is False
def test_a_narrower_view_after_a_wider_one_is_skipped(self):
now = time.time()
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=2), now) is False
+ self._publish(["a1", "b1"], n_nodes=5, now=now)
+ assert self._covered(["a1", "b1"], n_nodes=2, now=now) is True
- def test_a_narrow_admission_does_not_lower_the_bar(self):
+ def test_a_narrow_publish_does_not_lower_the_bar(self):
"""The window holds the widest claim, not the most recent one."""
now = time.time()
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b2"], n_nodes=2), now) is True
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=3), now) is False
+ self._publish(["a1", "b1"], n_nodes=5, now=now)
+ self._publish(["a1", "b2"], n_nodes=2, now=now)
+ assert self._covered(["a1", "b1"], n_nodes=3, now=now) is True
def test_claims_expire(self):
now = time.time()
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True
+ self._publish(["a1", "b1"], now=now)
later = now + solver_mod._SOLVER_RESOLVE_INTERVAL_S + 1.0
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), later) is True
+ assert self._covered(["a1", "b1"], now=later) is False
def test_an_input_without_track_provenance_always_runs(self):
"""Detection-level inputs carry no track ids — nothing to match on."""
now = time.time()
- assert solver_mod._claim_resolve_slot({"n_nodes": 2}, now) is True
- assert solver_mod._claim_resolve_slot({"n_nodes": 2}, now) is True
+ assert solver_mod._resolve_slot_covered({"n_nodes": 2}, now)[0] is False
+ solver_mod._record_resolve_slot(None, 2, now)
+ assert solver_mod._resolve_slot_covered({"n_nodes": 2}, now)[0] is False
def test_zero_interval_disables_suppression(self, monkeypatch):
monkeypatch.setattr(solver_mod, "_SOLVER_RESOLVE_INTERVAL_S", 0.0)
now = time.time()
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True
- assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True
+ self._publish(["a1", "b1"], now=now)
+ assert self._covered(["a1", "b1"], now=now) is False
+
+ def test_the_check_names_every_blocking_claim(self):
+ now = time.time()
+ self._publish(["a1", "b1"], n_nodes=4, now=now)
+ covered, blocking = solver_mod._resolve_slot_covered(self._s_in(["a1", "b1"], n_nodes=3), now)
+ assert covered is True
+ assert {b["track_id"]: b["held_n"] for b in blocking} == {"a1": 4, "b1": 4}
+
+ def test_an_admitted_candidate_reports_no_blockers(self):
+ covered, blocking = solver_mod._resolve_slot_covered(self._s_in(["a1", "b1"]), time.time())
+ assert (covered, blocking) == (False, [])
+
+ def _solve_fn(self, calls, rms_delay=0.5, lat=37.5, lon=-122.1):
+ def fn(s_in, cfgs):
+ calls.append(s_in)
+ return {
+ "success": True,
+ "lat": lat,
+ "lon": lon,
+ "alt_m": 9000.0,
+ "rms_delay": rms_delay,
+ "rms_doppler": 5.0,
+ "timestamp_ms": int(time.time() * 1000),
+ "contributing_node_ids": ["n1", "n2"],
+ "n_nodes": s_in.get("n_nodes", 2),
+ }
+
+ return fn
+
+ def test_a_rejected_candidate_does_not_block_an_identical_twin(self, monkeypatch):
+ """The bug this split exists to fix. A candidate the gate stack sank
+ put nothing on the map, so the next copy of the same aircraft is its
+ first real chance — and used to be blacked out for the full 12 s."""
+ _reset_state()
+ monkeypatch.setattr(state, "node_analytics", _StubAnalytics())
+ calls: list = []
+ s_in = self._s_in(["a1", "b1"])
+
+ # rms_delay past the gate: solves, then rejected, publishes nothing.
+ solver_mod._process_solver_item((dict(s_in), {}, time.time()), self._solve_fn(calls, rms_delay=10.0))
+ assert state.solver_fail_rms_delay == 1
+ assert not state.multinode_tracks
+
+ solver_mod._process_solver_item((dict(s_in), {}, time.time()), self._solve_fn(calls))
+ assert len(calls) == 2, "the twin must not be suppressed by a reject"
+ assert state.multinode_tracks
+ assert state.solver_resolve_skips == 0
+
+ def test_a_subset_for_another_aircraft_survives_a_rejected_superset(self, monkeypatch):
+ """Tracker track ids are shared across the candidates of DIFFERENT
+ aircraft, so a contaminated superset that the gates sank used to take
+ every clean subset behind it down with it — including its neighbour's
+ only candidate."""
+ _reset_state()
+ monkeypatch.setattr(state, "node_analytics", _StubAnalytics())
+ calls: list = []
+ superset = self._s_in(["a1", "b1", "c1"], n_nodes=3)
+ solver_mod._process_solver_item((superset, {}, time.time()), self._solve_fn(calls, rms_delay=10.0))
+ assert not state.multinode_tracks
+
+ # The neighbour: fewer nodes, sharing one contaminated track id.
+ subset = self._s_in(["a1", "b1"], n_nodes=2)
+ solver_mod._process_solver_item((subset, {}, time.time()), self._solve_fn(calls))
+ assert len(calls) == 2
+ assert state.multinode_tracks
+
+ def test_only_the_published_width_is_claimed(self, monkeypatch):
+ """A publish claims at the width it published, so a later narrower
+ copy is suppressed and a wider one still runs."""
+ _reset_state()
+ monkeypatch.setattr(state, "node_analytics", _StubAnalytics())
+ calls: list = []
+ solver_mod._process_solver_item((self._s_in(["a1", "b1"], n_nodes=3), {}, time.time()), self._solve_fn(calls))
+ assert state.multinode_tracks
+ now = time.time()
+ assert self._covered(["a1", "b1"], n_nodes=2, now=now) is True
+ assert self._covered(["a1", "b1"], n_nodes=4, now=now) is False
def test_a_skipped_item_never_reaches_the_solver(self, monkeypatch):
_reset_state()
@@ -407,6 +505,8 @@ def solve_fn(s_in, cfgs):
solver_mod._process_solver_item((s_in, {}, time.time()), solve_fn)
assert len(solve_calls) == 1
assert state.solver_successes == 1
+ # The first item PUBLISHED, which is what makes the second redundant.
+ assert state.multinode_tracks
assert solver_mod._process_solver_item((dict(s_in), {}, time.time()), solve_fn) is None
assert len(solve_calls) == 1, "the duplicate must not be solved"
@@ -415,6 +515,50 @@ def solve_fn(s_in, cfgs):
assert state.solver_failures == 0
assert state.solver_stale_drops == 0
+ def test_a_skip_is_recorded_with_the_claim_that_blocked_it(self):
+ """The counter alone cannot say WHOSE claim suppressed a candidate,
+ and tracker track ids are shared between different aircraft — so a
+ skip that suppressed a duplicate and one that suppressed a neighbour
+ looked identical. The deque carries the blocking claims."""
+ _reset_state()
+ state.solver_resolve_skips_recent.clear()
+ now = time.time()
+ s_in = dict(self._s_in(["a1", "b1"], n_nodes=4), initial_guess={"lat": 35.0, "lon": -82.0})
+ solver_mod._record_resolve_slot(["a1", "b1"], 4, now)
+ covered, blocking = solver_mod._resolve_slot_covered(dict(s_in), now)
+ assert covered is True
+ solver_mod._record_resolve_skip(dict(s_in), now, blocking)
+
+ assert state.solver_resolve_skips == 1
+ assert state.solver_resolve_skips_dark == 1
+ assert len(state.solver_resolve_skips_recent) == 1
+ rec = state.solver_resolve_skips_recent[0]
+ assert rec["lane"] == "dark"
+ assert rec["track_ids"] == ["a1", "b1"]
+ assert rec["n_nodes"] == 4
+ assert rec["guess_lat"] == 35.0
+ assert {b["track_id"]: b["held_n"] for b in rec["blocking"]} == {"a1": 4, "b1": 4}
+
+ def test_a_tagged_candidate_is_counted_but_not_as_dark(self):
+ _reset_state()
+ state.solver_resolve_skips_recent.clear()
+ now = time.time()
+ s_in = dict(self._s_in(["a1"], n_nodes=3), adsb_hex="abc123")
+ solver_mod._record_resolve_skip(s_in, now, [])
+ assert state.solver_resolve_skips == 1
+ assert state.solver_resolve_skips_dark == 0
+ assert state.solver_resolve_skips_recent[0]["lane"] == "adsb"
+
+ def test_skips_never_enter_the_solve_history(self):
+ """One skip per solve-history record would evict the solves the same
+ investigation needs — live, skips outrun dark records two to one."""
+ _reset_state()
+ state.mlat_solve_history.clear()
+ s_in = self._s_in(["a1", "b1"])
+ solver_mod._record_resolve_skip(s_in, time.time(), [])
+ assert not state.mlat_solve_history
+ assert not state.mlat_solve_history_known
+
class TestSolveBestAltitude:
"""Altitude-sweep helpers: n_nodes >= 3 uses a layer sweep, n_nodes = 2 uses initial_guess directly."""
diff --git a/backend/tests/test_track_filter.py b/backend/tests/test_track_filter.py
index 44beee81..2e15e94d 100644
--- a/backend/tests/test_track_filter.py
+++ b/backend/tests/test_track_filter.py
@@ -829,3 +829,156 @@ def _feed():
track_filter.reset()
assert track_filter.learned_velocity(key) is None
+
+
+class TestIndefiniteCovariance:
+ """An indefinite cov_en_km2 must never reach the filter.
+
+ This is the 2026-09-05 droplet failure (91 ValueError: math domain error
+ in 40 minutes out of learned_velocity), and the mechanism is not roundoff
+ — the Joseph form in _kf_correct was already deployed when it happened.
+ cov_en_km2 is the top-left 2x2 of s2 * inv(JtJ) for the solver's 5-state
+ fit, and the solver falls back to pinv only on an outright LinAlgError,
+ so an ill-conditioned-but-not-singular JtJ (near-parallel baselines)
+ inverts to garbage that is INDEFINITE while both diagonals stay positive
+ — passing the solver's own check and, before the fix, _measurement_R's.
+
+ Joseph preserves PSD for any gain but only GIVEN PSD P and R: its
+ K R K^T term inherits R's negative eigenvalue, and _init_entry seeds P's
+ position block straight from R. Hence the determinant check.
+
+ The covariances below are exactly that shape: equal diagonals with an
+ off-diagonal larger than their geometric mean, i.e. a "correlation"
+ above 1, which no real covariance has.
+ """
+
+ def setup_method(self):
+ track_filter.reset()
+ state.adsb_aircraft.clear()
+
+ def teardown_method(self):
+ track_filter.reset()
+ state.adsb_aircraft.clear()
+
+ @staticmethod
+ def _indefinite_cov(var_km2=1.0, corr=1.2):
+ return [[var_km2, corr * var_km2], [corr * var_km2, var_km2]]
+
+ def test_indefinite_cov_falls_back_to_the_base_floor(self):
+ r = track_filter._measurement_R(make_result(35.0, -82.0, 1_000, cov_en_km2=self._indefinite_cov()))
+
+ # Rejected outright, so R is the cov=0 limit — exactly `base`, the
+ # same answer a missing cov gets.
+ assert r[0, 1] == 0.0
+ assert math.sqrt(0.5 * (r[0, 0] + r[1, 1])) == pytest.approx(track_filter._KF_DEFAULT_POS_SIGMA_M)
+ # The property that actually matters downstream.
+ assert np.all(np.linalg.eigvalsh(r) >= 0.0)
+
+ def test_a_correlated_but_psd_cov_is_still_accepted(self):
+ """The determinant check must reject only the impossible matrices.
+
+ A genuine off-diagonal is real information about solve geometry (a
+ two-node baseline has a long axis), and rejecting it would quietly
+ throw away the relative weighting _KF_R_INFLATE exists to provide.
+ corr=0.9 is strongly correlated but perfectly valid: det > 0.
+ """
+ cov = self._indefinite_cov(corr=0.9)
+ r = track_filter._measurement_R(make_result(35.0, -82.0, 1_000, cov_en_km2=cov))
+
+ assert r[0, 1] != 0.0 # the correlation survived
+ assert np.all(np.linalg.eigvalsh(r) >= 0.0)
+
+ @pytest.mark.parametrize("cadence_s", [1.4, 20.0])
+ def test_indefinite_cov_keeps_the_filter_psd_and_learned_velocity_alive(self, monkeypatch, cadence_s):
+ """The end-to-end regression: 200 solves carrying an indefinite cov.
+
+ On origin/main this raised the droplet's ValueError within 16 solves
+ at the 20 s cadence real solves arrive on, and within 7 at 1.4 s.
+ """
+ monkeypatch.setenv("TRACK_SMOOTHER", "kf")
+ cov = self._indefinite_cov()
+ key = f"indef-{cadence_s}"
+ lat0, lon0 = 35.0, -82.0
+
+ for i in range(200):
+ lat_i, lon_i = offset_latlon_m(lat0, lon0, east_m=250.0 * cadence_s * i, north_m=0.0)
+ result = make_result(lat_i, lon_i, 1_000 + int(i * cadence_s * 1000), cov_en_km2=cov)
+ track_filter.smooth_solve(result, key, None)
+
+ entry = track_filter._KF_TRACKS.get(key)
+ assert entry is not None
+ assert np.all(np.diag(entry.P) >= 0.0), f"solve {i}: negative variance in {np.diag(entry.P)}"
+ assert np.allclose(entry.P, entry.P.T, rtol=0, atol=0), f"solve {i}: asymmetric covariance"
+ # The call that was throwing.
+ assert track_filter.learned_velocity(key) is not None
+
+ def test_tiny_position_sigma_cannot_produce_a_tight_r(self):
+ """A 1 m formal sigma is floored, not believed.
+
+ R is what makes the Joseph update ill-conditioned when it is tiny
+ relative to P, so the unmodeled-error floor is the other half of this
+ fix holding: frame-time skew, association contamination and altitude
+ pinning do not shrink because one solve's Jacobian was tight, and no
+ fix on this network is good to a metre. The floor is additive and
+ unconditional, so a metre-scale cov contributes only
+ (_KF_R_INFLATE * 1)^2 = 16 m^2 against a base of 1200^2, leaving the
+ composed sigma at the base to within a hundredth of a percent.
+ """
+ for sigma_m in (1e-6, 1.0):
+ cov_km2 = (sigma_m / 1000.0) ** 2
+ r = track_filter._measurement_R(
+ make_result(35.0, -82.0, 1_000, cov_en_km2=[[cov_km2, 0.0], [0.0, cov_km2]])
+ )
+ actual_sigma = math.sqrt(0.5 * (r[0, 0] + r[1, 1]))
+ assert actual_sigma == pytest.approx(track_filter._KF_DEFAULT_POS_SIGMA_M, rel=1e-4)
+ assert actual_sigma >= track_filter._KF_MIN_POS_SIGMA_M
+ # Nowhere near the few-metre R that would make the update
+ # ill-conditioned, and an order of magnitude above even a
+ # bare-sensor floor: the unmodeled-error terms dominate.
+ assert actual_sigma > 1000.0
+
+ def test_repeated_updates_at_a_tiny_r_stay_psd(self, monkeypatch):
+ """200 solves with an effectively-zero cov and alternating positions.
+
+ The floor above means the filter never actually sees a metre-scale R,
+ so this asserts the composite invariant end to end rather than the
+ ill-conditioning in isolation (TestJosephCovariance drives
+ _kf_correct at R = 1e-4 directly).
+ """
+ monkeypatch.setenv("TRACK_SMOOTHER", "kf")
+ key = "tiny-r"
+ tiny = (1e-6 / 1000.0) ** 2
+ cov = [[tiny, 0.0], [0.0, tiny]]
+
+ for i in range(200):
+ # Alternating, so the innovation never settles to zero.
+ lat_i, lon_i = offset_latlon_m(35.0, -82.0, east_m=100.0 * (i % 2), north_m=0.0)
+ track_filter.smooth_solve(make_result(lat_i, lon_i, 1_000 + i * 1_400, cov_en_km2=cov), key, None)
+
+ entry = track_filter._KF_TRACKS.get(key)
+ assert np.all(np.diag(entry.P) >= 0.0), f"solve {i}: negative variance in {np.diag(entry.P)}"
+ assert np.allclose(entry.P, entry.P.T, rtol=0, atol=0), f"solve {i}: asymmetric covariance"
+ assert track_filter.learned_velocity(key) is not None
+
+ def test_negative_velocity_variance_degrades_to_zero_sigma(self, monkeypatch):
+ """The last line of defence, on a hand-poisoned entry.
+
+ learned_velocity has two callers that each lose real work when it
+ raises — solver.py's multinode_key_decision drops the solve,
+ aircraft_feed's multinode_to_aircraft drops the whole broadcast — so
+ even a state no code path should now be able to reach must return a
+ number rather than throw.
+ """
+ monkeypatch.setenv("TRACK_SMOOTHER", "kf")
+ key = "neg-vel-var"
+ track_filter.smooth_solve(make_result(35.0, -82.0, 1_000), key, None)
+ track_filter.smooth_solve(make_result(35.001, -82.0, 21_000), key, None)
+
+ with track_filter._KF_LOCK:
+ entry = track_filter._KF_TRACKS[key]
+ entry.P[1, 1] = -1e6
+ entry.P[3, 3] = -1e6
+
+ lv = track_filter.learned_velocity(key)
+ assert lv is not None
+ assert lv[2] == 0.0 # clamped, not raised
diff --git a/docs/arc-display.md b/docs/arc-display.md
index 3eb17ead..8afb87cd 100644
--- a/docs/arc-display.md
+++ b/docs/arc-display.md
@@ -90,9 +90,9 @@ Geometry is frozen at creation; only style refreshes. Opacity fades linearly
over `ARC_FADE_MS` (5 s), after which the buffer entry is pruned.
Arc-only tracks dead-reckon their reference position for at most
-`ARC_DR_MAX_S` (10 s, vs 60 s for solved tracks) — the backend pins their
-position to the arc midpoint, so a long glide walks the reference off the
-measured locus.
+`ARC_DR_MAX_S` (10 s, vs `MN_DR_CAP_S` 15 s for solved tracks) — the backend
+pins their position to the arc midpoint, so a long glide walks the reference
+off the measured locus.
Selecting an arc track (from the list panel or by clicking the arc) highlights
its arcs in amber, draws the detecting node's beam wedge, and centers the map
diff --git a/docs/design-notes/2026-09-05-solve-uncertainty-disc.md b/docs/design-notes/2026-09-05-solve-uncertainty-disc.md
index 4bc7a6a3..2121072b 100644
--- a/docs/design-notes/2026-09-05-solve-uncertainty-disc.md
+++ b/docs/design-notes/2026-09-05-solve-uncertainty-disc.md
@@ -90,14 +90,20 @@ gain only to let a genuinely ill-conditioned solve show a bigger disc.
### Growth while dead-reckoning
-The feed dead-reckons a solve forward up to 30 s with the KF's learned
-velocity, then holds; the frontend continues its own DR up to 60 s. The disc
-grows accordingly:
+The feed dead-reckons a solve forward up to `MN_DR_CAP_S` with the KF's
+learned velocity, then holds; the frontend continues its own DR up to
+`UNCERTAINTY_DR_CAP_S`. The disc grows accordingly:
```
-σ(t) = sqrt( σ_solve² + (σ_v · min(t, 60))² )
+σ(t) = sqrt( σ_solve² + (σ_v · min(t, UNCERTAINTY_DR_CAP_S))² )
```
+Both caps were halved on 2026-09-05 (30 → 15 s and 60 → 30 s) once dark solves
+started landing every 1–3 s: a dark entry no longer survives to 60 s at all
+(`MN_DARK_EXPIRY_S` = 30 s), so the second half of the old growth curve
+described entries that cannot exist. See `docs/solverflow.md` for the
+error-vs-solve-age measurement the two budgets are cut from.
+
`t` is the age of the solve at draw time, `σ_v` the velocity sigma:
`track_filter.learned_velocity(key)[2]` when the KF has state for the key
(clamped to [5, 150] m/s), else `SOLVE_SIGMA_VEL_DEFAULT_MS = 25`. A fresh
diff --git a/docs/runbook.md b/docs/runbook.md
index c2d582b2..31406cdd 100644
--- a/docs/runbook.md
+++ b/docs/runbook.md
@@ -396,7 +396,8 @@ recognised as such and dropped without solving.
`SOLVER_RESOLVE_INTERVAL_S` (default 12 s) is the window an aircraft is not
re-solved in. Raising it trades map refresh rate for solver headroom — do not go
-past the 60 s `multinode_tracks` expiry, or tracks will lapse between solves. `0`
+past the `multinode_tracks` expiry (`MN_DARK_EXPIRY_S`, 30 s, for `mn-dark-*`
+entries; 60 s for `mn-adsb-*` ones), or tracks will lapse between solves. `0`
turns the suppression off.
---
diff --git a/docs/solverflow.md b/docs/solverflow.md
index c626e5bf..2542369d 100644
--- a/docs/solverflow.md
+++ b/docs/solverflow.md
@@ -11,10 +11,12 @@ beyond what publication needs, see [`pipeline.md`](pipeline.md) (its own §3 is
stale on the known lane and pool fallback — this doc is the current source for
those two topics).
-File:line references are repo-relative to `backend/`, except the `libs/*`
-paths, which are already fully qualified (those are separate submodule repos
-vendored under `libs/`). All references were checked against `main` at
-`0a1d30f`.
+References name a **file and a symbol**, never a line number: paths are
+repo-relative to `backend/`, except the `libs/*` ones, which are already fully
+qualified (those are separate submodule repos vendored under `libs/`). Line
+numbers were what this document used to carry, and they were stale within two
+weeks of being written — every one of them had drifted by the time anyone
+followed it. A symbol survives an edit above it, so grep for the name.
## Legend
@@ -32,7 +34,9 @@ flowchart LR
Solid arrows are the live path. Dashed arrows and the grey style mark branches
that exist in code but are switched off in production today (inline CV fit,
-the bottom-up doppler gate, every mode flag except `KNOWN_LANE_MODE`). Diamonds
+the bottom-up doppler gate, every mode flag except `KNOWN_LANE_MODE` and
+`DARK_FOLLOW_MODE`, the latter in `shadow` — it claims, solves and records, but
+publishes nothing and takes nothing away from the dark lane). Diamonds
are gates; a failed gate either drops the item or routes it to a fallback —
labeled on the arrow.
@@ -74,13 +78,15 @@ lane rides the solver loop's idle cycles rather than owning workers of its
own. Everything that reaches a solve passes through one gate stack
(`_process_solver_item`) before publication.
-| Constant | Value | File:line |
+| Constant | Value | Defined in |
|---|---|---|
-| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py:358-359` |
-| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py:365-366` |
-| `FRAME_WORKERS` | 4 (compose sets 6) | `main.py:164`, `docker-compose.yml:54` |
-| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py:31,67` |
-| `KNOWN_LANE_MODE` default | `binding` | `core/state.py:72-74` |
+| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py` |
+| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py` |
+| `FRAME_WORKERS` | 4 (compose sets 6) | `core/state.py` (`FRAME_WORKERS`), `docker-compose.yml` |
+| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py` (`_N_SOLVER_WORKERS`, `_make_solver_pool`) |
+| `KNOWN_LANE_MODE` default | `binding` | `core/state.py` (`KNOWN_LANE_MODE`) |
+| `SOLVER_ALT_MODE` default | `sweep` | `core/state.py` (`SOLVER_ALT_MODE`) |
+| `SOLVER_FREE_ALT_STARTS` default | 1 | `core/state.py` (`SOLVER_FREE_ALT_STARTS`) |
---
@@ -89,11 +95,11 @@ own. Everything that reaches a solve passes through one gate stack
```mermaid
flowchart TD
subgraph producers["Five producers"]
- p1["TCP (primary)
tcp_handler.py:326"]
- p2["blah2 bridge
blah2_bridge.py:289"]
- p3["v1 node HTTP API
node_stream.py:250"]
- p4["Legacy HTTP radar routes
routes/radar.py:151,202"]
- p5["Startup priming
node_pipeline.py:139"]
+ p1["TCP (primary)
tcp_handler._enqueue_detection"]
+ p2["blah2 bridge
blah2_bridge.blah2_bridge_task"]
+ p3["v1 node HTTP API
node_stream._file_frame"]
+ p4["Legacy HTTP radar routes
radar.ingest_detections(_bulk)"]
+ p5["Startup priming
node_pipeline.prime_pipeline"]
end
p1 --> gA{"Gate A: timestamp present?"}
@@ -131,25 +137,25 @@ flowchart TD
classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3
```
-The ordering inside `process_one_frame` (`services/frame_processor.py:294`) is
+The ordering inside `process_one_frame` (`services/frame_processor.py`) is
load-bearing, not incidental: claiming (2.3) runs **before** ADS-B seeding
(2.4) so a node-supplied `adsb` field is still distinguishable from a claim,
and both run **before** the tracker (2.5) so that, in `binding` mode, a
claimed detection never reaches the dark-lane tracker or association at all
-— see the ordering comment at `services/frame_processor.py:327-337`.
+— see the ordering comment at the head of `process_one_frame`'s claiming step.
Frame-level gates (A/B/C on TCP, plus the connected-node check on the v1 API)
sit ahead of everything else; nothing downstream sees a frame that failed
one of them.
-| Constant | Value | File:line |
+| Constant | Value | Defined in |
|---|---|---|
-| Gate A: timestamp required | — | `tcp_handler.py:513-516` |
-| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node | `tcp_handler.py:495,527-532` |
-| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler.py:536-552` |
-| `process_one_frame` entry | — | `services/frame_processor.py:294` |
-| Ordering rationale (claim → seed → tracker) | — | `services/frame_processor.py:327-337` |
-| Gate 2.10: `n_nodes < 2` skip | — | `services/frame_processor.py:409-426` |
-| blah2 poll interval | 1.0 s | `config/constants.py:266` |
+| Gate A: timestamp required | — | `tcp_handler._enqueue_detection` |
+| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node, counted as `node_frames_rate_limited` | `tcp_handler` (`_NODE_MIN_INTERVAL_S`, `_enqueue_detection`) |
+| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler._enqueue_detection` |
+| `process_one_frame` entry | — | `services/frame_processor.py` |
+| Ordering rationale (claim → seed → tracker) | — | `frame_processor.process_one_frame` |
+| Gate 2.10: `n_nodes < 2` skip | — | `frame_processor.process_one_frame` |
+| blah2 poll interval | 1.0 s | `config/constants.py` (`BLAH2_POLL_INTERVAL_S`) |
---
@@ -205,17 +211,17 @@ also reject — a differential property test in `test_known_claiming.py`
failure increments the same `known_claims_visibility_rejects` counter as a
gate failure: same event, same meaning, just caught cheaper.
-**Mode semantics** (`KNOWN_LANE_MODE`, read once at `core/state.py:72-74`,
+**Mode semantics** (`KNOWN_LANE_MODE`, read once in `core/state.py`,
default `binding`; an unrecognized value falls back to `shadow`, not to the
default — a typo should degrade to the inert mode, not the acting one):
| Mode | Claiming | Frame the dark lane sees | Known-lane solver | Publication |
|---|---|---|---|---|
-| `off` | never runs | untouched | returns 0 immediately (`known_lane.py:391-392`); worker never even calls it (`solver.py:1961`) | none |
+| `off` | never runs | untouched | returns 0 immediately (`known_lane.run_known_lane_pass`); worker never even calls it (`solver._run_solver_worker`) | none |
| `shadow` | runs, records claims + residuals + counters | untouched | runs: solves, classifies, records accuracy samples | never |
-| `binding` | runs | `strip_claimed_detections` removes claimed indices (`frame_processor.py:347`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish |
+| `binding` | runs | `strip_claimed_detections` removes claimed indices (called from `frame_processor.process_one_frame`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish |
-`strip_claimed_detections` (`services/known_claiming.py:343`) returns a copy
+`strip_claimed_detections` (`services/known_claiming.py`) returns a copy
with claimed indices removed from `delay`/`doppler`/`snr`/`adsb`; the
original frame still feeds the archive and ADS-B extraction (steps 2.11-2.12)
unchanged.
@@ -224,7 +230,7 @@ unchanged.
```mermaid
flowchart TD
- arm["Solver worker loop arms known_lane
at thread start (solver.py:1961)"]
+ arm["Solver worker loop arms known_lane
at thread start (solver._run_solver_worker)"]
arm --> drain["After every queue-drain iteration,
call maybe_run_pass"]
drain --> gm{"mode == off?"}
gm -->|"yes"| ret1["return"]:::inert
@@ -261,7 +267,7 @@ flowchart TD
gpub -->|"no"| noop["accuracy sample only,
no feed entry"]:::inert
```
-The docstring at `services/tasks/known_lane.py:19-27` calls this the "free
+The module docstring of `services/tasks/known_lane.py` calls this the "free
solve invariant": the ADS-B fix seeds the initial guess and pins altitude,
nothing else — no regularization pulls the solve toward the truth position,
so the residual (`err_km`) is a genuine measurement of radar accuracy, not a
@@ -269,21 +275,192 @@ circular check. One more intentional-by-omission detail: known-lane
measurements carry `snr = 0.0` (claim records have no `snr` key), which the
LM's SNR weighting maps to a uniform weight of 1.0.
-| Constant | Value | File:line |
+| Constant | Value | Defined in |
|---|---|---|
-| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `ADSB_SEED_MAX_DR_AGE_S`, `association.py:106`) |
-| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `ADSB_SEED_*`, `association.py:98,99`) |
-| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py:79` |
-| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py:205` |
-| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py:89`, applied in `known_claiming.py:_dark_global_projections` |
-| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py:73,77` |
-| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py:80` |
-| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py:85,86` |
-| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py:274-275` |
-| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py:105` |
-| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py:91,99` |
-| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py:110` |
-| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py:205` |
+| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `association.ADSB_SEED_MAX_DR_AGE_S`) |
+| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `association.ADSB_SEED_DELAY_GATE_US` / `_DOPPLER_GATE_HZ`) |
+| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py` |
+| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py` |
+| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py`, applied in `known_claiming._dark_global_projections` |
+| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py` |
+| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py` |
+| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py` |
+| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py` |
+| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py` |
+| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py` |
+| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py` |
+| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py` |
+
+### 3d. Dark track following (`DARK_FOLLOW_MODE`)
+
+The same inversion, applied to aircraft that have **no transponder**. Where the
+known lane claims detections against a dead-reckoned ADS-B fix, this lane
+claims them against an established `mn-dark-*` track's Kalman state — the only
+identity a dark aircraft has. It exists for two measured problems in the
+bottom-up dark lane (§4):
+
+- **Continuity.** Consecutive solves of one aircraft less than 5 s apart land
+ on a different key 15% of the time, any gap in solving re-mints the key from
+ scratch, and neighbours 3 km apart can share one. The key is chosen *after*
+ the solve, by proximity (`multinode_key_decision`), so it is a guess.
+- **The initial guess.** Nothing tells the solver where the aircraft is
+ expected to be; a dark input starts from a quantised 3 km grid centroid.
+
+Following fixes both at once, because the pseudo-state that predicts the
+observation is also the key: the follow input carries `anchor_key` = the
+followed key, and `multinode_key_decision`'s anchor branch keys the solve back
+onto the same track. Continuity is then *by construction* rather than by the
+proximity scan happening to pick the right neighbour.
+
+Node-track ids were the obvious cheaper mechanism and are not safe: attaching
+each solve to the newest key sharing a `source_track_ids` entry linked the
+**wrong aircraft 12%** of the time in a dense metro cluster — the same reason
+`_supersession_match` (§6) stopped trusting a bare shared id.
+
+```mermaid
+flowchart TD
+ build["dark_follow.follow_targets()
rebuilt at most 1/s, TTL-cached"]
+ build --> gkey{"key starts mn-dark-?"}
+ gkey -->|"no"| skip0["ADS-B tracks are never followed"]:::inert
+ gkey --> gcool{"in cooldown?"}
+ gcool -->|"yes"| skip1["dropped key, waiting out
DARK_FOLLOW_COOLDOWN_S"]:::inert
+ gcool --> gelig{"age <= MAX_AGE_S 20s
AND solve_count >= 3
AND n_nodes >= 3"}
+ gelig -->|"no"| skip2["not established enough"]:::inert
+ gelig --> gkf{"track_filter.learned_velocity
has state?"}
+ gkf -->|"no"| skip3["nothing to dead-reckon with"]:::inert
+ gkf --> gsig{"vel sigma <= 60 m/s?"}
+ gsig -->|"no"| drop["drop_target + cooldown
dark_follow_dropped"]:::inert
+ gsig -->|"yes"| target["pseudo-state:
lat/lon/alt, KF velocity,
pos+vel sigma, world"]
+
+ target --> claim["known_claiming path 3,
per frame per node"]
+ claim --> gfree{"detections left by
ADS-B paths 1+2?"}
+ gfree -->|"none"| skip4["ADS-B always wins"]:::inert
+ gfree --> gworld{"same world as the node?"}
+ gworld --> gvis{"_point_in_beam on the
dead-reckoned position"}
+ gvis --> gate{"Hungarian one-to-one under
widened gates (below)"}
+ gate -->|"infeasible"| skip5["stays in the dark pool"]:::inert
+ gate -->|"claimed"| rec["state.known_claims[mn-dark-key]
dark_follow: True, follow_fix
dark_follow_claims"]
+
+ rec --> mode{"DARK_FOLLOW_MODE"}
+ mode -->|"off"| m0["no targets built at all"]:::inert
+ mode -->|"shadow"| m1["frame untouched;
pass solves + records
outcome dark_follow_shadow"]
+ mode -->|"binding"| m2["strip_claimed_detections;
input onto solver_queue"]
+
+ m2 --> gates["normal dark gate stack (§5)
anchor_dr keying, KF smoothing,
published under the SAME key"]
+ m1 --> guard{"solve rejected?"}
+ gates --> guard
+ guard -->|"2 in a row"| drop
+ guard -->|"published"| streakclear["streak cleared"]
+
+ classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3
+```
+
+**The widened gates** (`dark_follow.follow_gates`). A dark pseudo-state carries
+its own uncertainty, where an ADS-B fix is treated as truth, so the claim gate
+is the known lane's gate **plus** that uncertainty projected into observation
+space:
+
+```
+d_gate_us = 10.0 * _gate_scale(dt) + 2 * (pos_sigma_m + vel_sigma_ms * dt) / c_m_per_us
+f_gate_hz = 25.0 * _gate_scale(dt) + 2 * vel_sigma_ms * fc_hz / c_m_per_s
+```
+
+A position error of `s` metres moves the bistatic range by at most `2s` (the
+target can be displaced toward both transmitter and receiver); a velocity error
+of `u` m/s moves the bistatic Doppler by at most `2u/λ`. Both are worst-case
+projections — the true geometry factor is a cosine ≤ 1 — which is the safe
+direction for a gate. Both are capped (40 µs / 100 Hz, 4× the base gates): past
+there the prediction is not constraining anything and the aircraft should be
+re-found bottom-up.
+
+**The ghost lock-in guard** is load-bearing, not tidiness. Following is a
+positive feedback loop — the solve keeps the key alive, the key keeps claiming
+detections, and in binding mode those detections never reach the lane that
+would disagree. A followed key is therefore dropped for
+`DARK_FOLLOW_COOLDOWN_S` on **two consecutive rejected follow-solves** or a
+velocity sigma past the ceiling, and the bottom-up lane has to re-find it.
+Every follow-solve outcome reaches the guard through one hook in
+`solver._record_solve_history` (published, every `rejected_*`, unconverged, and
+the shadow pass's own record).
+
+**Anchor dead-reckoning.** The anchor branch's flat 6 km check compares the
+solve against where the entry was last *stored*. For a follow input that is
+wrong by construction — its guess IS a prediction of where the anchor drifted
+to — and the numbers bite: the dark displacement cap is 6.0 km and the flat
+anchor gate is 6.0 km, so a solve at the edge of the gate that let it through
+is at the edge of the gate that must key it, before any drift is added; at the
+lane's 20 s staleness limit a 270 m/s target adds 5.4 km more. Follow inputs
+therefore pass `anchor_dr=True`, which dead-reckons the anchor and applies the
+proximity scan's own age-scaled gate. Every other anchored input is unchanged.
+
+**Key ownership.** Following an aircraft is not enough on its own — the
+bottom-up lane keeps solving the same sky, and its solves are still keyed by
+proximity onto whatever entry is nearest. Measured on test with the lane
+binding (20 min, 625 six-plus-node dark samples, `DARK_FOLLOW_MODE=binding`):
+
+| | |
+|---|---|
+| follow solves published | 224, **all** anchor-keyed |
+| follow-solve position error | median 0.24 km, p90 1.35 km |
+| dark aircraft with 6+ nodes in cone shown on the map | **89%** (76% before the lane) |
+| bottom-up solves keyed by proximity onto an existing key | 425 |
+| ...that landed on a key owned by a **different** aircraft | 90 (**21%**) |
+| ...onto a key the follow lane had published on within 6 s | 12 |
+
+A cross-keyed solve moves the entry 5+ km, corrupts the KF velocity it feeds,
+and can supersede the right key. A tighter spatial gate cannot separate the two
+populations: same-aircraft re-key distances are p50 1.5 km / p90 4.3 km (mostly
+n=2 bottom-up solves whose own position error is ~2.4 km median), which overlaps
+the wrong-aircraft distances entirely. What *can* separate them is that the
+follow lane already supplies every solve an established track needs — so a
+bottom-up solve arriving at a freshly-followed key is either a duplicate of that
+aircraft (harmful: it competes with the anchored solve and drags the filter) or
+a different one (harmful: it steals the key). Neither should join.
+
+So in **binding mode only**, `multinode_key_decision`'s proximity scan gives the
+follow lane ownership of the keys it follows:
+
+- a key the lane published on within `DARK_FOLLOW_OWN_S` (6 s, three follow-solve
+ intervals) is **not a join candidate** for a bottom-up (non-anchored,
+ non-ADS-B) solve;
+- if the nearest such key is within `DARK_FOLLOW_SHADOW_KM` (2.0 km) of the
+ solve — same dead-reckoning and same distance as `key_dist_km` — the solve is
+ refused outright: `how == "shadowed"`, history outcome `shadowed_by_follow`
+ carrying `follow_key` and the distance, counter `dark_bottomup_shadowed`, no
+ publish, no KF update, no entry written;
+- farther away it falls through to the non-followed candidates and, failing
+ those, mints a key of its own as before.
+
+The anchor and ADS-B branches are untouched, which is what keeps the follow
+lane's own solves landing on their key (they are anchored, and the anchor branch
+returns before the scan). The `n>=3` case rule 2 also covers implicitly: an n=2
+bottom-up solve can no longer join a recently-followed key whose last published
+solve had `n>=3`. `shadow` and `off` modes never consult ownership at all, so
+the keying rule is byte-identical to before there.
+
+| Constant | Value | File |
+|---|---|---|
+| `DARK_FOLLOW_MODE` | `shadow` (env) | `core/state.py` |
+| `DARK_FOLLOW_OWN_S` | 6.0 s (env) | `services/dark_follow.py` |
+| `DARK_FOLLOW_SHADOW_KM` | 2.0 km (env) | `services/dark_follow.py` |
+| `DARK_FOLLOW_MAX_AGE_S` | 20 s (env) | `services/dark_follow.py` |
+| `DARK_FOLLOW_MIN_SOLVES` / `DARK_FOLLOW_MIN_NODES` | 3 / 3 | `services/dark_follow.py` |
+| `DARK_FOLLOW_MAX_VEL_SIGMA_MS` | 60 m/s (env) | `services/dark_follow.py` |
+| `DARK_FOLLOW_INTERVAL_S` | 2.0 s (env) | `services/dark_follow.py` |
+| `DARK_FOLLOW_COOLDOWN_S` | 30 s (env) | `services/dark_follow.py` |
+| `_MAX_CONSECUTIVE_REJECTS` | 2 | `services/dark_follow.py` |
+| `_TARGETS_TTL_S` (pseudo-state cache) | 1.0 s | `services/dark_follow.py` |
+| Gate caps `_MAX_DELAY_GATE_US` / `_MAX_DOPPLER_GATE_HZ` | 40 µs / 100 Hz | `services/dark_follow.py` |
+
+Observability: `/api/test/solver-stats` `counters` carries the funnel
+`dark_follow_targets` (a live gauge) → `dark_follow_claims` →
+`dark_follow_inputs` → `dark_follow_published`, plus `dark_follow_dropped` and
+`dark_bottomup_shadowed` (the ownership refusals above). Records are classified
+`lane: "dark_follow"` in `lane_split` and kept out of the bottom-up dark funnel,
+and each carries `guess_source: "prediction"` and `follow_key`. A shadowed
+record stays in the **bottom-up** funnel — it is a bottom-up solve — and shows
+up as `rejects.by_reason["shadowed_by_follow"]`, with `follow_key` naming the
+track that refused it.
---
@@ -291,7 +468,7 @@ LM's SNR weighting maps to a uniform weight of 1.0.
```mermaid
flowchart TD
- frame["pipeline.process_frame
passive_radar.py:672"]
+ frame["PassiveRadarPipeline.process_frame
pipeline/passive_radar.py"]
frame --> tracker["retina_tracker
Kalman + GNN"]
tracker --> geo["_run_geolocation per track
with new data"]
@@ -345,7 +522,7 @@ flowchart TD
classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3
```
-`compute_overlap_zone` (`libs/retina-analytics/.../association.py:578`)
+`compute_overlap_zone` (`libs/retina-analytics/.../association.py`)
underlies both the confirmed-track association round and the overlap-grid
cache: it fast-prunes non-overlapping node pairs by receiver separation,
grids the shared coverage at `ASSOC_GRID_STEP_KM` on six altitude layers that
@@ -353,22 +530,22 @@ must match the solver's `_SOLVER_ALT_LAYERS_KM`, and requires each grid
column to fall in **both** beams (`_point_in_beam`, FOV-aware only when
`FOV_MODE=active`).
-| Constant | Value | File:line |
+| Constant | Value | Defined in |
|---|---|---|
-| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py:194` |
-| Single-node min detections | 3 | `passive_radar.py:356-361` |
-| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py:59` |
-| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py:220` |
-| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py:22` |
-| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py:23` |
-| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py:31`, `association.py:1043` |
-| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.py:883` |
-| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.py:884` |
-| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py:166` |
-| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py:57-58` |
-| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.py:2160` |
-| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py:21` |
-| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py:111` |
+| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py` (applied as `_GEO_INTERVAL_S` in `_run_geolocation`) |
+| Single-node min detections | 3 | `pipeline/passive_radar.py` (`_geolocate_track_event`, `min_det`) |
+| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py` |
+| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py` |
+| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py` |
+| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py` |
+| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py`, `association.py` |
+| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.compute_overlap_zone` (default arg) |
+| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.compute_overlap_zone` (default arg) |
+| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py` |
+| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py` |
+| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.InterNodeAssociator.format_track_pairs_for_solver` (local) |
+| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py` |
+| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py` |
---
@@ -376,17 +553,34 @@ column to fall in **both** beams (`_point_in_beam`, FOV-aware only when
The centerpiece: every candidate from either lane, once dequeued from
`solver_queue`, runs through `_process_solver_item`
-(`services/tasks/solver.py:1344`) as a strict, ordered chain. A failure at
+(`services/tasks/solver.py`) as a strict, ordered chain. A failure at
any gate stops the chain, bumps a counter, and (from 6.5 onward) writes a
named record to solve history.
+**6.2 claims on publication, not on admission.** The suppression rule is
+"this aircraft is already on the map at this width, at every track it is
+built from" — so `_resolve_slot_covered` only *reads* the claims, and
+`_record_resolve_slot` takes them from the publish path, with the
+**post-trim survivors** (`result["source_track_ids"]`, rebuilt from the
+surviving `track_ids_by_node`). Claiming on admission instead meant a
+candidate that never reached the map still blacked out every later candidate
+sharing any of its track ids for the full 12 s — including *other aircraft's*,
+since tracker track ids are shared across the association candidates of
+different aircraft (the same finding behind `_supersession_match`'s spatial
+guard; see Caveats). Live that ran at ~1 537 skips per 646 dark attempts per
+30 min: more candidates suppressed than solved, by a factor of two. The price
+of the split is that the check no longer claims under the same lock, so two
+workers can now both solve duplicates that arrived together; that costs one
+extra solve and is resolved downstream by keying and supersession, which
+handle exactly this case already.
+
```mermaid
flowchart TD
deq["Dequeue (s_in, node_cfgs, enqueued_at)"]
deq --> g61{"6.1 Staleness
age_s > _SOLVER_MAX_QUEUE_AGE_S 45.0s?"}
g61 -->|"yes"| f61["solver_stale_drops
(no history record)"]:::inert
- g61 -->|"no"| g62{"6.2 Re-solve suppression
_claim_resolve_slot False?"}
- g62 -->|"yes"| f62["solver_resolve_skips"]:::inert
+ g61 -->|"no"| g62{"6.2 Re-solve suppression
_resolve_slot_covered (pure)?"}
+ g62 -->|"yes"| f62["solver_resolve_skips (+_dark)
+ skip record with blockers"]:::inert
g62 -->|"no"| g63["6.3 Solve dispatch:
no guess -> bare solve_fn;
n>=3 -> consensus? then
_solve_best_altitude (sweep);
n=2 -> _solve_best_altitude_n2
(single altitude)"]
g63 -->|"exception"| f63["solver_failures +
solver_fail_exception,
result=None"]:::inert
g63 --> g64{"6.4 Trim & resolve (recovery):
guess AND n>=4 AND
rms_delay > 3.0us?"}
@@ -430,14 +624,14 @@ flowchart TD
```
`SOLVER_CONSENSUS_MODE` is `off` in production (see the mode-flag table in
-[`architecture.md:94-110`](architecture.md#feature-gates)), so in practice
+[`architecture.md`](architecture.md#feature-gates)), so in practice
this sub-branch never reaches `active` outside staging.
### The LM itself
-`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py:518`,
+`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py`,
invoked through the process pool via `_pool_solve_multinode`
-(`services/tasks/solver.py:1915`).
+(`services/tasks/solver.py`).
```mermaid
flowchart TD
@@ -453,25 +647,78 @@ flowchart TD
m6 -->|"no"| m7["vz_saturated if vz on bound;
rms recomputed unweighted;
cov_en_km2 from s^2(J^T J)^-1"]
m7 --> alt{"n_nodes >= 3?"}
- alt -->|"yes"| sweep["_solve_best_altitude wrapper:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"]
+ alt -->|"yes"| mode{"SOLVER_ALT_MODE"}
+ mode -->|"sweep (default)"| sweep["_solve_best_altitude:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"]
+ mode -->|"free"| freealt["_solve_best_altitude:
ONE pool call to
solve_multinode_multistart,
SOLVER_FREE_ALT_STARTS start
layers (1 by default), z solved"]
alt -->|"no, n=2"| single["_solve_best_altitude_n2:
one LM call at the
association altitude"]
classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3
```
-| Constant | Value | File:line |
+#### `SOLVER_ALT_MODE` — how the n>=3 solve gets its altitude
+
+`solve_multinode` pins altitude from `initial_guess.alt_km`, so the fix is only
+as good as the altitude the caller found for it. `sweep`, the default, searches
+the six fixed layers of `_SOLVER_ALT_LAYERS_KM` — 2 km apart, so the pin is
+systematically up to 1 km wrong. On noise-free replay of this fleet's geometry
+that quantisation alone left `rms_delay` at a 1.76 us median against the 3.0 us
+gate at 6.5, while a solve at the true altitude reaches 0. Most of the gate's
+budget is spent on the ladder, and the residual left over gets blamed on nodes:
+trimming (6.4) drops measurements that were never the problem.
+
+`free` instead calls `solve_multinode_multistart`, which runs the LM with
+altitude as a sixth unknown (state `[x, y, z, vx, vy, vz]`, z bounded
+0.05–20 km, the `vz` bound unchanged) from `SOLVER_FREE_ALT_STARTS` start
+layers, keeping the lowest `rms_delay`. It is also cheaper: **one** process-pool
+round trip per candidate instead of six, each of which pickles the node configs
+the input needs.
+
+`SOLVER_FREE_ALT_STARTS` defaults to **1** — the layer nearest the association
+guess, or the guess altitude itself when that came from ADS-B and was spliced
+into the ladder (the same splice the sweep does). Freeing z removes the
+ladder's quantisation but not the LM's locality, and extra starts are what
+would stop a solve settling on the wrong side of a bistatic ellipse; on this
+fleet's geometry they had almost nothing to stop. Over a 20-minute window of
+1019 free-mode solves on test, the three starts' `rms_delay` differed by more
+than 0.1 us in **13** of them, and the nearest-layer start was more than 0.5 us
+worse than the best start in **2** — ~0.2% of solves helped, at three times the
+solver CPU, while the pool is the binding constraint (~1.7 attempts/s against a
+2.0 s average latency on two workers). Set it above 1 for a geometry where that
+locality does bite; `_free_alt_starts` clamps it into `[1, len(layers)]` and
+values above 1 give the same neighbour window as before, so `3` restores the
+original behaviour exactly.
+
+At n=2 the mode is inert — four residuals cannot support six unknowns, so the
+geolocator pins altitude regardless and `_solve_best_altitude_n2` is unchanged.
+Trimming re-solves through `_solve_best_altitude`, so a trim round inherits
+whichever mode its first solve used.
+
+Both modes stamp `altitude_mode` (`"free"` / `"pinned"`) on every
+`mlat_solve_history` record, published or rejected; `free` adds `alt_starts_km`,
+`alt_start_rms_us` (each start's residual) and `z_saturated` (the altitude
+analogue of `vz_saturated` — z stopped on a bound rather than converging, so
+`alt_m` is the bound and not a fit). That is the comparison channel: deploy one
+mode per environment and read the two lanes' `rms_delay` and `gt_error_km` off
+`/api/test/mlat-history`.
+
+| Mode | Pool calls per n>=3 candidate | Altitude |
+|---|---|---|
+| `sweep` (default) | 6 (one per layer) | quantised to the nearest layer |
+| `free` | 1 (`SOLVER_FREE_ALT_STARTS` starts inside it, 1 by default) | solved, 0.05–20 km |
+
+| Constant | Value | Defined in |
|---|---|---|
-| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py:701` |
-| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py:744` |
-| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py:160-162` |
-| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py:132` |
-| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py:173` |
-| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py:205` |
-| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py:56` |
-| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py:807` |
-| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py:154` |
-| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py:51,52` |
-| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py:57,63` |
+| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py` |
+| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py` (`_SOLVER_RESOLVE_INTERVAL_S`, `_resolve_slot_covered`, `_record_resolve_slot`) |
+| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py` |
+| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py` (`_SOLVER_RMS_DELAY_MAX_US`) |
+| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py` |
+| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py` |
+| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py` |
+| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py` |
+| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py` |
+| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py` |
+| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py` |
---
@@ -507,14 +754,17 @@ flowchart TD
popped --> store["state.multinode_tracks[key] = result"]
blocked --> store
store --> archive["track-archive buffer append"]
- archive --> histpub["_record_solve_history: published"]
+ archive --> claimslot["_record_resolve_slot:
claim the POST-TRIM survivors
for _SOLVER_RESOLVE_INTERVAL_S"]
+ claimslot --> histpub["_record_solve_history: published"]
histpub --> feed["build_combined_aircraft_json
(1 Hz flush)"]
- feed --> gN2{"n=2 display gate:
solve_count < MN_N2_MIN_SOLVES 2?"}
+ feed --> gExpiry{"entry expiry:
mn-adsb-* age_s > 60s,
mn-dark-* age_s >
MN_DARK_EXPIRY_S 30.0s?"}
+ gExpiry -->|"yes"| dropExpired["popped from
state.multinode_tracks,
anomaly hex discarded"]:::inert
+ gExpiry -->|"no"| gN2{"n=2 display gate:
solve_count < MN_N2_MIN_SOLVES 2?"}
gN2 -->|"yes"| retainN2["retained, not rendered"]:::inert
gN2 -->|"no"| gOneshot{"n>=3 one-shot:
solve_count==1 AND
age_s > MN_ONESHOT_TTL_S 15.0s?"}
gOneshot -->|"yes"| dropOneshot["not rendered"]:::inert
- gOneshot -->|"no"| dr["dead reckoning,
capped 30s"]
+ gOneshot -->|"no"| dr["dead reckoning,
capped MN_DR_CAP_S 15.0s,
then holds"]
dr --> dedup["dedup_aircraft:
rank by _DEDUP_SOURCE_RANK,
3.0km / 2000ft gate"]
dedup --> out["aircraft.json + WebSocket -> map"]
@@ -525,26 +775,63 @@ flowchart TD
| Value | Set at | Meaning |
|---|---|---|
-| `multinode_solve` | `aircraft_feed.py:132` | published multi-node solve |
-| `solver_adsb_seed` | `track_gates.py:330` | single-node LM with fresh ADS-B fix |
-| `solver_single_node` | `track_gates.py:330` | single-node LM, no ADS-B |
-| `single_node_ellipse_arc` | `track_gates.py:378` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint |
-| `adsb_single_node` | `aircraft_feed.py:_claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` |
-| `known_lane_truth_match` / `known_lane_ghost` | `known_lane.py:260` | accuracy-sample-only, not a feed entry |
-
-| Constant | Value | File:line |
+| `multinode_solve` | `aircraft_feed.multinode_to_aircraft` | published multi-node solve |
+| `solver_adsb_seed` | `track_gates.track_entry` | single-node LM with fresh ADS-B fix |
+| `solver_single_node` | `track_gates.track_entry` | single-node LM, no ADS-B |
+| `single_node_ellipse_arc` | `track_gates.track_entry` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint |
+| `adsb_single_node` | `aircraft_feed._claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` |
+| `known_lane_truth_match` / `known_lane_ghost` | `known_lane._record_accuracy` | accuracy-sample-only, not a feed entry |
+
+| Constant | Value | Defined in |
|---|---|---|
| `_MN_ASSOC_MAX_DIST_KM` / `_MN_ASSOC_MAX_AGE_S` (identity step 2/3) | 6.0 km / 60.0 s | `services/tasks/solver.py` |
| `_MN_ASSOC_DRIFT_KM_PER_S` / `_MN_ASSOC_MAX_DIST_CAP_KM` (step 3 only — the gate grows with the matched entry's age) | 0.13 km/s / 12.0 km | `services/tasks/solver.py` |
| Supersession gate (`_supersession_match`) — the same age-scaled `_mn_assoc_gate_km` and `_MN_ASSOC_MAX_AGE_S` as step 3, applied to the solve's RAW position | 6.0 + 0.13·dt km, cap 12.0 / 60.0 s | `services/tasks/solver.py` |
-| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py:77` |
-| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py:63` |
-| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py:66` |
-| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py:37-43` |
-| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py:131-139` |
-| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py:49-50` |
-| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py:167` |
-| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py:206,213` |
+| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py` |
+| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py` |
+| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py` |
+| `MN_DR_CAP_S` (dead-reckoning horizon past the last solve; the entry then holds its last DR'd point) | 15.0 s | `config/constants.py` |
+| `MN_DARK_EXPIRY_S` (entry expiry, `mn-dark-*` only — `mn-adsb-*` keeps 60 s) | 30.0 s | `config/constants.py` |
+| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py` |
+| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py` |
+| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py` (`_DEDUP_PROXIMITY_KM`, `_DEDUP_ALT_GATE_FT`) |
+| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py` |
+| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py` |
+
+---
+
+## 7. Reading the pipeline from outside
+
+Three endpoints answer questions about the two lanes, and each has a shape
+worth knowing before it is trusted.
+
+**`/api/test/mlat-history`** dumps solve records. Both lanes write their own
+deque (`state.mlat_solve_history`, `state.mlat_solve_history_known`) and every
+reader merges them. `?lane=dark|known|adsb|all` narrows the answer;
+`?limit=` (default 1 000, max 5 000) is applied **per lane**, so a known-lane
+burst can never push dark records out of the response — the flat cap that
+preceded it left a 30 min request holding only the newest ~6 min of dark
+records, which reads exactly like a quiet dark lane. `lane_counts` is
+reported pre-cap so a truncated `records` list is legible.
+`?kind=resolve_skips` dumps a different store entirely — see below.
+
+**`/api/test/solver-stats`** is the Solver Report panel's source. Its funnel,
+error percentiles, ghosts, fragmentation, `contamination` and `resolve_skips`
+are all the DARK lane; `lane_split` gives the per-lane record counts and
+`known_lane` that lane's own numbers.
+
+| Block | Says | Watch for |
+|---|---|---|
+| `contamination` | Of the dark records that matched ground truth, how many carried a node that could not see the aircraft (`foreign_node_ids` on the record; verdict is the associator's own `_point_in_beam`, the same gate known-lane claiming uses) | `pct` is the live version of the offline ~60 % the cluster-splitting work exists to move. Records with no GT match, or no registered geometry for any contributing node, are **out of the denominator** — abstention, not innocence |
+| `resolve_skips` | Candidates the re-solve suppression refused in this window, from `state.solver_resolve_skips_recent`, with the claims that blocked each one | `attempts_ratio` is all-lane skips over DARK attempts. It read ~2.4 while 6.2 claimed on admission; with the claim on publication it should sit at or below 0.5. The deque holds 500 entries against a live rate of tens per minute, so read `window_effective_minutes` before reading `total` as a window count |
+| `counters.resolve_skips_dark` | Dark share of the since-boot skip counter | — |
+| `counters.node_frames_rate_limited` | Frames `NODE_FRAME_MIN_INTERVAL_S` refused before the tracker saw them (Gate B in §2) | Not the same event as `/api/admin/metrics`' `frames_dropped`, which is `frame_queue` saturation and normally reads zero |
+
+A skip is deliberately **not** a solve-history record: skips outrun dark
+records roughly two to one on the live fleet, so writing them into
+`mlat_solve_history` would evict exactly the solves an investigation needs.
+They are also not counted as attempts or rejects — a skipped candidate never
+reached a solve.
---
@@ -565,16 +852,31 @@ flowchart TD
`mn_superseded` / `mn_superseded_blocked` in `/api/test/solver-stats`
(`fragmentation`) and `superseded_keys` / `superseded_blocked` on each
published `mlat_solve_history` record are how this is watched.
+- **A displayed dark entry is only as good as its solve age, and the budgets
+ say so.** Measured on the test droplet over 20 minutes (dark multinode feed
+ entries against ground truth): median position error 1.05 km under 3 s of
+ solve age, 1.21 km at 3–8 s, 1.50 km at 8–15 s, 2.02 km at 15–30 s (7% more
+ than 5 km off), and 3.99 km at 30–60 s (12% of all displayed dark entries,
+ 32% more than 5 km off). `MN_DR_CAP_S` (15 s) and `MN_DARK_EXPIRY_S` (30 s)
+ are cut at that curve's two knees. Both were looser — a 30 s cap and a flat
+ 60 s expiry — from when the solver refused to re-solve the same tracks inside
+ `SOLVER_RESOLVE_INTERVAL_S` = 12 s and the extra window bought coverage; a
+ dark aircraft now re-solves every 1–3 s while it is tracked, so a 15 s gap is
+ a lost track rather than a cadence gap and extrapolating it only invents
+ motion. `mn-adsb-*` entries keep the 60 s expiry: a transponder hex anchors
+ them, so the same gap is the ADS-B feed breathing. The frontend's matching
+ budgets are `DR_ICON_HIDE_DISTANCE_DARK_M` (3 km) and `UNCERTAINTY_DR_CAP_S`
+ (30 s).
- **Node-trust residuals are measure-only.** `node_bias.py` computes them but
- nothing in the solver consumes them yet (`node_bias.py:33-40` docstring).
+ nothing in the solver consumes them yet (`node_bias.py` module docstring).
- **`docs/pipeline.md` §3 is stale.** It predates the known lane and the
process-pool inline fallback; this doc supersedes it for both topics.
- **The bottom-up doppler gate is inert.** `doppler_gate_hz` in the dark
lane's coarse pairing step is defined but the grid gate is delay-only in
- practice (`libs/retina-analytics/.../association.py:884`).
+ practice (`association.compute_overlap_zone`'s `doppler_gate_hz`).
- **Production runs with every mode flag off** except `KNOWN_LANE_MODE`, which
is `binding` everywhere by code default and is set in no environment's
`.env`. The in-repo statement of what each environment sets is
- [`architecture.md:94-110`](architecture.md#feature-gates); the actual
+ [`architecture.md`](architecture.md#feature-gates); the actual
values live in the gitignored `backend/.env` on each host, not in this
repo.
diff --git a/frontend/src/components/LiveAircraftMap.tsx b/frontend/src/components/LiveAircraftMap.tsx
index b0ef25e9..727b6e8e 100644
--- a/frontend/src/components/LiveAircraftMap.tsx
+++ b/frontend/src/components/LiveAircraftMap.tsx
@@ -21,6 +21,8 @@ import {
GT_PRUNE_GRACE_MS,
POSITION_SOURCE_ARC_ONLY,
ARC_DR_MAX_S,
+ MLAT_HISTORY_REFRESH_MS,
+ newSolveArrived,
groundTruthKey,
applyGroundTruthFixes,
pruneGroundTruthFixes,
@@ -1520,12 +1522,16 @@ export default function LiveAircraftMap() {
: null;
// Per-solve history for the selected MLAT track (debug): fetched once per
- // selection + refreshed on the backend's ~30 s recording cadence. Tagged
- // with the hex it was fetched for so a selection change never shows the
- // previous track's solves while the new fetch is in flight.
+ // selection, then polled every MLAT_HISTORY_REFRESH_MS. Tagged with the hex
+ // it was fetched for so a selection change never shows the previous track's
+ // solves while the new fetch is in flight.
const selectedMnHex =
selectedAc?.position_source === "multinode_solve" ? selectedAc.hex : null;
+ const selectedMnSeen = selectedMnHex ? selectedAc?.seen ?? null : null;
const [mlatHistory, setMlatHistory] = useState(null);
+ // The poll's loader, published for the `seen` watcher below to call. A ref
+ // rather than a dependency so a refetch never restarts the interval.
+ const reloadMlatHistoryRef = useRef(null);
useEffect(() => {
if (!selectedMnHex) {
setMlatHistory(null);
@@ -1537,11 +1543,28 @@ export default function LiveAircraftMap() {
if (!cancelled && d && d.hex === selectedMnHex) setMlatHistory(d);
});
};
+ reloadMlatHistoryRef.current = load;
load();
- const interval = setInterval(load, 30000);
- return () => { cancelled = true; clearInterval(interval); };
+ const interval = setInterval(load, MLAT_HISTORY_REFRESH_MS);
+ return () => {
+ cancelled = true;
+ clearInterval(interval);
+ reloadMlatHistoryRef.current = null;
+ };
}, [selectedMnHex]);
+ // A fall in `seen` is the feed announcing a fresh solve for this track — the
+ // one event worth a fetch off the poll's schedule (see newSolveArrived).
+ // Dark solves arrive every 1-3 s while a track is held, faster than the
+ // poll, and the dots are the surface someone selected the aircraft to read.
+ const prevMnSeenRef = useRef({ hex: null, seen: null });
+ useEffect(() => {
+ const prev = prevMnSeenRef.current;
+ const next = { hex: selectedMnHex, seen: selectedMnSeen };
+ prevMnSeenRef.current = next;
+ if (newSolveArrived(prev, next)) reloadMlatHistoryRef.current?.();
+ }, [selectedMnHex, selectedMnSeen]);
+
// Nodes with a live detection of the selected simulated object — read from
// the detection-presence oracle (per-aircraft signals ∪ the detecting_nodes
// feed key). trailTick advances on every ingest, so this refreshes at the
diff --git a/frontend/src/components/map/constants.ts b/frontend/src/components/map/constants.ts
index 15295401..84486a16 100644
--- a/frontend/src/components/map/constants.ts
+++ b/frontend/src/components/map/constants.ts
@@ -71,10 +71,10 @@ export const ADSB_SINGLE_ARC_ICON_MULTIPLE = 2.5;
export const ARC_DR_MAX_S = 10;
// Dead-reckoning drift budget (metres) past which the plane ICON stops being
-// drawn as an ordinary live target. The backend keeps feeding an mn entry for
-// 60 s after its last solve, dead-reckoned the whole way, so a target whose
-// solves stop is drawn kilometres from where it actually is — the icon reads as
-// a real target because nothing about it looks stale.
+// drawn as an ordinary live target. The backend keeps feeding an mn entry past
+// its last solve (MN_DARK_EXPIRY_S 30 s dark, 60 s ADS-B-assisted), so a target
+// whose solves stop is drawn kilometres from where it actually is — the icon
+// reads as a real target because nothing about it looks stale.
//
// The budget is LANE-AWARE, because the two multi-node lanes re-solve at very
// different rates and one budget cannot describe both:
@@ -86,24 +86,28 @@ export const ARC_DR_MAX_S = 10;
// displacement gate (_MAX_DISPLACEMENT_KM) and sits under backend dedup's
// 3 km proximity gate, so a second icon cannot appear at the true position
// while the drifted one is still shown.
-// * Dark lane (mn-dark-*) — 6 km. 2 km is a budget the dark lane cannot meet
-// at all: the solver refuses to re-solve the same tracks inside
-// SOLVER_RESOLVE_INTERVAL_S = 12 s, and 2000 m / 12 s is 324 kt while
-// simulated dark speeds run 233–525 kt. Measured dark cadence is 9.0 s
-// median / 24.9 s p90 with ~3 s of publish latency on top, and a 26-minute
-// capture had 48% of published dark track-frames hidden by the 2 km gate —
-// 80% would be drawable at 4 km, 90% at 6 km.
+// * Dark lane (mn-dark-*) — 3 km. It was 6 km, sized when the solver refused
+// to re-solve the same tracks inside SOLVER_RESOLVE_INTERVAL_S = 12 s and
+// the measured dark cadence was 9.0 s median / 24.9 s p90: at that cadence
+// 2 km hid 48% of published dark track-frames and only 6 km got 90% of them
+// drawn. Dark solves now land every 1–3 s, so the drift a healthy track
+// accrues is metres, and the budget stopped buying coverage and started
+// buying wrong icons: measured against ground truth over 20 minutes, dark
+// entries run 1.50 km median error at 8–15 s of solve age but 2.02 km at
+// 15–30 s (7% over 5 km) and 3.99 km at 30–60 s (32% over 5 km). 3 km sits
+// at that knee — it still tolerates the odd missed solve, and it no longer
+// draws a confident icon 6 km from any aircraft.
//
// Exceeding the budget does not mean the same thing in both lanes, so neither
// does the rendering (see drIconState in icons.ts): an assisted track over
// budget is a genuine anomaly and loses its icon, while a dark track over
-// budget is the normal consequence of a 12 s re-solve floor and is drawn in a
+// budget is the normal consequence of a missed solve and is drawn in a
// degraded "stale solve" style instead — "solved but stale" has to stay
// distinguishable from "not solved". Either way the TRACK stays alive —
// stores, trails, list, selection — so a new solve restores the normal icon on
// the next 2 Hz render.
export const DR_ICON_HIDE_DISTANCE_M = 2000;
-export const DR_ICON_HIDE_DISTANCE_DARK_M = 6000;
+export const DR_ICON_HIDE_DISTANCE_DARK_M = 3000;
// Ground speed (knots) assumed when a multi-node entry carries no `gs` at all.
// The backend deletes gs from entries whose velocity vector it does not trust
diff --git a/frontend/src/components/map/icons.test.ts b/frontend/src/components/map/icons.test.ts
index c9ab0047..b6f4cc11 100644
--- a/frontend/src/components/map/icons.test.ts
+++ b/frontend/src/components/map/icons.test.ts
@@ -139,15 +139,17 @@ describe("lane-aware drift budget", () => {
expect(isDarkMultinodeSolve(null)).toBe(false);
});
- it("keeps a dark solve inside budget across one 12 s re-solve floor", () => {
+ it("keeps a dark solve inside budget across a missed solve", () => {
// 450 kt for 12 s is ~2.8 km: over the 2 km budget the old gate applied to
- // every lane, comfortably inside the dark one. This is the 48%-hidden bug.
+ // every lane, still inside the dark one. This is the 48%-hidden bug.
expect(hideDrIcon({ ...dark(), seen: 12 }, NOW)).toBe(false);
expect(hideDrIcon({ ...assisted(), seen: 12 }, NOW)).toBe(true);
});
it("still trips the dark budget on a long enough gap", () => {
- // 450 kt for 30 s is ~6.9 km — past 6 km.
+ // 450 kt for 15 s is ~3.5 km — past the 3 km dark budget. Under the old
+ // 6 km one this same entry was drawn as an ordinary live target.
+ expect(hideDrIcon({ ...dark(), seen: 15 }, NOW)).toBe(true);
expect(hideDrIcon({ ...dark(), seen: 30 }, NOW)).toBe(true);
});
});
diff --git a/frontend/src/components/map/index.ts b/frontend/src/components/map/index.ts
index b19b5bac..9604040d 100644
--- a/frontend/src/components/map/index.ts
+++ b/frontend/src/components/map/index.ts
@@ -19,6 +19,7 @@ export {
solveSigmaM,
solveUncertaintyRadiusM,
} from "./uncertainty";
+export { MLAT_HISTORY_REFRESH_MS, newSolveArrived } from "./mlatHistory";
export { mergeTrailPositions, sampleTrailPositions, buildTrailSegments } from "./trails";
export { PLANE_PATH, getAircraftColor, altitudeColor, ALTITUDE_LEGEND, aircraftIconSize, makeAircraftIcon, makeDroneIcon, nodeIcon, drDriftM, drGsKt, drIconBudgetM, drIconState, hideDrIcon, isDarkMultinodeSolve, isMultinodeSolve } from "./icons";
export { FitBounds, ViewportTracker, MapClickClear } from "./MapControls";
diff --git a/frontend/src/components/map/mlatHistory.test.ts b/frontend/src/components/map/mlatHistory.test.ts
new file mode 100644
index 00000000..be564f86
--- /dev/null
+++ b/frontend/src/components/map/mlatHistory.test.ts
@@ -0,0 +1,43 @@
+import { describe, it, expect } from "vitest";
+import { MLAT_HISTORY_REFRESH_MS, newSolveArrived } from "./mlatHistory";
+
+const at = (hex: string | null, seen: number | null) => ({ hex, seen });
+
+describe("MLAT_HISTORY_REFRESH_MS", () => {
+ it("keeps up with the 1-3 s dark solve cadence", () => {
+ expect(MLAT_HISTORY_REFRESH_MS).toBe(3_000);
+ });
+});
+
+describe("newSolveArrived", () => {
+ it("fires when seen falls on the same track", () => {
+ // 8 s old, then 0.4 s old: a solve landed between the two flushes.
+ expect(newSolveArrived(at("mnabc123", 8), at("mnabc123", 0.4))).toBe(true);
+ });
+
+ it("does not fire while the same solve simply ages", () => {
+ expect(newSolveArrived(at("mnabc123", 2), at("mnabc123", 5))).toBe(false);
+ expect(newSolveArrived(at("mnabc123", 2), at("mnabc123", 2))).toBe(false);
+ });
+
+ it("does not compare ages across a selection change", () => {
+ // The new track's seen is a different clock, and the selection effect
+ // refetches from scratch anyway.
+ expect(newSolveArrived(at("mnabc123", 30), at("mndef456", 1))).toBe(false);
+ expect(newSolveArrived(at(null, null), at("mnabc123", 1))).toBe(false);
+ });
+
+ it("does not fire on a deselection", () => {
+ expect(newSolveArrived(at("mnabc123", 8), at(null, null))).toBe(false);
+ });
+
+ it("needs an age on both sides", () => {
+ expect(newSolveArrived(at("mnabc123", null), at("mnabc123", 1))).toBe(false);
+ expect(newSolveArrived(at("mnabc123", 8), at("mnabc123", null))).toBe(false);
+ });
+
+ it("treats a zeroed age as a new solve", () => {
+ // The backend clamps a negative age to 0, so 0 is a real value here.
+ expect(newSolveArrived(at("mnabc123", 4), at("mnabc123", 0))).toBe(true);
+ });
+});
diff --git a/frontend/src/components/map/mlatHistory.ts b/frontend/src/components/map/mlatHistory.ts
new file mode 100644
index 00000000..7cac1b5f
--- /dev/null
+++ b/frontend/src/components/map/mlatHistory.ts
@@ -0,0 +1,52 @@
+/* ------------------------------------------------------------------ */
+/* Refresh policy for the selected track's per-solve history */
+/* ------------------------------------------------------------------ */
+
+/**
+ * When LiveAircraftMap should re-read `/api/test/mlat-history` for the
+ * multi-node aircraft the operator has selected. That payload draws
+ * MlatSolveHistoryLayer's per-solve dots and the detail panel's solve table —
+ * the surface someone selects an aircraft in order to read, so it lagging the
+ * marker is the whole bug.
+ *
+ * Two rules, because a poll alone cannot be both cheap and prompt: a floor
+ * interval that runs regardless, and an event that fires inside it. No
+ * Leaflet, no React — unit-tested on its own.
+ */
+
+/**
+ * Poll interval for the selected track's solve history, in ms.
+ *
+ * It was 30 s, sized when the solver refused to re-solve the same tracks
+ * inside SOLVER_RESOLVE_INTERVAL_S = 12 s, so a poll could not miss much.
+ * Dark solves now land every 1–3 s while a track is held, and at 30 s the dots
+ * trailed the live marker by up to half a minute — a decomposition of the
+ * track that was mostly missing. 3 s tracks that cadence at ~20 requests a
+ * minute for ONE selected track, and only while something is selected.
+ */
+export const MLAT_HISTORY_REFRESH_MS = 3_000;
+
+/** The two fields that decide a refetch: which track, and how old its last
+ * solve was when the feed last said. */
+export interface SelectedSolveAge {
+ /** Feed hex of the selected multi-node entry, or null when none is. */
+ hex: string | null;
+ /** `seen` from that entry — the backend's age of its last solve, seconds. */
+ seen: number | null;
+}
+
+/**
+ * True when the feed has just announced a fresh solve for the SAME selected
+ * track, so its history is worth refetching ahead of the next poll.
+ *
+ * `seen` is an age, so it climbs on every flush and only ever falls when a new
+ * solve replaced the one it was measuring — that fall is the event. A change
+ * of `hex` is not a comparison at all but a different clock, and the selection
+ * effect refetches from scratch anyway; a null on either side is a track that
+ * is not a multi-node solve, or a feed entry that never carried an age.
+ */
+export function newSolveArrived(prev: SelectedSolveAge, next: SelectedSolveAge): boolean {
+ if (prev.hex !== next.hex || next.hex == null) return false;
+ if (prev.seen == null || next.seen == null) return false;
+ return next.seen < prev.seen;
+}
diff --git a/frontend/src/components/map/uncertainty.test.ts b/frontend/src/components/map/uncertainty.test.ts
index 0a3abfa5..ddb38af7 100644
--- a/frontend/src/components/map/uncertainty.test.ts
+++ b/frontend/src/components/map/uncertainty.test.ts
@@ -50,11 +50,13 @@ describe("solveSigmaM", () => {
);
});
- it("stops growing at the 60 s dead-reckoning cap", () => {
+ it("stops growing at the dead-reckoning cap", () => {
+ // The cap tracks the backend's dark-lane entry expiry (MN_DARK_EXPIRY_S).
+ expect(UNCERTAINTY_DR_CAP_S).toBe(30);
const ac = mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25 });
const atCap = solveSigmaM(ac, UNCERTAINTY_DR_CAP_S);
expect(solveSigmaM(ac, 600)).toBe(atCap);
- expect(atCap).toBeCloseTo(Math.sqrt(200 * 200 + 1500 * 1500), 6);
+ expect(atCap).toBeCloseTo(Math.sqrt(200 * 200 + 750 * 750), 6);
});
it("clamps a negative age to the solve epoch", () => {
@@ -106,21 +108,21 @@ describe("solveUncertaintyRadiusM", () => {
).toBeCloseTo(UNCERTAINTY_K95 * 200, 6);
});
- it("grows while dead-reckoning, then holds at the 60 s cap", () => {
+ it("grows while dead-reckoning, then holds at the cap", () => {
const ac = mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25, _updatedAt: NOW - 10_000 });
const fresh = solveUncertaintyRadiusM(mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25 }), NOW);
const aged = solveUncertaintyRadiusM(ac, NOW);
expect(aged).toBeGreaterThan(fresh);
// Past the cap the radius stops moving.
- const at60 = solveUncertaintyRadiusM(
- mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25, _updatedAt: NOW - 60_000 }),
+ const atCap = solveUncertaintyRadiusM(
+ mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25, _updatedAt: NOW - UNCERTAINTY_DR_CAP_S * 1000 }),
NOW,
);
const at300 = solveUncertaintyRadiusM(
mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25, _updatedAt: NOW - 300_000 }),
NOW,
);
- expect(at300).toBe(at60);
+ expect(at300).toBe(atCap);
});
it("caps the radius at 10 km", () => {
diff --git a/frontend/src/components/map/uncertainty.ts b/frontend/src/components/map/uncertainty.ts
index 530e9bda..55383b1f 100644
--- a/frontend/src/components/map/uncertainty.ts
+++ b/frontend/src/components/map/uncertainty.ts
@@ -22,9 +22,16 @@ import type { Aircraft } from "../../types";
/** Rayleigh 95% radius factor (k_50 = 1.177 CEP, k_68 = 1.510). */
export const UNCERTAINTY_K95 = 2.4477;
-/** Dead-reckoning growth is capped here: past 60 s the icon itself is stale
- * and a disc that kept growing would just be a claim about nothing. */
-export const UNCERTAINTY_DR_CAP_S = 60;
+/** Dead-reckoning growth is capped here: past this the icon itself is stale
+ * and a disc that kept growing would just be a claim about nothing.
+ *
+ * 30 s, down from 60 s, to match the backend's dark-lane entry expiry
+ * (`MN_DARK_EXPIRY_S`): a dark entry no longer survives to 60 s at all, so
+ * the second half of the old growth curve described entries that cannot
+ * exist. Assisted entries do live to 60 s, but they are anchored to a
+ * transponder fix rather than extrapolated, so growing their disc past the
+ * point the dark budget stops is not the honest reading either. */
+export const UNCERTAINTY_DR_CAP_S = 30;
/** Hard ceiling on the drawn radius. A degenerate solve (near-parallel
* baselines) can report an astronomically large formal sigma; without a cap
@@ -58,7 +65,7 @@ export function solveAgeS(ac: UncertaintyEntry | null | undefined, nowMs: number
/**
* Per-axis position sigma in metres at age `ageS`:
- * `sqrt(pos_sigma_m² + (pos_sigma_vel_ms · min(age, 60))²)`.
+ * `sqrt(pos_sigma_m² + (pos_sigma_vel_ms · min(age, UNCERTAINTY_DR_CAP_S))²)`.
*
* Returns null when the entry carries no usable `pos_sigma_m` — an older
* backend, or a solve whose node count was unknown. A missing or non-finite
diff --git a/libs/retina-analytics b/libs/retina-analytics
index 14504176..c58b662d 160000
--- a/libs/retina-analytics
+++ b/libs/retina-analytics
@@ -1 +1 @@
-Subproject commit 145041767422723d89d98ec003f843347ddbb880
+Subproject commit c58b662d764c8ef51cd003f98b21c3b69640d346
diff --git a/libs/retina-geolocator b/libs/retina-geolocator
index 2da39822..6979943a 160000
--- a/libs/retina-geolocator
+++ b/libs/retina-geolocator
@@ -1 +1 @@
-Subproject commit 2da3982220374f05dd622633576108e57bd9e34f
+Subproject commit 6979943aa7ae3d05e61a9cb355490eafcc8b45b3