From 664462819fe1d381edc4b101af2c334ad8ae8e03 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 06:25:08 +0000 Subject: [PATCH 01/25] Gate overlap zones on node world, and report the skipped pairs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps libs/retina-analytics to the branch commit that stops register_node and rebuild_zones_for building an overlap zone between two nodes in different worlds (offworldlabs/retina-analytics#25). This fleet is the reason: 50 synthetic nodes and 8 receivers share one footprint over Greenville, so the associator held a zone for all 400 sim/real pairs — 39 of them a live adjacency edge — and a pairing drawn from one of those grids can only ever match a simulated tracklet against a real echo. Real node ids showed up in 48 of 156 synthetic dark solver records over a 7-minute window. node_world is already the single authority for the question and is already injected as node_world_provider, so nothing new is wired here. /api/radar/association/status gains assoc_world_skipped_pairs next to overlap_zones: without it a fleet whose cross-world pairs are being refused reads exactly like a fleet whose pairs never overlapped. Co-Authored-By: Claude Fable 5.1 --- backend/routes/analytics.py | 8 +++++++ backend/tests/test_adsb_seed_backend.py | 29 +++++++++++++++++++++++++ backend/tests/test_analytics_routes.py | 12 ++++++++++ libs/retina-analytics | 2 +- 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/backend/routes/analytics.py b/backend/routes/analytics.py index 07d83a9c..23225ccf 100644 --- a/backend/routes/analytics.py +++ b/backend/routes/analytics.py @@ -125,6 +125,14 @@ async def association_status(): return { "registered_nodes": len(_a.node_geometries), "overlap_zones": len(_a.overlap_zones), + # Node pairs that got no grid because the two nodes are in different + # worlds (node_world above). Read next to overlap_zones: on a fleet + # of 50 synthetic nodes over the same city as 8 receivers it is the + # 400 sim/real pairs whose grids could only ever have paired a + # simulated echo with a real one. Counted per pair considered, so it + # keeps rising as nodes re-register — zero means the fleet is single- + # world (or untagged), not that the gate is off. + "assoc_world_skipped_pairs": getattr(_a, "assoc_world_skipped_pairs", 0), # Confirmed single-node tracks each node last submitted; these are what # pairings are drawn from. "pending_tracks": {nid: len(tracks) for nid, tracks in list(_a._pending_tracks.items())}, diff --git a/backend/tests/test_adsb_seed_backend.py b/backend/tests/test_adsb_seed_backend.py index cba3bb36..77b3fc86 100644 --- a/backend/tests/test_adsb_seed_backend.py +++ b/backend/tests/test_adsb_seed_backend.py @@ -605,3 +605,32 @@ def test_associator_gets_the_state_world_resolver(self): must consult the same resolver claiming and the auto-tag filter use, or one consumer accepts what another rejects.""" assert state.node_associator.node_world_provider is state.node_world + + def test_a_sim_and_a_real_node_over_one_footprint_get_no_overlap_zone(self): + """The same resolver, one level down: bottom-up pairing must not build + a grid across worlds either. Registering a synthetic node and a + hardware node on overlapping coverage used to leave a zone whose only + possible pairing was a simulated echo against a real one — which is how + real node ids reached the synthetic fleet's dark solves.""" + _a = state.node_associator + try: + _a.register_node("synth-GVL-9001", dict(_NODE_CFG)) + _a.register_node("hw-9001", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36)) + + assert _a.overlap_zones == {} + assert _a._neighbors.get("synth-GVL-9001", set()) == set() + assert _a.assoc_world_skipped_pairs == 1 + finally: + state._reset_for_tests() + + def test_two_synthetic_nodes_over_one_footprint_still_pair(self): + """The gate is the world difference, not the registration.""" + _a = state.node_associator + try: + _a.register_node("synth-GVL-9001", dict(_NODE_CFG)) + _a.register_node("synth-GVL-9002", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36)) + + assert _a.overlap_zones + assert _a.assoc_world_skipped_pairs == 0 + finally: + state._reset_for_tests() diff --git a/backend/tests/test_analytics_routes.py b/backend/tests/test_analytics_routes.py index 22a821e0..73c691d4 100644 --- a/backend/tests/test_analytics_routes.py +++ b/backend/tests/test_analytics_routes.py @@ -129,6 +129,18 @@ def test_status_returns_expected_fields(self, client): assert "overlap_zones" in body assert "overlaps" in body + def test_status_reports_world_skipped_pairs(self, client): + """The world gate on overlap zones is otherwise invisible: a fleet + whose sim/real pairs are being refused looks exactly like a fleet whose + pairs never overlapped, and only this counter separates them.""" + _a = state.node_associator + _a.assoc_world_skipped_pairs += 7 + try: + body = client.get("/api/radar/association/status").json() + assert body["assoc_world_skipped_pairs"] == 7 + finally: + state._reset_for_tests() + def test_status_includes_claiming_block(self, client): """Top-down claiming (ASSOC_CLAIM_MODE) since boot — off by default in tests, so this pins the shape rather than any particular mode.""" diff --git a/libs/retina-analytics b/libs/retina-analytics index 14504176..4439c2c7 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 145041767422723d89d98ec003f843347ddbb880 +Subproject commit 4439c2c7d2d4fb49ca6646000f3bedc25f9701c4 From b488400597455bfb8cbe8db2459e090ba447d2f7 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 06:37:45 +0000 Subject: [PATCH 02/25] Measure candidate contamination in the association bench The ghost rate cannot see the failure the cluster-merge rework targets. A solver input that merges two aircraft usually still solves within MATCH_KM of one of them, so it counts as *matched* while carrying 4-5 km of position error and a node that never saw the aircraft it was published as. Nothing in the bench reported that, so the fix had no acceptance criterion. --mode track now scores every solver input association emits, ahead of the solve and ahead of every gate below it, against a truth side-channel: which aircraft actually produced each detection. That channel cannot come from the frame's own ADS-B list -- the simulator appends None there for every aircraft with has_adsb False, and dark aircraft are exactly the population under study -- so each detection is matched back to the aircraft whose noiseless (delay, doppler) it is nearest, at a gate ~5 sigma wider than the simulator's own measurement noise. It is built before _strip_adsb and never reaches association, so the blind discipline is intact. Reports contaminated_inputs_pct and foreign_nodes_per_input, per seed and pooled, alongside the ghost rate and matched error already there. Also adds the cluster-merge knobs (--merge-dist-km, --pair-vel-exclusive, --merge-vel-consistent and the two velocity thresholds) so each sub-step of the rework can be swept on its own; each defaults to None, meaning "leave the library's default alone", so a plain run measures what the library currently ships rather than freezing today's values into the bench. Pins retina-analytics at fix/cluster-contamination. Co-Authored-By: Claude Fable 5.1 --- backend/scripts/association_bench.py | 232 +++++++++++++++++++++++++++ libs/retina-analytics | 2 +- 2 files changed, 233 insertions(+), 1 deletion(-) diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py index e08999bb..14dc6cdc 100644 --- a/backend/scripts/association_bench.py +++ b/backend/scripts/association_bench.py @@ -69,6 +69,7 @@ # superseded detection path that --mode detection measures as the baseline, so # constructing it unconditionally leaves --mode track unaffected. import retina_analytics.association as _assoc_module # noqa: E402 +from retina_analytics.association import predict_observation # noqa: E402 from retina_analytics.detection_association import DetectionAssociator # noqa: E402 from retina_analytics.manager import NodeAnalyticsManager # noqa: E402 from retina_geolocator.consensus import solve_consensus # noqa: E402 @@ -248,6 +249,121 @@ def _frame_to_detections(frame: dict) -> list[dict]: return dets +# ── Contamination scoring (truth side-channel) ─────────────────────────── +# How far a detection may sit from an aircraft's noiseless (delay, doppler) +# and still be attributed to it. The simulator's own measurement noise is +# gauss(0, 0.1-0.2 us) in delay and gauss(0, 2-4 Hz) in Doppler +# (world.generate_detections_for_node), so these are ~5 sigma: wide enough +# that a real echo is never mistaken for clutter, tight enough that clutter +# — uniform over tens of us — almost never lands on an aircraft. +_TRUTH_DELAY_GATE_US = 1.0 +_TRUTH_DOPPLER_GATE_HZ = 25.0 +# Sentinel for "this measurement matched two different aircraft equally +# well". Neither foreign nor own — excluded from the numerator so an +# ambiguity in the scorer is never reported as a contamination. +_TRUTH_AMBIGUOUS = "?ambiguous" + + +def _index_detection_truth(det_truth: dict, geo, node_id: str, frame: dict, aircraft: list) -> None: + """Record which aircraft produced each detection in one node's frame. + + Truth side-channel, for the CONTAMINATION metric only: it is built from + the frame BEFORE _strip_adsb and never reaches association, so the blind + discipline is intact. It cannot be read off the frame's own ``adsb`` + list either — the simulator appends None there for every aircraft with + has_adsb False, and dark aircraft are exactly the population this metric + exists to score. So each detection is instead matched back to the + aircraft whose noiseless observation it is nearest. + + Keyed on the (delay, doppler) floats themselves because that is the only + handle the metric gets downstream: a solver input's measurement carries + its track's latest delay/doppler verbatim (history[-1] -> the detection + dict -> here), and the simulator rounds both to 2 dp, so the equality is + exact rather than approximate. + """ + delays = frame.get("delay") or [] + if not delays: + return + dopplers = frame.get("doppler") or [] + preds = [] + for ac in aircraft: + d_us, f_hz = predict_observation( + geo, + ac.lat, + ac.lon, + ac.alt_km, + ac.vel_east * 1000.0, + ac.vel_north * 1000.0, + ac.vel_up * 1000.0, + ) + preds.append((d_us, f_hz, ac.object_id)) + for d, f in zip(delays, dopplers): + best = best2 = None + for d_us, f_hz, oid in preds: + dd, df = abs(d - d_us), abs(f - f_hz) + if dd > _TRUTH_DELAY_GATE_US or df > _TRUTH_DOPPLER_GATE_HZ: + continue + # Normalised so the two axes are comparable at their own gates. + cost = (dd / _TRUTH_DELAY_GATE_US) ** 2 + (df / _TRUTH_DOPPLER_GATE_HZ) ** 2 + if best is None or cost < best[0]: + best, best2 = (cost, oid), best + elif best2 is None or cost < best2[0]: + best2 = (cost, oid) + if best is None: + continue # clutter: left absent, which the scorer reads as foreign + oid = best[1] + if best2 is not None and best2[0] < 4.0 * best[0]: + oid = _TRUTH_AMBIGUOUS + key = (node_id, float(d), float(f)) + prev = det_truth.get(key) + # The same (node, delay, doppler) recurring for a different aircraft + # later in the run would silently relabel an earlier measurement, so + # a collision demotes the key rather than overwriting it. + det_truth[key] = oid if (prev is None or prev == oid) else _TRUTH_AMBIGUOUS + + +def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) -> None: + """Count the nodes in one solver input that are not looking at its aircraft. + + The input's own aircraft is the plurality of its measurements' true + aircraft — the honest reading of "what is this candidate mostly about", + and the one that does not assume the (possibly contaminated) initial + guess is anywhere near a target. Ties are broken by the nearest ground + truth to the initial guess, which is the criterion the ghost/matched + split already uses. + + A node is foreign when its measurement belongs to a different aircraft, + or to no aircraft at all (clutter that survived the tracker's M-of-N and + the delay grid). Ambiguous attributions are counted in neither. + """ + oids = [ + det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"])), None) + for m in s_in.get("measurements") or [] + ] + if not oids: + return + counts = Counter(o for o in oids if o is not None and o != _TRUTH_AMBIGUOUS) + if not counts: + return + top_n = max(counts.values()) + contenders = sorted(o for o, c in counts.items() if c == top_n) + if len(contenders) > 1: + guess = s_in.get("initial_guess") or {} + contenders.sort( + key=lambda o: min( + (_haversine_km(guess.get("lat", 0.0), guess.get("lon", 0.0), a, b) for a, b, oid, _ in truth if oid == o), + default=float("inf"), + ) + ) + own = contenders[0] + foreign = sum(1 for o in oids if o != own and o != _TRUTH_AMBIGUOUS) + res.inputs_scored += 1 + res.input_nodes += len(oids) + res.foreign_nodes += foreign + if foreign: + res.inputs_contaminated += 1 + + def _strip_adsb(frame: dict) -> dict: """Return the frame as a real receiver would see it. @@ -482,6 +598,9 @@ class Result: gate_accepted: int = 0 gate_unfitted: int = 0 gate_superseded: int = 0 + # Position clusters that held two different tracks of one node and were + # split into one solver input each, straight off the associator. + cluster_splits: int = 0 # Deferred mode only: what the *solver-side* n=2 gate did. In production the # associator emits unscored pairings and this gate is the one that runs, so # without these the shipped configuration's selection is invisible. @@ -531,6 +650,20 @@ class Result: keys_real: int = 0 keys_ghost: int = 0 + # ── Candidate contamination (--mode track) ──────────────────────────── + # Scored on every solver input association emits, BEFORE the solve and + # before every downstream gate: the question is what association handed + # the solver, not what survived it. A contaminated input is one whose + # measurements do not all belong to the same aircraft — the failure the + # cluster-merge rework targets, and the one the ghost rate cannot see + # (a two-aircraft merge usually still solves within MATCH_KM of one of + # them, so it counts as matched while carrying 4-5 km of position + # error). See _score_contamination. + inputs_scored: int = 0 + inputs_contaminated: int = 0 + foreign_nodes: int = 0 + input_nodes: int = 0 + # Stone-Soup GOSPA/SIAP scalars for this one run (--ss-metrics), or None # when it was off, stonesoup wasn't available, or the recorder had # nothing to score (see stonesoup_metrics.MetricRecorder.compute). Not @@ -634,6 +767,11 @@ def keys_per_object(self): "distinct_keys", "keys_real", "keys_ghost", + "inputs_scored", + "inputs_contaminated", + "foreign_nodes", + "input_nodes", + "cluster_splits", ) _EXTEND_FIELDS = ( "errors_km", @@ -685,6 +823,14 @@ def merge(self, other: Result, tag: str) -> None: def ghost_pct(self): return 100.0 * self.ghosts / self.total if self.total else 0.0 + @property + def contaminated_inputs_pct(self): + return 100.0 * self.inputs_contaminated / self.inputs_scored if self.inputs_scored else 0.0 + + @property + def foreign_nodes_per_input(self): + return self.foreign_nodes / self.inputs_scored if self.inputs_scored else 0.0 + def build_scene( seed: int, @@ -789,6 +935,7 @@ def run( ss_metric_dt=5.0, ss_hold_s=12.0, smoother_legs=None, + cluster_opts=None, ) -> Result: import random @@ -861,12 +1008,17 @@ def fov_provider(node_id): # emits unscored pairings and the solver worker fits and arbitrates. The # two are different code paths, so they need separate baselines. deferred = mode == "track" and cv_fit_mode == "deferred" + # The cluster-merge knobs are passed only when the caller overrode them, + # so a plain run measures whatever the library currently ships rather than + # freezing today's defaults into the bench. + _cluster_kwargs = {k: v for k, v in (cluster_opts or {}).items() if v is not None} assoc = DetectionAssociator( grid_step_km=3.0, cv_fit=(fit_constant_velocity if (mode == "track" and not deferred) else None), cv_chi2_max=chi2_max, cv_min_span_s=min_span_s, cv_exclusive=exclusive, + **_cluster_kwargs, ) n2_gate = DeferredN2Gate(chi2_max, claim_ttl_s=claim_ttl_s, claim_policy=claim_policy) if deferred else None # One tracker per node, driven by every frame — mirrors @@ -915,6 +1067,10 @@ def fov_provider(node_id): _BENCH_MN_MAX_AGE_MS = 60_000 res = Result() + # (node_id, delay_us, doppler_hz) -> the aircraft that produced that + # detection. Truth side-channel for the contamination metric only, built + # from the un-stripped frame below — see _index_detection_truth. + det_truth: dict = {} _all_keys_seen: set = set() _keys_real: set = set() _keys_ghost: set = set() @@ -960,6 +1116,10 @@ def _geo_key(nid): for nid in due_nodes: next_send[nid] += frame_interval frame = world.generate_detections_for_node(nid, ts_ms) + if mode == "track": + # Before _strip_adsb, and never fed to association: the + # contamination metric's truth channel. + _index_detection_truth(det_truth, assoc.node_geometries[nid], nid, frame, world.aircraft) if fov_analytics is not None: # The truth channel, not the (possibly blind) association # stream below -- a real node's ADS-B calibration reaches @@ -1001,6 +1161,11 @@ def _geo_key(nid): res.cluster_sizes[(_k, len(s_in.get("track_ids") or []))] += 1 if s_in.get("n_nodes", 0) < 2: continue + if mode == "track": + # Scored here, ahead of the solve and every gate below: + # this measures what association emitted, which is the + # thing the cluster-merge rework changes. + _score_contamination(res, s_in, det_truth, truth) try: _t0 = time.perf_counter() out = solve_fn(s_in, node_cfgs) @@ -1153,6 +1318,7 @@ def _geo_key(nid): res.gate_accepted = assoc.track_pairs_accepted res.gate_unfitted = assoc.track_pairs_unfitted res.gate_superseded = assoc.track_pairs_superseded + res.cluster_splits = getattr(assoc, "cluster_splits", 0) res.claims_matched = assoc.claims_matched res.claim_conflicts = assoc.claim_conflicts res.anchored_inputs = assoc.anchored_inputs_emitted @@ -1329,6 +1495,15 @@ def report(label: str, r: Result, truth_max_kt: float | None = None): f" solves faster than any real aircraft ({truth_max_kt:.0f} kt): " f"{over} ({100 * over / len(r.speeds_kt):.0f}%)" ) + if r.inputs_scored: + print( + f" CONTAMINATION: {r.inputs_contaminated}/{r.inputs_scored} solver inputs carry a foreign node" + f" -> {r.contaminated_inputs_pct:5.1f}% " + f"foreign nodes/input {r.foreign_nodes_per_input:.2f}" + f" ({r.foreign_nodes}/{r.input_nodes} nodes)" + ) + if r.cluster_splits: + print(f" cluster splits (same-node track conflict): {r.cluster_splits}") if r.gate_gated: print( f" CV gate: {r.gate_gated} pairings past the delay grid " @@ -1498,6 +1673,33 @@ def main(): help="track mode: disable one-to-one hypothesis selection " "(each pairing then answers only to the chi2 threshold)", ) + # Cluster-merge knobs (track mode). Each defaults to None, meaning "leave + # the library's own default alone", so the bench does not silently pin a + # value the library later changes — and so a sweep leg reads as exactly + # the deviation it is testing. + p.add_argument( + "--merge-dist-km", + type=float, + default=None, + help="track mode: how close two pairings must be to merge into one " + "solver input (association._MERGE_DIST_KM)", + ) + p.add_argument( + "--pair-vel-exclusive", + choices=("on", "off"), + default=None, + help="track mode, deferred only: drop a pairing whose implied velocity " + "contradicts a better-scoring pairing that claims the same track", + ) + p.add_argument( + "--merge-vel-consistent", + choices=("on", "off"), + default=None, + help="track mode: require implied-velocity agreement, not just " + "proximity, before two pairings are merged into one cluster", + ) + p.add_argument("--pair-vel-dv-ms", type=float, default=None, help="velocity-conflict speed threshold (m/s)") + p.add_argument("--pair-vel-dtheta-deg", type=float, default=None, help="velocity-conflict heading threshold (deg)") p.add_argument("--min-aircraft", type=int, default=10, help="matches FLEET_AIRCRAFT lower bound") p.add_argument("--max-aircraft", type=int, default=20) p.add_argument("--metro-traffic-frac", type=float, default=0.85, help="matches FLEET_METRO_TRAFFIC_FRAC") @@ -1568,6 +1770,14 @@ def main(): ) args = p.parse_args() + cluster_opts = { + "merge_dist_km": args.merge_dist_km, + "pair_vel_exclusive": None if args.pair_vel_exclusive is None else args.pair_vel_exclusive == "on", + "merge_vel_consistent": None if args.merge_vel_consistent is None else args.merge_vel_consistent == "on", + "pair_vel_dv_ms": args.pair_vel_dv_ms, + "pair_vel_dtheta_deg": args.pair_vel_dtheta_deg, + } + # --ss-metrics auto/on/off resolution. "on" without stonesoup installed # is a hard error (the user explicitly asked for numbers this image # cannot produce); "auto" degrades quietly except for one notice line so @@ -1622,6 +1832,7 @@ def main(): + f", fov={args.fov}" + f", ss-metrics={'on' if ss_metrics_enabled else 'off'}" + (f", smoother-legs={','.join(lbl for lbl, _, _ in smoother_legs)}" if smoother_legs else "") + + "".join(f", {k.replace('_', '-')}={v}" for k, v in sorted(cluster_opts.items()) if v is not None) ) # chi2 only means anything in track mode; keep one pass otherwise. @@ -1632,6 +1843,7 @@ def main(): solve_fn = _ESTIMATORS[estimator_name] rates, solve_rates, reals, fakes, speed_errs = [], [], [], [], [] n2_rates = [] + contam_rates, foreign_rates, med_errs = [], [], [] agg = Result() last = None for k in range(args.repeat): @@ -1666,6 +1878,7 @@ def main(): ss_metric_dt=args.ss_metric_dt, ss_hold_s=args.ss_hold_s, smoother_legs=smoother_legs, + cluster_opts=cluster_opts, ) agg.merge(last, tag=f"s{args.seed + k}") # Track-level is the comparable metric — solve-level and @@ -1678,6 +1891,9 @@ def main(): fakes.append(len(last.ghost_tracks)) if last.speed_err_ms: speed_errs.append(statistics.median(last.speed_err_ms)) + contam_rates.append(last.contaminated_inputs_pct) + foreign_rates.append(last.foreign_nodes_per_input) + med_errs.append(statistics.median(last.errors_km) if last.errors_km else float("nan")) label = f"assoc_interval={interval:g}s" if chi2_max is not None: label += f" chi2/dof<={chi2_max:g}" @@ -1700,6 +1916,22 @@ def main(): f"({', '.join(f'{x:.0f}%' for x in n2_rates)})" ) print(f" by solve: {', '.join(f'{x:.1f}%' for x in solve_rates)}") + if any(contam_rates): + print( + f" contaminated inputs per seed: " + f"{', '.join(f'{x:.0f}%' for x in contam_rates)}" + f" mean {statistics.mean(contam_rates):.1f}%" + ) + print( + f" foreign nodes/input per seed: " + f"{', '.join(f'{x:.2f}' for x in foreign_rates)}" + f" mean {statistics.mean(foreign_rates):.2f}" + ) + print( + f" median matched error per seed: " + f"{', '.join(f'{x:.2f}' for x in med_errs)} km" + f" real tracks {', '.join(str(x) for x in reals)}" + ) if speed_errs: print( f" median speed error per seed: " diff --git a/libs/retina-analytics b/libs/retina-analytics index 14504176..4ff48fca 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 145041767422723d89d98ec003f843347ddbb880 +Subproject commit 4ff48fcaa3abdaad1d42eb576f8573342addad4e From a1395caa259187123d41657ab9c0b696d2d171a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 06:37:04 +0000 Subject: [PATCH 03/25] solver: SOLVER_ALT_MODE=free, and ship only the configs a candidate uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent changes to the same enqueue-and-solve path. 1. SOLVER_ALT_MODE (core/state.py, default "sweep" — nothing changes until it is set). The n>=3 altitude sweep solves once per fixed layer and keeps the lowest rms_delay. Its layers are 2 km apart, so the altitude it pins is systematically up to 1 km wrong, and on noise-free replay of this fleet's geometry that quantisation ALONE left rms_delay at a 1.76 us median against the 3.0 us reject gate — most of the gate's budget spent on an altitude the measurements themselves determine, and the residual left over then blamed on nodes, so _trim_and_resolve drops measurements that were never the problem. In "free" mode _solve_best_altitude makes ONE pool call to the geolocator's solve_multinode_multistart with three start layers (the one nearest the association guess and its two neighbours), altitude solved as a sixth unknown — against the sweep's six round trips, each of which pickles the node configs for a child. n=2 is untouched: altitude is unobservable there, so _solve_best_altitude_n2 keeps its single pinned solve and the geolocator pins regardless of the flag. Trimming re-solves through _solve_best_altitude, so a trim round inherits the mode its first solve used — mixing a swept altitude with a free one would make the rms it compares between rounds a different quantity. Both modes stamp altitude_mode on every history record, published or rejected, and free adds the per-start residuals and z_saturated. That is the whole comparison channel: one deploy of each, read off /api/test/mlat-history. There is deliberately no shadow mode — the two produce the same shape of result, so running both would double the solver's cost to learn what a deploy of each already says. 2. The frame path shipped ALL 58 node configs with every solver input. The pool is a spawn pool, so that whole set is pickled per solve while a candidate carries 2-8 measurements. configs_for_solver_input restricts it to the measurement node ids; nothing downstream needs the rest — the solver builds NodeSetups from the measurements, trimming and consensus only ever narrow that set, the beam gate iterates contributing_node_ids, and cv_epochs is built from the same matched nodes in all three input shapes association emits. The known lane fetches its own configs and is unaffected. Co-Authored-By: Claude Fable 5.1 --- backend/.env.example | 10 + backend/core/state.py | 20 ++ backend/services/frame_processor.py | 23 ++- backend/services/tasks/solver.py | 118 +++++++++++- backend/tests/test_solver_alt_mode.py | 265 ++++++++++++++++++++++++++ docs/solverflow.md | 43 ++++- libs/retina-geolocator | 2 +- 7 files changed, 470 insertions(+), 11 deletions(-) create mode 100644 backend/tests/test_solver_alt_mode.py diff --git a/backend/.env.example b/backend/.env.example index 20a107b4..488ecbdc 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -129,9 +129,19 @@ MENDER_PAT= # from its association guess before it is rejected; the default is 6 km = 2 x # the 3 km association grid step, since a dark guess is a quantised grid point # rather than an ADS-B fix. ADS-B-anchored solves keep the fixed 2 km cap. +# +# SOLVER_ALT_MODE is how an n>=3 solve gets its altitude. sweep (default) +# solves once per fixed altitude layer and keeps the lowest rms_delay — six +# process-pool round trips per candidate, and an altitude quantised to a +# ladder whose 2 km spacing puts up to 1 km of error straight into the +# residual the reject gate reads. free makes one pool call to the geolocator's +# multi-start helper, which solves altitude as a sixth unknown. Both modes +# stamp altitude_mode on the solve-history record, so /api/test/mlat-history +# can compare them across a deploy of each. See docs/solverflow.md. # SOLVER_WORKERS=2 # SOLVER_RESOLVE_INTERVAL_S=12 # SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0 +# SOLVER_ALT_MODE=sweep # Detection mirror. Production only. Every accepted v1 detection frame is # forwarded to another environment's /api/radar/detections/bulk, batched once a diff --git a/backend/core/state.py b/backend/core/state.py index 634aa212..f1239459 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -74,6 +74,26 @@ if KNOWN_LANE_MODE not in ("off", "shadow", "binding"): KNOWN_LANE_MODE = "shadow" +# How the n>=3 solve gets its altitude (see services/tasks/solver.py's +# _solve_best_altitude). sweep/free, read here rather than in that module so +# it sits with its sibling mode flags and a test can monkeypatch it without +# reimporting the solver. +# sweep (default) — solve once per fixed altitude layer, keep the lowest +# rms_delay. Six pool round trips per candidate, and an altitude +# quantised to the ladder: layers are 2 km apart, so the pin is +# systematically up to 1 km wrong and that error lands in the +# residual the reject gate reads. +# free — one pool call to retina_geolocator's multi-start helper, which +# solves altitude as a sixth unknown from three start layers. +# Not off/shadow/active: there is no shadow here, because the two modes +# produce the same shape of result and the history record carries +# altitude_mode either way — running both would double the solver's cost to +# learn what one deploy of each already says. An unrecognised value falls +# back to "sweep", the same degrade-to-inert rule the sibling flags use. +SOLVER_ALT_MODE = os.getenv("SOLVER_ALT_MODE", "sweep").lower() +if SOLVER_ALT_MODE not in ("sweep", "free"): + SOLVER_ALT_MODE = "sweep" + node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE) diff --git a/backend/services/frame_processor.py b/backend/services/frame_processor.py index 0fb2bbd5..fe7b086d 100644 --- a/backend/services/frame_processor.py +++ b/backend/services/frame_processor.py @@ -156,6 +156,27 @@ def get_node_configs() -> dict[str, dict]: return configs +def configs_for_solver_input(node_cfgs: dict[str, dict], s_in: dict) -> dict[str, dict]: + """The subset of ``node_cfgs`` a solver input can actually reach. + + The solver runs in a *spawn* process pool, so everything queued with an + input is pickled and shipped to a child on every call — and the fleet is + 58 nodes while a candidate carries 2-8 measurements. Sending the whole + set meant ~50 configs per solve that no code path could look at. + + Nothing downstream needs the rest. The solver builds NodeSetups from the + measurements only; trimming and consensus both narrow that set further + (_filter_s_in_to_nodes) and never widen it; the beam gate iterates the + result's contributing_node_ids, which are measurement node ids by + construction; and cv_epochs is built from the same matched nodes as the + measurements in all three input shapes association emits. The known lane + is unaffected — it fetches its own configs (known_lane.run_known_lane_pass) + rather than reusing what was queued here. + """ + wanted = {m.get("node_id") for m in (s_in.get("measurements") or ())} + return {nid: cfg for nid, cfg in node_cfgs.items() if nid in wanted} + + # ── Per-node pipeline factory ───────────────────────────────────────────────── @@ -427,7 +448,7 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP if s_in["n_nodes"] < 2: continue try: - state.solver_queue.put_nowait((s_in, node_cfgs, time.time())) + state.solver_queue.put_nowait((s_in, configs_for_solver_input(node_cfgs, s_in), time.time())) except Exception: state.bump_counter("solver_queue_drops") if state.solver_queue_drops % 100 == 1: diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 7c6b947c..41bb53ec 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -100,6 +100,24 @@ def _pool_call(fn, *args): return fn(*args) +def _pool_solve_multistart(s_in, node_cfgs, alt_starts_km): + """solve_multinode_multistart via the process pool (inline when none). + + Defined here rather than beside _pool_solve_multinode at the foot of this + module for the reason _pool_select_consensus is: it is a default argument + value, resolved when the ``def`` executes, so it has to be bound before + _solve_best_altitude's signature is reached. + + A module-level function taking only picklable arguments, because the pool + is a *spawn* pool — a child imports retina_geolocator and nothing of the + backend, so what crosses is this function's qualified name plus the input + dicts. + """ + from retina_geolocator.multinode_solver import solve_multinode_multistart + + return _pool_call(solve_multinode_multistart, s_in, node_cfgs, alt_starts_km, True) + + # Altitude layers (km) tried when n_nodes ≥ 3. For an overdetermined system # (3+ delay equations, 2 unknowns after altitude pinning) only the correct # altitude layer yields rms_delay ≈ 0; wrong layers give rms > 0, so picking @@ -320,17 +338,72 @@ def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[floa return best_result -def _solve_best_altitude(s_in: dict, node_cfgs: dict, solve_fn) -> dict | None: - """Altitude sweep for n≥3: pick by minimum rms_delay. +# How many start altitudes the free mode hands the multi-start helper: the +# layer nearest the guess and its two neighbours. Freeing z removes the +# ladder's quantisation but not the LM's locality, so the starts are still +# what stops a solve settling on the wrong side of a bistatic ellipse — but +# three of them, in one pool call, rather than the sweep's six. +_FREE_ALT_N_STARTS = 3 +# Fewest measurements the free mode is used at. Below this altitude is not +# observable and retina_geolocator pins it anyway; the sweep is left in place +# so the n=2 path keeps its documented behaviour exactly. +_FREE_ALT_MIN_NODES = 3 + + +def _free_alt_starts(ig_alt_km, layers: list[float]) -> list[float]: + """The layer nearest ``ig_alt_km`` and its neighbours — _FREE_ALT_N_STARTS + of them, clamped to the ends of the ladder so the count never shrinks + there (the top and bottom layers are where a wrong start is least + recoverable, not most). + """ + if not layers: + return [] + alt = float(ig_alt_km) if ig_alt_km is not None else 7.0 + nearest = min(range(len(layers)), key=lambda i: abs(layers[i] - alt)) + lo = max(0, min(nearest - 1, len(layers) - _FREE_ALT_N_STARTS)) + return layers[lo : lo + _FREE_ALT_N_STARTS] + - If the initial_guess already carries an ADS-B altitude (not one of the fixed - grid layers), include it in the sweep so the correct exact altitude is tried. +def _solve_best_altitude( + s_in: dict, + node_cfgs: dict, + solve_fn, + multistart_fn=_pool_solve_multistart, +) -> dict | None: + """Altitude for n≥3, by whichever rule state.SOLVER_ALT_MODE names. + + sweep (default): solve once per layer, pick by minimum rms_delay. If the + initial_guess already carries an ADS-B altitude (not one of the fixed grid + layers), include it in the sweep so the correct exact altitude is tried. + + free: one call to the multi-start helper, which solves altitude as a sixth + unknown from _free_alt_starts. The sweep cannot do better than half its + 2 km layer spacing, and on noise-free replay of this fleet's geometry that + quantisation alone left rms_delay at a 1.76 µs median against the 3.0 µs + reject gate — spending most of the gate's budget on an altitude the + measurements themselves determine, and provoking _trim_and_resolve to drop + nodes that were never the problem. Costs one pool round trip per + candidate instead of six. + + The mode is read per call rather than captured at import, so a test (and a + live config reload) sees the value it set. Read here and not inside + _process_solver_item because _trim_and_resolve re-enters through this same + function: a trim must re-solve under the mode its first solve used, or the + residuals it is comparing are not the same quantity. """ ig_alt = s_in.get("initial_guess", {}).get("alt_km") if ig_alt is not None and ig_alt not in _SOLVER_ALT_LAYERS_KM: layers = sorted(set(_SOLVER_ALT_LAYERS_KM + [round(float(ig_alt), 3)])) else: layers = _SOLVER_ALT_LAYERS_KM + n_meas = len({m.get("node_id") for m in (s_in.get("measurements") or [])}) + if state.SOLVER_ALT_MODE == "free" and n_meas >= _FREE_ALT_MIN_NODES: + # No fall back to the sweep when this returns None: a helper that got + # no solve out of three starts is reporting the same thing the sweep + # reports when every layer fails, and sweeping anyway would cost the + # six round trips this mode exists to avoid on exactly the candidates + # that are least likely to repay them. + return multistart_fn(s_in, node_cfgs, _free_alt_starts(ig_alt, layers)) return _sweep_altitudes(s_in, node_cfgs, solve_fn, layers, "rms_delay") @@ -386,6 +459,7 @@ def _trim_and_resolve( node_cfgs: dict, solve_fn, result: dict, + multistart_fn=_pool_solve_multistart, ) -> tuple[dict, dict, dict | None]: """Drop the worst-residual node(s) and re-solve, down to _TRIM_MIN_NODES. @@ -395,6 +469,11 @@ def _trim_and_resolve( so re-solving on the survivors after dropping the offending node recovers a solve the blanket gate would otherwise discard outright. + Re-solves through _solve_best_altitude, so it inherits whichever altitude + mode is in force — the loop compares this round's rms against the previous + round's, and mixing a swept altitude with a free one would make that + comparison meaningless. + Returns (final_result, final_s_in, trim_meta). trim_meta is None only when no round ever produced a successful re-solve — i.e. no trimming was actually performed — never when trimming ran but rms stayed high (that @@ -439,7 +518,7 @@ def _trim_and_resolve( s_next = _filter_s_in_to_nodes(s_in, survivors) try: - new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn) + new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn, multistart_fn) except Exception: logging.exception("Solver trim re-solve failed") break @@ -1693,7 +1772,12 @@ def fov_gate_verdict(fov, n_nodes: int, brg: float, dist_km: float, range_rule_p return range_rule_pass or fov_pass -def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus) -> dict | None: +def _process_solver_item( + item: tuple, + solve_fn, + select_fn=_pool_select_consensus, + multistart_fn=_pool_solve_multistart, +) -> dict | None: """Process a single solver queue entry. Returns the solver result (or None). Extracted from the worker loop so the success/failure/latency bookkeeping @@ -1704,6 +1788,10 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus initial_guess and _CONSENSUS_MODE != "off" — n=2 (mirror-disambiguation is the displacement/beam gates' job, not consensus's) and detection-level inputs (no initial_guess to pin an altitude with) never call it. + + multistart_fn is the free-altitude solve (_pool_solve_multistart by + default; tests substitute a stub), reached only when + state.SOLVER_ALT_MODE is "free" — see _solve_best_altitude. """ s_in, node_cfgs = item[0], item[1] enqueued_at: float | None = item[2] if len(item) > 2 else None @@ -1736,7 +1824,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus if _CONSENSUS_MODE != "off": s_in, consensus_meta = _consensus_select(s_in, node_cfgs, select_fn) n_nodes = s_in.get("n_nodes", n_nodes) - result = _solve_best_altitude(s_in, node_cfgs, solve_fn) + result = _solve_best_altitude(s_in, node_cfgs, solve_fn, multistart_fn) else: result = _solve_best_altitude_n2(s_in, node_cfgs, solve_fn) except Exception: @@ -1762,7 +1850,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus and (result.get("rms_delay") or 0) > _SOLVER_RMS_DELAY_MAX_US and result.get("per_node_delay_res_us") ): - result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result) + result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result, multistart_fn) n_nodes = result.get("n_nodes", n_nodes) # Built once and threaded through every history record below @@ -1771,6 +1859,20 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus _extra: dict | None = dict(trim_meta) if trim_meta else {} if consensus_meta is not None: _extra["consensus_meta"] = consensus_meta + # How this solve got its altitude, and — in free mode — what each + # start altitude fitted to. Stamped on every record, published or + # rejected, and in BOTH modes (the sweep's solves report + # altitude_mode "pinned"), because the only way to judge SOLVER_ALT_MODE + # live is to compare the two lanes' rms_delay and gt_error_km over the + # same history buffer. The per-start list is what says whether the + # three starts were worth keeping or one would have done. + if result.get("altitude_mode"): + _extra["altitude_mode"] = result["altitude_mode"] + if result.get("rms_by_start") is not None: + _extra["alt_starts_km"] = result.get("alt_starts_km") + _extra["alt_start_rms_us"] = [None if v is None else round(float(v), 3) for v in result["rms_by_start"]] + if result.get("z_saturated"): + _extra["z_saturated"] = True _extra = _extra or None rms_delay = result.get("rms_delay", 0) or 0 diff --git a/backend/tests/test_solver_alt_mode.py b/backend/tests/test_solver_alt_mode.py new file mode 100644 index 00000000..8a374dbc --- /dev/null +++ b/backend/tests/test_solver_alt_mode.py @@ -0,0 +1,265 @@ +"""SOLVER_ALT_MODE: how the n>=3 solve gets its altitude. + +sweep (the default) calls the LM once per fixed altitude layer and keeps the +lowest rms_delay — six process-pool round trips, and an altitude quantised to +a ladder 2 km wide, which puts up to 1 km of error into the residual the +reject gate reads. free makes ONE call to the geolocator's multi-start +helper, which solves altitude as a sixth unknown from three start layers. + +These tests are about the routing, not the physics: the geolocator's own +suite (tests/test_free_altitude.py there) measures what the free solve +actually fits. What matters here is that the default is byte-identical to +the sweep, that free spends one call and not six, that trimming re-solves +under the same mode, and that both modes leave enough on the history record +to be compared live. +""" + +import time + +import pytest + +from core import state +from services import frame_processor +from services.tasks import solver as solver_mod + +LAT, LON = 35.0, -82.0 + + +def _s_in(node_ids, alt_km=9.0, **overrides): + s_in = { + "initial_guess": {"lat": LAT, "lon": LON, "alt_km": alt_km}, + "measurements": [{"node_id": nid, "delay_us": 10.0, "doppler_hz": 1.0, "snr": 15.0} for nid in node_ids], + "n_nodes": len(node_ids), + "timestamp_ms": int(time.time() * 1000), + } + s_in.update(overrides) + return s_in + + +def _stub_result(node_ids, rms_delay=0.5, **overrides): + result = { + "success": True, + "lat": LAT, + "lon": LON, + "alt_m": 9000.0, + "timestamp_ms": int(time.time() * 1000), + "vel_east": 0.0, + "vel_north": 0.0, + "rms_delay": rms_delay, + "rms_doppler": 5.0, + "n_nodes": len(node_ids), + "n_measurements": len(node_ids), + "contributing_node_ids": list(node_ids), + } + result.update(overrides) + return result + + +class _Recorder: + """A solve_fn / multistart_fn that records every call it is given.""" + + def __init__(self, result_for): + self.calls: list[tuple] = [] + self._result_for = result_for + + def __call__(self, s_in, node_cfgs, *rest): + self.calls.append((s_in, node_cfgs, rest)) + nodes = tuple(m["node_id"] for m in s_in["measurements"]) + return self._result_for(nodes, s_in, *rest) + + +class _AltModeBase: + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + +class TestFreeAltStarts: + """The three starts handed to the multi-start helper.""" + + @pytest.mark.parametrize( + "alt_km,expected", + [ + (9.0, [7.0, 9.0, 11.0]), + (7.0, [5.0, 7.0, 9.0]), + (8.2, [7.0, 9.0, 11.0]), + # Clamped at the ends: the ladder's first and last layers still get + # three starts, not one or two. + (1.5, [1.5, 3.0, 5.0]), + (0.4, [1.5, 3.0, 5.0]), + (11.0, [7.0, 9.0, 11.0]), + (40.0, [7.0, 9.0, 11.0]), + ], + ) + def test_window_around_the_nearest_layer(self, alt_km, expected): + starts = solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) + assert starts == expected + assert len(starts) == solver_mod._FREE_ALT_N_STARTS + + def test_an_adsb_altitude_in_the_ladder_is_a_start(self): + """_solve_best_altitude splices an ADS-B altitude into the layers, and + the window is taken over that spliced list — otherwise the one exact + altitude available would never be started from.""" + layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4])) + assert solver_mod._free_alt_starts(8.4, layers) == [7.0, 8.4, 9.0] + + def test_no_layers_gives_no_starts(self): + assert solver_mod._free_alt_starts(9.0, []) == [] + + +class TestSweepIsTheDefault(_AltModeBase): + def test_sweep_calls_the_lm_once_per_layer_and_never_the_multistart(self): + nodes = ["n1", "n2", "n3"] + solve = _Recorder(lambda n, s, *r: _stub_result(n)) + multistart = _Recorder(lambda n, s, *r: pytest.fail("multistart called in sweep mode")) + + result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) + + assert result is not None and result["success"] + assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM) + assert [c[0]["initial_guess"]["alt_km"] for c in solve.calls] == solver_mod._SOLVER_ALT_LAYERS_KM + assert multistart.calls == [] + assert state.SOLVER_ALT_MODE == "sweep" + + +class TestFreeMode(_AltModeBase): + def setup_method(self): + super().setup_method() + self._saved_mode = state.SOLVER_ALT_MODE + state.SOLVER_ALT_MODE = "free" + + def teardown_method(self): + state.SOLVER_ALT_MODE = self._saved_mode + super().teardown_method() + + def test_one_multistart_call_with_three_starts(self): + nodes = ["n1", "n2", "n3"] + solve = _Recorder(lambda n, s, *r: pytest.fail("sweep ran in free mode")) + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[1.2, 0.4, 0.9])) + + result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) + + assert result is not None and result["success"] + assert solve.calls == [] + assert len(multistart.calls) == 1 + (_, _, rest) = multistart.calls[0] + assert rest == ([7.0, 9.0, 11.0],) + + def test_n2_keeps_the_sweep(self): + """Altitude is unobservable at n=2 — the free path is not entered even + with the mode on.""" + nodes = ["n1", "n2"] + solve = _Recorder(lambda n, s, *r: _stub_result(n)) + multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken at n=2")) + + result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) + + assert result is not None + assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM) + assert multistart.calls == [] + + def test_a_failed_multistart_is_a_failed_solve(self): + """No silent fall back to the sweep: three starts producing nothing is + the same verdict as every layer producing nothing.""" + solve = _Recorder(lambda n, s, *r: pytest.fail("swept after a failed multistart")) + multistart = _Recorder(lambda n, s, *r: None) + + assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart) is None + + def test_history_carries_the_mode_and_the_per_start_residuals(self): + nodes = ["n1", "n2", "n3"] + multistart = _Recorder( + lambda n, s, *r: _stub_result( + n, altitude_mode="free", rms_by_start=[1.2345, None, 0.4321], alt_starts_km=[5.0, 7.0, 9.0] + ) + ) + solver_mod._process_solver_item( + (_s_in(nodes), {}, time.time()), + lambda s, c: pytest.fail("sweep ran in free mode"), + multistart_fn=multistart, + ) + + rec = state.mlat_solve_history[-1] + assert rec["outcome"] == "published" + assert rec["altitude_mode"] == "free" + assert rec["alt_starts_km"] == [5.0, 7.0, 9.0] + assert rec["alt_start_rms_us"] == [1.234, None, 0.432] + + def test_z_saturation_reaches_the_history(self): + nodes = ["n1", "n2", "n3"] + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", z_saturated=True)) + solver_mod._process_solver_item((_s_in(nodes), {}, time.time()), lambda s, c: None, multistart_fn=multistart) + assert state.mlat_solve_history[-1]["z_saturated"] is True + + def test_trimming_re_solves_through_the_multistart(self): + """A trim round must use the mode its first solve used, or the rms it + compares against the previous round is a different quantity.""" + full = ["n1", "n2", "n3", "n4", "bad"] + trimmed = ["n1", "n2", "n3", "n4"] + + def _result(nodes, s_in, *rest): + if "bad" in nodes: + return _stub_result( + nodes, + rms_delay=8.0, + altitude_mode="free", + per_node_delay_res_us={n: (12.0 if n == "bad" else 0.5) for n in nodes}, + ) + return _stub_result( + nodes, + rms_delay=0.8, + altitude_mode="free", + per_node_delay_res_us={n: 0.3 for n in nodes}, + ) + + multistart = _Recorder(_result) + result = solver_mod._process_solver_item( + (_s_in(full), {}, time.time()), + lambda s, c: pytest.fail("sweep ran during a free-mode trim"), + multistart_fn=multistart, + ) + + assert result is not None and result["n_nodes"] == 4 + assert sorted(m["node_id"] for m in multistart.calls[-1][0]["measurements"]) == trimmed + rec = state.mlat_solve_history[-1] + assert rec["outcome"] == "published" + assert rec["trimmed_node_ids"] == ["bad"] + assert rec["altitude_mode"] == "free" + + def test_an_unrecognised_mode_would_sweep(self): + """The flag degrades to the inert mode, like its siblings — asserted on + the resolution rule rather than by re-importing core.state.""" + state.SOLVER_ALT_MODE = "definitely-not-a-mode" + solve = _Recorder(lambda n, s, *r: _stub_result(n)) + multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken for a bad mode")) + assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart) + assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM) + + +class TestConfigsForSolverInput: + """Only the configs a candidate can reach are queued with it. + + The pool is a spawn pool, so whatever is queued is pickled and shipped on + every solve — 58 fleet configs against a candidate's 2-8 measurements. + """ + + _FLEET = {f"n{i}": {"rx_lat": 35.0 + i, "rx_lon": -82.0} for i in range(8)} + + def test_restricted_to_the_measurement_nodes(self): + s_in = _s_in(["n1", "n3", "n5"]) + cfgs = frame_processor.configs_for_solver_input(self._FLEET, s_in) + assert sorted(cfgs) == ["n1", "n3", "n5"] + assert cfgs["n3"] is self._FLEET["n3"] + + def test_unknown_measurement_nodes_are_simply_absent(self): + """A measurement from a node with no config is the case + solve_multinode already handles by skipping it — not an error here.""" + cfgs = frame_processor.configs_for_solver_input(self._FLEET, _s_in(["n1", "ghost"])) + assert sorted(cfgs) == ["n1"] + + def test_no_measurements_gives_nothing(self): + assert frame_processor.configs_for_solver_input(self._FLEET, {"measurements": []}) == {} + assert frame_processor.configs_for_solver_input(self._FLEET, {}) == {} diff --git a/docs/solverflow.md b/docs/solverflow.md index c626e5bf..50101e85 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -81,6 +81,7 @@ own. Everything that reaches a solve passes through one gate stack | `FRAME_WORKERS` | 4 (compose sets 6) | `main.py:164`, `docker-compose.yml:54` | | `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py:31,67` | | `KNOWN_LANE_MODE` default | `binding` | `core/state.py:72-74` | +| `SOLVER_ALT_MODE` default | `sweep` | `core/state.py:77-95` | --- @@ -453,12 +454,52 @@ flowchart TD m6 -->|"no"| m7["vz_saturated if vz on bound;
rms recomputed unweighted;
cov_en_km2 from s^2(J^T J)^-1"] m7 --> alt{"n_nodes >= 3?"} - alt -->|"yes"| sweep["_solve_best_altitude wrapper:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"] + alt -->|"yes"| mode{"SOLVER_ALT_MODE"} + mode -->|"sweep (default)"| sweep["_solve_best_altitude:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"] + mode -->|"free"| freealt["_solve_best_altitude:
ONE pool call to
solve_multinode_multistart,
3 start layers, z solved"] alt -->|"no, n=2"| single["_solve_best_altitude_n2:
one LM call at the
association altitude"] classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` +#### `SOLVER_ALT_MODE` — how the n>=3 solve gets its altitude + +`solve_multinode` pins altitude from `initial_guess.alt_km`, so the fix is only +as good as the altitude the caller found for it. `sweep`, the default, searches +the six fixed layers of `_SOLVER_ALT_LAYERS_KM` — 2 km apart, so the pin is +systematically up to 1 km wrong. On noise-free replay of this fleet's geometry +that quantisation alone left `rms_delay` at a 1.76 us median against the 3.0 us +gate at 6.5, while a solve at the true altitude reaches 0. Most of the gate's +budget is spent on the ladder, and the residual left over gets blamed on nodes: +trimming (6.4) drops measurements that were never the problem. + +`free` instead calls `solve_multinode_multistart`, which runs the LM with +altitude as a sixth unknown (state `[x, y, z, vx, vy, vz]`, z bounded +0.05–20 km, the `vz` bound unchanged) from three start layers — the one nearest +the association guess and its two neighbours — and keeps the lowest `rms_delay`. +Freeing z removes the ladder's quantisation but not the LM's locality, which is +what the several starts are still for. It is also cheaper: **one** process-pool +round trip per candidate instead of six, each of which pickles the node configs +the input needs. + +At n=2 the mode is inert — four residuals cannot support six unknowns, so the +geolocator pins altitude regardless and `_solve_best_altitude_n2` is unchanged. +Trimming re-solves through `_solve_best_altitude`, so a trim round inherits +whichever mode its first solve used. + +Both modes stamp `altitude_mode` (`"free"` / `"pinned"`) on every +`mlat_solve_history` record, published or rejected; `free` adds `alt_starts_km`, +`alt_start_rms_us` (each start's residual) and `z_saturated` (the altitude +analogue of `vz_saturated` — z stopped on a bound rather than converging, so +`alt_m` is the bound and not a fit). That is the comparison channel: deploy one +mode per environment and read the two lanes' `rms_delay` and `gt_error_km` off +`/api/test/mlat-history`. + +| Mode | Pool calls per n>=3 candidate | Altitude | +|---|---|---| +| `sweep` (default) | 6 (one per layer) | quantised to the nearest layer | +| `free` | 1 (three starts inside it) | solved, 0.05–20 km | + | Constant | Value | File:line | |---|---|---| | `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py:701` | diff --git a/libs/retina-geolocator b/libs/retina-geolocator index 2da39822..b87b4707 160000 --- a/libs/retina-geolocator +++ b/libs/retina-geolocator @@ -1 +1 @@ -Subproject commit 2da3982220374f05dd622633576108e57bd9e34f +Subproject commit b87b4707cecb331fea2a8895f3e63f256183f4a9 From a82e6b25d30fe16c26fcb6cd8eb700d0e1fb8f57 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 06:49:45 +0000 Subject: [PATCH 04/25] Satisfy the lint gate on the bench contamination scorer ruff format wrapped the truth-channel lookup and SIM910 wanted the redundant None default off dict.get. Co-Authored-By: Claude Fable 5.1 --- backend/scripts/association_bench.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py index 14dc6cdc..9e5792db 100644 --- a/backend/scripts/association_bench.py +++ b/backend/scripts/association_bench.py @@ -337,7 +337,7 @@ def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) the delay grid). Ambiguous attributions are counted in neither. """ oids = [ - det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"])), None) + det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"]))) for m in s_in.get("measurements") or [] ] if not oids: @@ -351,7 +351,11 @@ def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) guess = s_in.get("initial_guess") or {} contenders.sort( key=lambda o: min( - (_haversine_km(guess.get("lat", 0.0), guess.get("lon", 0.0), a, b) for a, b, oid, _ in truth if oid == o), + ( + _haversine_km(guess.get("lat", 0.0), guess.get("lon", 0.0), a, b) + for a, b, oid, _ in truth + if oid == o + ), default=float("inf"), ) ) @@ -1681,8 +1685,7 @@ def main(): "--merge-dist-km", type=float, default=None, - help="track mode: how close two pairings must be to merge into one " - "solver input (association._MERGE_DIST_KM)", + help="track mode: how close two pairings must be to merge into one solver input (association._MERGE_DIST_KM)", ) p.add_argument( "--pair-vel-exclusive", From 1c9e5def9b8988ee22214727ca211eec0938d627 Mon Sep 17 00:00:00 2001 From: jehanazad Date: Sat, 5 Sep 2026 06:56:57 +0000 Subject: [PATCH 05/25] Make the dark lane's blind spots readable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three of the signals the dark-lane recovery work is judged on could not be read from a running server, so every judgement about it has been offline replay. This adds no behaviour: nothing about which candidates solve, which solves publish, or what reaches the map changes. 1. /api/test/mlat-history caps per lane. The known lane writes ~16x the dark lane's volume, so the flat records[:1000] answered a 30 min request with the newest ~6 min of dark records and the rest of the window read as a quiet period — the same failure the shared deque had before #289 split it, moved into the reader. ?lane= narrows to one lane, ?limit= (max 5 000) is applied per lane, and lane_counts is reported pre-cap so truncation stays legible. 2. Resolve-slot skips are recorded, not just counted. Live the rule refuses ~1 537 candidates per 646 dark attempts per 30 min, and nothing said WHICH claim did it — so a skip that suppressed a duplicate and one that suppressed a different aircraft (tracker track ids are shared across candidates) were indistinguishable. A separate 500-entry deque carries the blocking claims; solver-stats windows it as resolve_skips, and ?kind=resolve_skips dumps it. Its own deque and not the solve history: skips outrun dark records two to one and would evict them. 3. Contamination is measured live. A dark record matched to ground truth now says which of its own contributing nodes could not see that aircraft (foreign_node_ids/contaminated), using the associator's own visibility predicate — the same gate known-lane claiming uses, so the two cannot drift apart. The GT trail lookup already happened; this costs one cone test per node. Records nothing could be asked about stay out of the denominator rather than counting as clean. 4. NODE_FRAME_MIN_INTERVAL_S drops are counted. frames_dropped is published but is the queue-saturation counter and reads zero throughout; the frames the per-node rate limiter refuses were uncounted entirely, so "the tracker sees what this node sent" looked true from every metric. docs/solverflow.md's file:line references are replaced with file + symbol — every one of them had drifted — and gain a section on reading these endpoints. Co-Authored-By: Claude Fable 5.1 --- backend/core/state.py | 35 +++- backend/routes/test.py | 135 ++++++++++++- backend/services/tasks/solver.py | 132 ++++++++++++- backend/services/tcp_handler.py | 6 + backend/tests/test_mlat_history.py | 288 ++++++++++++++++++++++++++++ backend/tests/test_solver_stats.py | 119 ++++++++++++ backend/tests/test_solver_worker.py | 44 +++++ docs/solverflow.md | 231 ++++++++++++---------- 8 files changed, 887 insertions(+), 103 deletions(-) diff --git a/backend/core/state.py b/backend/core/state.py index 634aa212..246907c9 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -475,6 +475,15 @@ def _adsb_for_seeding() -> dict[str, dict]: # Monotonic counter for dropped frames (useful for monitoring) frames_dropped: int = 0 +# Frames the per-node rate limiter refused before they ever reached +# frame_queue (tcp_handler's NODE_FRAME_MIN_INTERVAL_S gate). A different +# event from frames_dropped, which is queue saturation: this one is the +# pipeline deliberately sampling a node down to ~1 Hz, and a node streaming at +# 22 fps therefore reports a large number here while dropping nothing. It was +# uncounted, so "how much of a node's evidence does the tracker actually see" +# had no answer at all — the frames_dropped that IS published +# (/api/admin/metrics) says zero throughout. +node_frames_rate_limited: int = 0 frames_processed: int = 0 solver_successes: int = 0 solver_failures: int = 0 @@ -543,6 +552,25 @@ def _adsb_for_seeding() -> dict[str, dict]: # solver_stale_drops that means work was lost. solver_resolve_skips: int = 0 +# The dark-lane share of the counter above, split out because the two lanes +# read completely differently: an ADS-B-anchored duplicate that is skipped +# costs nothing (the transponder keeps the track alive anyway), while a +# skipped dark candidate may be the only chance that aircraft had of reaching +# the map this window. Lane is decided by solver._is_dark_solver_input, the +# same predicate routes.test._record_lane falls back to for a record that +# never got a key — and a skip never gets one. +solver_resolve_skips_dark: int = 0 + +# The last few hundred resolve-slot skips, with the claims that blocked them. +# Deliberately NOT the solve-history deque: a skip is not a solve outcome, and +# writing one record per skip into mlat_solve_history would evict the real +# records at roughly twice their rate (live: ~1 537 skips per 646 dark +# attempts per 30 min). Small and separate, read by +# /api/test/solver-stats' resolve_skips block and dumped by +# /api/test/mlat-history?kind=resolve_skips. ~250 B/entry. +SOLVER_RESOLVE_SKIPS_RECENT_MAX = 500 +solver_resolve_skips_recent: deque = deque(maxlen=SOLVER_RESOLVE_SKIPS_RECENT_MAX) + # Multinode entries removed because a later solve shared a source single-node # track with them AND the spatial/identical-inputs guard in solver.py's # _supersession_match agreed they are the same aircraft — the age-scaled @@ -746,12 +774,14 @@ def _reset_for_tests() -> None: global latest_mlat_accuracy_bytes, latest_mlat_verification_bytes global latest_storage_bytes, simulation_config global frames_dropped, frames_processed, solver_successes, solver_failures + global node_frames_rate_limited global adsb_seed_frames_autotagged, adsb_capture_ts_fallback global known_claims_made, known_claim_contentions, known_claims_bound global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes global coverage_rebuild_backlog global solver_queue_drops, solver_stale_drops, solver_resolve_skips + global solver_resolve_skips_dark global mn_superseded, mn_superseded_blocked, solver_trimmed global solver_consensus_selected, solver_consensus_filtered global solver_consensus_fallback, solver_consensus_shadow @@ -801,6 +831,7 @@ def _reset_for_tests() -> None: track_archive_buffer.clear() mlat_solve_history.clear() mlat_solve_history_known.clear() + solver_resolve_skips_recent.clear() accuracy_samples.clear() mlat_samples.clear() for q in (frame_queue, solver_queue): @@ -832,7 +863,7 @@ def _reset_for_tests() -> None: simulation_config = dict(_SIMULATION_CONFIG_DEFAULTS) with counters_lock: - frames_dropped = frames_processed = 0 + frames_dropped = frames_processed = node_frames_rate_limited = 0 solver_successes = solver_failures = n2_unconfirmed = 0 adsb_seed_frames_autotagged = adsb_capture_ts_fallback = 0 known_claims_made = known_claim_contentions = known_claims_bound = 0 @@ -841,7 +872,7 @@ def _reset_for_tests() -> None: coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0 coverage_rebuild_backlog = 0 solver_stale_drops = 0 - solver_resolve_skips = 0 + solver_resolve_skips = solver_resolve_skips_dark = 0 mn_superseded = mn_superseded_blocked = 0 solver_trimmed = 0 solver_consensus_selected = solver_consensus_filtered = 0 diff --git a/backend/routes/test.py b/backend/routes/test.py index e4018b36..5909ff9d 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -752,11 +752,40 @@ def _record_lane(rec: dict) -> str: return "adsb" if hexn and is_transponder_hex(hexn) else "dark" +_LANES = ("dark", "known", "adsb") + + +def _cap_per_lane(records: list[dict], limit: int) -> list[dict]: + """Keep the ``limit`` newest records OF EACH LANE, newest first. + + ``records`` must already be newest-first. A single flat ``[:limit]`` + made the cap a race between lanes rather than a retention rule, exactly + as the shared deque did before PR #289 split it: the known lane writes + ~16x the dark lane's volume, so a flat 1 000-record answer to a 30 min + request held only the newest ~6 min of dark records and the rest of the + window read as a quiet period. Capping per lane means known-lane volume + can never evict a dark record from a response. + """ + kept: list[dict] = [] + counts: dict[str, int] = {} + for r in records: + lane = _record_lane(r) + n = counts.get(lane, 0) + if n >= limit: + continue + counts[lane] = n + 1 + kept.append(r) + return kept + + @router.get("/api/test/mlat-history") async def mlat_history( hex: str | None = None, all: int = 0, minutes: float = 30.0, + lane: str = "all", + limit: int = 1000, + kind: str = "solves", ): """Per-solve MLAT history from the last ~30 minutes. @@ -770,22 +799,73 @@ async def mlat_history( merged here, so both lanes answer either query exactly as they did when they shared a deque. + ``?lane=dark|known|adsb`` narrows the answer to one lane (default + ``all``, classified by ``_record_lane``); ``?limit=`` caps the record + list (default 1 000, max 5 000) and is applied PER LANE, so a known-lane + burst can never push dark records out of an ``all`` response — see + _cap_per_lane. + + ``?kind=resolve_skips`` dumps a different store entirely: the solver's + recent resolve-slot refusals (state.solver_resolve_skips_recent), each + with the claims that blocked it. Those are not solve outcomes and + deliberately do not live in the solve-history deques. + ``window_effective_minutes`` is how much of the requested window the stores actually hold — below ``window_minutes`` the answer is truncated. """ + if lane not in ("all", *_LANES): + return Response( + content=orjson.dumps({"error": f"lane must be one of all,{','.join(_LANES)}"}), + media_type="application/json", + status_code=400, + ) + if kind not in ("solves", "resolve_skips"): + return Response( + content=orjson.dumps({"error": "kind must be solves or resolve_skips"}), + media_type="application/json", + status_code=400, + ) minutes = max(0.0, min(minutes, 35.0)) + limit = max(1, min(int(limit), 5000)) cutoff_ms = int((time.time() - minutes * 60.0) * 1000) + + if kind == "resolve_skips": + skips = [ + s + for s in list(state.solver_resolve_skips_recent) + if s["ts_ms"] >= cutoff_ms and (lane == "all" or s["lane"] == lane) + ] + skips.reverse() # newest first + payload = { + "kind": "resolve_skips", + "window_minutes": minutes, + "lane": lane, + "lane_counts": {ln: sum(1 for s in skips if s["lane"] == ln) for ln in _LANES}, + "n_records": len(skips), + "records": skips[:limit], + } + return Response(content=orjson.dumps(payload), media_type="application/json") + merged = _merged_solve_history() effective_minutes = _window_effective_minutes(merged, minutes) records = [r for r in merged if r["ts_ms"] >= cutoff_ms] records.reverse() # newest first + if lane != "all": + records = [r for r in records if _record_lane(r) == lane] + lane_counts = dict.fromkeys(_LANES, 0) + for r in records: + lane_counts[_record_lane(r)] += 1 if all: payload = { "window_minutes": minutes, "window_effective_minutes": effective_minutes, + "lane": lane, + # Pre-cap, so a truncated `records` can be read against what the + # window actually held. + "lane_counts": lane_counts, "n_records": len(records), - "records": records[:1000], + "records": _cap_per_lane(records, limit), } return Response(content=orjson.dumps(payload), media_type="application/json") @@ -813,6 +893,8 @@ async def mlat_history( "hex": norm, "window_minutes": minutes, "window_effective_minutes": effective_minutes, + "lane": lane, + "lane_counts": lane_counts, "n_solves": len(solves), "solves": solves[:500], "rejects_nearby": { @@ -914,6 +996,46 @@ def _solver_window_stats(minutes: float) -> dict: reason = outcome[len("rejected_") :] if outcome.startswith("rejected_") else outcome by_reason[reason] = by_reason.get(reason, 0) + 1 + # ── cluster contamination (dark, windowed) ────────────────────────────── + # Of the dark records this window that matched ground truth, how many + # carried a node that could not see the aircraft they were matched to — + # the live version of the offline number Phase 2 exists to move (~60 %). + # Records without the stamp are records nothing could be asked about (no + # GT match, or no registered geometry for any contributing node) and stay + # out of the denominator rather than counting as clean; see + # solver._stamp_foreign_nodes. + judged = [r for r in records if r.get("foreign_node_ids") is not None] + contaminated = [r for r in judged if r.get("contaminated")] + n_judged = len(judged) + contamination = { + "records_with_gt": n_judged, + "contaminated": len(contaminated), + "pct": round(100.0 * len(contaminated) / n_judged, 1) if n_judged else None, + "foreign_nodes_per_record": ( + round(sum(len(r["foreign_node_ids"]) for r in judged) / n_judged, 2) if n_judged else None + ), + } + + # ── resolve-slot skips (windowed, from the skip deque) ────────────────── + # The counter in "counters" below is since-boot; these are the skips that + # happened inside this window, so they can be read against the attempts in + # the same window. attempts_ratio is (all-lane skips / DARK attempts) — + # the shape the acceptance target for the claim-on-publish fix is quoted + # in (live baseline ~1 537 / 646 = 2.4), not a per-lane rate. The dark + # numerator is published beside it for anyone who wants one. + all_skips = list(state.solver_resolve_skips_recent) + skips = [s for s in all_skips if s["ts_ms"] >= cutoff_ms] + resolve_skips = { + "total": len(skips), + "dark": sum(1 for s in skips if s["lane"] == "dark"), + "attempts_ratio": round(len(skips) / attempts, 3) if attempts else None, + # The skip deque is 500 entries against a live rate of ~50/min, so a + # long window IS truncated here even when the solve-history stores + # cover it. Same honesty rule as window_effective_minutes above: read + # it before reading total as a window count. + "window_effective_minutes": _window_effective_minutes(all_skips, minutes), + } + pos_errors.sort() n_err = len(pos_errors) median_err = pos_errors[n_err // 2] if n_err else None @@ -1066,6 +1188,9 @@ def _solver_window_stats(minutes: float) -> dict: "published": {"total": n2 + n3plus, "n2": n2, "n3plus": n3plus}, "rejects": {"total": reject_total, "by_reason": by_reason}, "position_error_km": {"median": median_err, "p90": p90_err, "n": n_err}, + # Both windowed and both DARK-lane, like the funnel above them. + "contamination": contamination, + "resolve_skips": resolve_skips, "ghosts": { # Scoped to dark tracks: precision_pct's denominator is # dark_tracks, and it is None (not 100.0) when there are none. @@ -1192,7 +1317,15 @@ def _solver_window_stats(minutes: float) -> dict: "solver_trimmed": state.solver_trimmed, "stale_drops": state.solver_stale_drops, "resolve_skips": state.solver_resolve_skips, + # Dark share of the line above. The windowed version, with the + # blocking claims, is the "resolve_skips" block further up. + "resolve_skips_dark": state.solver_resolve_skips_dark, "queue_drops": state.solver_queue_drops, + # Frames the per-node rate limiter refused before the tracker ever + # saw them (tcp_handler's NODE_FRAME_MIN_INTERVAL_S). Not the + # same event as /api/admin/metrics' frames_dropped, which is + # frame_queue saturation and normally reads zero. + "node_frames_rate_limited": state.node_frames_rate_limited, "worker_errors": state.solver_worker_errors, "vel_untrusted_published": state.solver_vel_untrusted_published, }, diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 7c6b947c..5faf7bc5 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -11,6 +11,8 @@ from collections import deque from concurrent.futures.process import BrokenProcessPool +from retina_analytics.association import _point_in_beam + from config.constants import ( ARC_ONLY_ANOMALY_ALLOWLIST, ASSOC_GRID_STEP_KM, @@ -1036,6 +1038,64 @@ def _claim_resolve_slot(s_in, now_s: float) -> bool: return True +def _resolve_slot_blockers(track_ids, now_s: float) -> list[dict]: + """The live claims covering ``track_ids``, for a skip record. + + Read-only, and taken after the refusal rather than during it: the check + itself must stay one atomic test-and-claim, and a skip is rare enough + (relative to the queue drain rate) that a second lock acquisition on that + path costs nothing. Any claim that moves between the two is a claim the + diagnosis would have wanted to name anyway. + """ + cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S + out: list[dict] = [] + with _RECENT_SOLVES_LOCK: + for tid in track_ids: + held = _RECENT_SOLVES.get(tid) + if held is not None and held[0] > cutoff: + out.append({"track_id": tid, "held_ts": round(held[0], 3), "held_n": held[1]}) + return out + + +def _record_resolve_skip(s_in, now_s: float, blocking: list[dict] | None = None) -> None: + """Count and remember one resolve-slot refusal. + + The counter alone could not answer the question the suppression rule + raises — *whose* claim blocked this, and was it even the same aircraft. + Live on the test droplet the rule refuses ~1 537 candidates per 646 dark + attempts per 30 min, and nothing recorded which claim did it, so a skip + that suppressed a genuinely different aircraft (tracker track ids are + shared across candidates — see _supersession_match) was indistinguishable + from one that suppressed a duplicate. The deque carries the blocking + claims and the candidate's own guess position so the two can be told apart + after the fact. + + Deliberately NOT a solve-history record: skips outrun real dark records + roughly two to one, and writing them into that deque would evict the + solves the same investigation needs (see state.solver_resolve_skips_recent). + """ + s = s_in if isinstance(s_in, dict) else {} + track_ids = list(s.get("track_ids") or []) + dark = _is_dark_solver_input(s) + state.bump_counter("solver_resolve_skips") + if dark: + state.bump_counter("solver_resolve_skips_dark") + ig = s.get("initial_guess") or {} + state.solver_resolve_skips_recent.append( + { + "ts_ms": int(now_s * 1000), + # No key is minted for a candidate that never solves, so lane is + # the same fallback routes.test._record_lane uses for a reject. + "lane": "dark" if dark else "adsb", + "track_ids": track_ids, + "n_nodes": int(s.get("n_nodes") or 0), + "blocking": _resolve_slot_blockers(track_ids, now_s) if blocking is None else blocking, + "guess_lat": round(float(ig["lat"]), 6) if ig.get("lat") else None, + "guess_lon": round(float(ig["lon"]), 6) if ig.get("lon") else None, + } + ) + + # Which single-node track pair currently owns a published n=2 track, and how # well it fitted. One track is one aircraft, so two pairings sharing a track # are mutually exclusive; the better chi2 wins and the loser is withheld. @@ -1384,6 +1444,59 @@ def _is_dark_solver_input(s_in) -> bool: return not (hx and is_transponder_hex(hx)) +def _stamp_foreign_nodes(rec: dict) -> None: + """Stamp which of a dark record's own nodes could not see the aircraft. + + Cluster contamination is the dark lane's largest known defect — a + candidate assembled by format_track_pairs_for_solver can carry a node + whose track belongs to a *different* aircraft, and the solver then fits a + geometry no single aircraft ever occupied. Offline the audit measured it + at ~60 % of dark candidates; this makes the same number live. + + The test is the associator's own visibility predicate applied whole + (retina_analytics.association._point_in_beam against the registered + NodeGeometry), which is the same gate known-lane claiming uses — claiming + and the dark lane must mean the same thing by "this node can see there", + and a second bespoke rule here would let the two disagree. Two + consequences worth knowing: it is a ground-projected bearing/footprint + test with no altitude term, and under FOV_MODE=active it is the learned + FOV rather than the theoretical wedge. Both are exactly what the rest of + the pipeline believes about coverage, which is the point. + + Position is the matched ground-truth point already stamped on the record + (gt_lat/gt_lon at the solve epoch), so this costs no extra trail lookup — + only one cone test per contributing node. Nodes trimmed out by + _trim_and_resolve are included: a node dropped for a bad residual is + precisely the contamination this measures, and leaving it out would hide + every case trimming already rescued. + + A node with no registered geometry is not judged either way. When that + leaves nothing judgeable the record is left unstamped rather than stamped + clean, so contamination_pct never counts an abstention as innocence. + """ + lat, lon = rec.get("gt_lat"), rec.get("gt_lon") + if lat is None or lon is None: + return + node_ids = list(rec.get("contributing_node_ids") or []) + node_ids += [nid for nid in (rec.get("trimmed_node_ids") or []) if nid not in node_ids] + if not node_ids: + return + geometries = state.node_associator.node_geometries + judged = 0 + foreign: list[str] = [] + for nid in node_ids: + geo = geometries.get(nid) + if geo is None: + continue + judged += 1 + if not _point_in_beam(lat, lon, geo): + foreign.append(nid) + if not judged: + return + rec["foreign_node_ids"] = foreign + rec["contaminated"] = bool(foreign) + + def _record_dark_accuracy_sample(rec: dict) -> None: """Offer one published DARK solve to the rolling accuracy store. @@ -1460,7 +1573,14 @@ def _record_solve_history( ``extra`` merges caller-supplied fields (trim metadata, beam-rejection diagnostics) into the record. Applied before the GT stamp so it can - never clobber gt_hex/gt_error_km/gt_lat/gt_lon. + never clobber gt_hex/gt_error_km/gt_lat/gt_lon — and so the trimmed node + ids it carries are in hand for the contamination stamp below. + + ``foreign_node_ids``/``contaminated`` are stamped on DARK records that + matched ground truth: which of this candidate's own nodes could not see + the aircraft it was matched to (see _stamp_foreign_nodes). Absent on + every other record, which is what /api/test/solver-stats' contamination + block counts as "not judged" rather than as clean. """ r = result if isinstance(result, dict) else {} s = s_in if isinstance(s_in, dict) else {} @@ -1574,6 +1694,11 @@ def _record_solve_history( rec["vel_err_ms"] = round(math.hypot(ve - gt_ve, vn - gt_vn), 1) else: rec["vel_err_ms"] = None + # Live cluster-contamination metric, dark lane only and only where ground + # truth actually matched — without a truth position there is nothing to + # ask "could this node see it?" about. See _stamp_foreign_nodes. + if _dark and rec.get("gt_hex"): + _stamp_foreign_nodes(rec) if rec["outcome"] == "published" and _dark and rec.get("gt_error_km") is not None: _record_dark_accuracy_sample(rec) # Route by lane: the known lane's per-hex-per-pass volume would otherwise @@ -1724,8 +1849,9 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus # here rather than at enqueue: the frame path must not carry solver state, # and a copy that queued before its twin was solved can only be recognised # once it reaches a worker. - if not _claim_resolve_slot(s_in, time.time()): - state.bump_counter("solver_resolve_skips") + _now_s = time.time() + if not _claim_resolve_slot(s_in, _now_s): + _record_resolve_skip(s_in, _now_s) return None n_nodes = s_in.get("n_nodes", 0) if isinstance(s_in, dict) else 0 consensus_meta: dict | None = None diff --git a/backend/services/tcp_handler.py b/backend/services/tcp_handler.py index bbe82197..6b21be77 100644 --- a/backend/services/tcp_handler.py +++ b/backend/services/tcp_handler.py @@ -529,6 +529,12 @@ def _enqueue_detection(msg: dict, node_id: str | None): if node_id: last = _per_node_last_enqueue.get(node_id, 0.0) if (now_m - last) < _NODE_MIN_INTERVAL_S: + # Counted, not silent: this is the only place a node's detections + # are discarded on purpose, and until now nothing said how many. + # state.frames_dropped is the queue-saturation counter and reads + # zero throughout, so "the tracker sees every frame this node + # sent" looked true from every published metric. + state.bump_counter("node_frames_rate_limited") return # position already updated; skip expensive queue work _per_node_last_enqueue[node_id] = now_m diff --git a/backend/tests/test_mlat_history.py b/backend/tests/test_mlat_history.py index e03d9da4..e8e91bbe 100644 --- a/backend/tests/test_mlat_history.py +++ b/backend/tests/test_mlat_history.py @@ -777,3 +777,291 @@ def test_known_lane_records_are_not_sampled_by_this_path(self): extra={"known_lane": True, "label": "truth_match", "published": True}, ) assert not state.accuracy_samples + + +def _register_geo(node_id, beam_azimuth_deg, rx_lat=LAT, rx_lon=LON, max_range_km=50.0): + """Register one node geometry with the associator, aimed as given. + + The contamination stamp asks the associator's own visibility predicate, + so a test node has to exist there rather than in a config dict. + """ + from retina_analytics.association import NodeGeometry + + geo = NodeGeometry( + node_id=node_id, + rx_lat=rx_lat, + rx_lon=rx_lon, + rx_alt_km=0.0, + tx_lat=rx_lat + 0.5, + tx_lon=rx_lon + 0.5, + tx_alt_km=0.3, + beam_azimuth_deg=beam_azimuth_deg, + beam_width_deg=41.0, + max_range_km=max_range_km, + ) + state.node_associator.node_geometries[node_id] = geo + return geo + + +class TestForeignNodeStamp: + """A dark record matched to ground truth says which of its own nodes + could not have seen that aircraft. + + Cluster contamination — a solver candidate assembled from tracks of two + different aircraft — is the dark lane's largest known defect, and until + now it was measurable only offline. The verdict is the associator's own + visibility predicate, the same one known-lane claiming gates on. + """ + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + def _run(self, contributing=("n_in", "n_out"), **extra): + return solver_mod._process_solver_item( + (dict(_CONFIRMED_N2), {}, time.time()), + _solve_fn(contributing_node_ids=list(contributing), **extra), + ) + + def test_a_node_aimed_away_is_named_foreign(self): + # Ground truth sits due north of both nodes; n_in is aimed at it and + # n_out at the opposite bearing. + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=180.0) + self._run() + rec = state.mlat_solve_history[0] + assert rec["gt_hex"] == "abc123" + assert rec["foreign_node_ids"] == ["n_out"] + assert rec["contaminated"] is True + + def test_all_nodes_in_cone_is_not_contaminated(self): + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=10.0) + self._run() + rec = state.mlat_solve_history[0] + assert rec["foreign_node_ids"] == [] + assert rec["contaminated"] is False + + def test_a_node_out_of_range_is_foreign(self): + """Range, not only bearing: the predicate applies whole.""" + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=0.0, max_range_km=1.0) + self._run() + assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_out"] + + def test_trimmed_nodes_are_judged_too(self): + """A node dropped by _trim_and_resolve is exactly the contamination + this measures — excluding it would hide every case trimming already + rescued.""" + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_trimmed", beam_azimuth_deg=180.0) + solver_mod._record_solve_history( + "published", + dict(_CONFIRMED_N2), + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2, "contributing_node_ids": ["n_in"]}, + solve_key="mn-dark-1", + raw_lat=LAT, + raw_lon=LON, + extra={"trimmed_node_ids": ["n_trimmed"], "trim_rounds": 1}, + ) + assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_trimmed"] + + def test_no_ground_truth_means_no_stamp(self): + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=180.0) + self._run() + rec = state.mlat_solve_history[0] + assert "foreign_node_ids" not in rec + assert "contaminated" not in rec + + def test_unregistered_nodes_are_not_stamped_clean(self): + """Nothing judgeable is an abstention, not innocence.""" + _put_gt(lat=LAT + 0.05, lon=LON) + self._run() + rec = state.mlat_solve_history[0] + assert rec["gt_hex"] == "abc123" + assert "foreign_node_ids" not in rec + + def test_an_adsb_record_is_not_stamped(self): + """Dark lane only — the tagged lane's identity is not in doubt.""" + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=180.0) + s_in = dict(_CONFIRMED_N2, adsb_hex="abc123") + solver_mod._process_solver_item( + (s_in, {}, time.time()), + _solve_fn(contributing_node_ids=["n_in", "n_out"]), + ) + assert "foreign_node_ids" not in state.mlat_solve_history[0] + + +class TestLaneFilterAndPerLaneCap: + """?lane= and ?limit= on /api/test/mlat-history. + + The flat records[:1000] cap made the response a race between lanes: the + known lane writes ~16x the dark lane's volume, so a 30 min request held + only the newest ~6 min of dark records and the rest of the window read as + a quiet period. The cap is now per lane. + """ + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + def _client(self): + from main import app + + return TestClient(app) + + def _dark(self, n=1): + for _ in range(n): + solver_mod._record_solve_history( + "published", + {"n_nodes": 3}, + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3}, + solve_key="mn-dark-1", + raw_lat=LAT, + raw_lon=LON, + ) + + def _known(self, n=1): + for _ in range(n): + solver_mod._record_solve_history( + "known_truth_match", + {"n_nodes": 2, "adsb_hex": "abc123", "initial_guess": {"lat": LAT, "lon": LON}}, + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2}, + extra={"known_lane": True, "label": "truth_match", "published": False}, + ) + + def _adsb(self, n=1): + for _ in range(n): + solver_mod._record_solve_history( + "published", + {"n_nodes": 3, "adsb_hex": "abc123"}, + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3}, + solve_key="mn-adsb-abc123", + raw_lat=LAT, + raw_lon=LON, + ) + + def test_default_lane_is_all_and_counts_every_lane(self): + self._dark() + self._known() + self._adsb() + data = self._client().get("/api/test/mlat-history?all=1").json() + assert data["lane"] == "all" + assert data["lane_counts"] == {"dark": 1, "known": 1, "adsb": 1} + assert data["n_records"] == 3 + + def test_lane_dark_returns_only_dark_records(self): + self._dark(2) + self._known(3) + self._adsb(1) + data = self._client().get("/api/test/mlat-history?all=1&lane=dark").json() + assert data["n_records"] == 2 + assert data["lane_counts"] == {"dark": 2, "known": 0, "adsb": 0} + assert all(r["solve_key"] == "mn-dark-1" for r in data["records"]) + + def test_lane_known_returns_only_known_records(self): + self._dark(2) + self._known(3) + data = self._client().get("/api/test/mlat-history?all=1&lane=known").json() + assert data["n_records"] == 3 + assert all(r["known_lane"] for r in data["records"]) + + def test_unknown_lane_is_rejected(self): + assert self._client().get("/api/test/mlat-history?all=1&lane=bogus").status_code == 400 + + def test_known_volume_cannot_evict_dark_records_from_the_response(self): + """The bug the per-lane cap fixes, at 1/500 scale.""" + self._dark(2) + self._known(20) + data = self._client().get("/api/test/mlat-history?all=1&limit=2").json() + # 2 dark + 2 known survive the cap; the flat cap would have returned + # the 2 newest records overall, both known. + lanes = [("known" if r.get("known_lane") else "dark") for r in data["records"]] + assert sorted(lanes) == ["dark", "dark", "known", "known"] + # n_records / lane_counts stay pre-cap so truncation is legible. + assert data["n_records"] == 22 + assert data["lane_counts"] == {"dark": 2, "known": 20, "adsb": 0} + + def test_limit_is_clamped_to_the_maximum(self): + self._dark(3) + data = self._client().get("/api/test/mlat-history?all=1&limit=99999").json() + assert len(data["records"]) == 3 + + def test_hex_lookup_reports_the_lane_block_too(self): + self._dark() + rec = state.mlat_solve_history[0] + data = self._client().get(f"/api/test/mlat-history?hex={rec['solver_hex']}").json() + assert data["lane"] == "all" + assert data["lane_counts"]["dark"] == 1 + + +class TestResolveSkipDump: + """?kind=resolve_skips dumps the solver's skip deque. + + A skip is not a solve outcome and must not be written into the + solve-history deques: on the live fleet skips outrun dark records roughly + two to one and would evict exactly the records an investigation needs. + """ + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + def _client(self): + from main import app + + return TestClient(app) + + def _skip(self, track_ids=("a1", "b1"), n_nodes=3, **s_in): + now = time.time() + s = dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids), **s_in) + solver_mod._claim_resolve_slot(s, now) + assert solver_mod._claim_resolve_slot(dict(s), now) is False + solver_mod._record_resolve_skip(dict(s), now) + + def test_skip_records_the_blocking_claim(self): + self._skip() + data = self._client().get("/api/test/mlat-history?kind=resolve_skips").json() + assert data["kind"] == "resolve_skips" + assert data["n_records"] == 1 + rec = data["records"][0] + assert rec["lane"] == "dark" + assert rec["track_ids"] == ["a1", "b1"] + assert rec["n_nodes"] == 3 + assert {b["track_id"] for b in rec["blocking"]} == {"a1", "b1"} + assert all(b["held_n"] == 3 for b in rec["blocking"]) + + def test_skips_do_not_land_in_the_solve_history(self): + self._skip() + assert not state.mlat_solve_history + assert not state.mlat_solve_history_known + + def test_lane_filter_applies_to_skips(self): + self._skip(track_ids=("a1", "b1")) + self._skip(track_ids=("a2", "b2"), adsb_hex="abc123") + assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=dark").json()["n_records"] == 1 + assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=adsb").json()["n_records"] == 1 + assert self._client().get("/api/test/mlat-history?kind=resolve_skips").json()["lane_counts"] == { + "dark": 1, + "known": 0, + "adsb": 1, + } + + def test_unknown_kind_is_rejected(self): + assert self._client().get("/api/test/mlat-history?kind=bogus").status_code == 400 diff --git a/backend/tests/test_solver_stats.py b/backend/tests/test_solver_stats.py index 67bc599c..806e8992 100644 --- a/backend/tests/test_solver_stats.py +++ b/backend/tests/test_solver_stats.py @@ -257,6 +257,8 @@ def test_consensus_and_counters_reflect_state(self): state.solver_consensus_fallback = 9 state.solver_consensus_shadow = 10 state.solver_vel_untrusted_published = 11 + state.solver_resolve_skips_dark = 9 + state.node_frames_rate_limited = 13 out = _solver_window_stats(10.0) assert out["counters"] == { "successes": 5, @@ -265,7 +267,9 @@ def test_consensus_and_counters_reflect_state(self): "solver_trimmed": 3, "stale_drops": 4, "resolve_skips": 12, + "resolve_skips_dark": 9, "queue_drops": 6, + "node_frames_rate_limited": 13, "worker_errors": 0, "vel_untrusted_published": 11, } @@ -788,3 +792,118 @@ def get(self, *a, **kw): state.multinode_tracks["mn-dark-1"] = {"lat": 35.0, "lon": -82.0} state.adsb_aircraft["real1"] = _MutatingFix({"lat": 35.009, "lon": -82.0, "last_seen_ms": now_ms}) assert _solver_window_stats(10.0)["ghosts"]["ghost_tracks"] == 0 + + +def _skip_rec(lane="dark", age_s=0.0, track_ids=("a1",), n_nodes=3): + return { + "ts_ms": int((time.time() - age_s) * 1000), + "lane": lane, + "track_ids": list(track_ids), + "n_nodes": n_nodes, + "blocking": [{"track_id": track_ids[0], "held_ts": time.time() - age_s, "held_n": n_nodes}], + "guess_lat": None, + "guess_lon": None, + } + + +class TestResolveSkipBlock: + """Resolve-slot skips are windowed from their own deque, not from the + since-boot counter, so they can be read against the attempts in the same + window — the ratio the claim-on-publish fix is judged on.""" + + def setup_method(self): + state._reset_for_tests() + + def test_totals_split_by_lane(self): + for _ in range(3): + state.solver_resolve_skips_recent.append(_skip_rec("dark")) + state.solver_resolve_skips_recent.append(_skip_rec("adsb")) + out = _solver_window_stats(10.0)["resolve_skips"] + assert out["total"] == 4 + assert out["dark"] == 3 + + def test_window_excludes_old_skips(self): + state.solver_resolve_skips_recent.append(_skip_rec(age_s=20 * 60)) + state.solver_resolve_skips_recent.append(_skip_rec(age_s=1)) + assert _solver_window_stats(10.0)["resolve_skips"]["total"] == 1 + + def test_attempts_ratio_is_skips_over_dark_attempts(self): + for _ in range(4): + state.solver_resolve_skips_recent.append(_skip_rec()) + state.mlat_solve_history.append(_rec("published")) + state.mlat_solve_history.append(_rec("rejected_beam")) + out = _solver_window_stats(10.0) + assert out["attempts"] == 2 + assert out["resolve_skips"]["attempts_ratio"] == 2.0 + + def test_attempts_ratio_is_none_without_attempts(self): + state.solver_resolve_skips_recent.append(_skip_rec()) + assert _solver_window_stats(10.0)["resolve_skips"]["attempts_ratio"] is None + + def test_window_effective_minutes_exposes_a_truncated_deque(self): + """The deque is 500 entries against ~50 skips/min live, so a long + window IS truncated here even when the solve stores cover it.""" + state.solver_resolve_skips_recent.append(_skip_rec(age_s=6 * 60)) + out = _solver_window_stats(30.0)["resolve_skips"] + assert 5.9 <= out["window_effective_minutes"] <= 6.1 + + def test_a_skip_is_not_an_attempt_or_a_reject(self): + """Skips must not leak into the funnel — they never reached a solve.""" + for _ in range(5): + state.solver_resolve_skips_recent.append(_skip_rec()) + out = _solver_window_stats(10.0) + assert out["attempts"] == 0 + assert out["rejects"]["total"] == 0 + + +def _gt_rec(foreign=(), **kw): + """A dark record carrying the contamination stamp.""" + rec = _rec("published", **kw) + rec["gt_hex"] = "abc123" + rec["foreign_node_ids"] = list(foreign) + rec["contaminated"] = bool(foreign) + return rec + + +class TestContaminationBlock: + """Live cluster contamination: of the dark records that matched ground + truth, how many carried a node that could not see the aircraft.""" + + def setup_method(self): + state._reset_for_tests() + + def test_pct_and_mean_over_judged_records(self): + state.mlat_solve_history.append(_gt_rec(foreign=["n1"])) + state.mlat_solve_history.append(_gt_rec(foreign=["n1", "n2"])) + state.mlat_solve_history.append(_gt_rec(foreign=[])) + state.mlat_solve_history.append(_gt_rec(foreign=[])) + out = _solver_window_stats(10.0)["contamination"] + assert out["records_with_gt"] == 4 + assert out["contaminated"] == 2 + assert out["pct"] == 50.0 + assert out["foreign_nodes_per_record"] == 0.75 + + def test_unstamped_records_are_out_of_the_denominator(self): + """No GT match, or no judgeable node geometry, is an abstention — not + a clean record.""" + state.mlat_solve_history.append(_gt_rec(foreign=["n1"])) + state.mlat_solve_history.append(_rec("published")) + out = _solver_window_stats(10.0)["contamination"] + assert out["records_with_gt"] == 1 + assert out["pct"] == 100.0 + + def test_empty_window_abstains_rather_than_reporting_zero(self): + out = _solver_window_stats(10.0)["contamination"] + assert out == { + "records_with_gt": 0, + "contaminated": 0, + "pct": None, + "foreign_nodes_per_record": None, + } + + def test_known_lane_records_are_not_counted(self): + """Dark lane only, like every other block in the funnel.""" + rec = _gt_rec(foreign=["n1"]) + rec["known_lane"] = True + state.mlat_solve_history_known.append(rec) + assert _solver_window_stats(10.0)["contamination"]["records_with_gt"] == 0 diff --git a/backend/tests/test_solver_worker.py b/backend/tests/test_solver_worker.py index 6f5ea7b3..a4a6c8e9 100644 --- a/backend/tests/test_solver_worker.py +++ b/backend/tests/test_solver_worker.py @@ -38,6 +38,7 @@ def _reset_state(): state.n2_unconfirmed = 0 state.solver_stale_drops = 0 state.solver_resolve_skips = 0 + state.solver_resolve_skips_dark = 0 state.multinode_tracks.clear() state.task_last_success.clear() @@ -415,6 +416,49 @@ def solve_fn(s_in, cfgs): assert state.solver_failures == 0 assert state.solver_stale_drops == 0 + def test_a_skip_is_recorded_with_the_claim_that_blocked_it(self): + """The counter alone cannot say WHOSE claim suppressed a candidate, + and tracker track ids are shared between different aircraft — so a + skip that suppressed a duplicate and one that suppressed a neighbour + looked identical. The deque carries the blocking claims.""" + _reset_state() + state.solver_resolve_skips_recent.clear() + now = time.time() + s_in = dict(self._s_in(["a1", "b1"], n_nodes=4), initial_guess={"lat": 35.0, "lon": -82.0}) + assert solver_mod._claim_resolve_slot(dict(s_in), now) is True + assert solver_mod._claim_resolve_slot(dict(s_in), now) is False + solver_mod._record_resolve_skip(dict(s_in), now) + + assert state.solver_resolve_skips == 1 + assert state.solver_resolve_skips_dark == 1 + assert len(state.solver_resolve_skips_recent) == 1 + rec = state.solver_resolve_skips_recent[0] + assert rec["lane"] == "dark" + assert rec["track_ids"] == ["a1", "b1"] + assert rec["n_nodes"] == 4 + assert rec["guess_lat"] == 35.0 + assert {b["track_id"]: b["held_n"] for b in rec["blocking"]} == {"a1": 4, "b1": 4} + + def test_a_tagged_candidate_is_counted_but_not_as_dark(self): + _reset_state() + state.solver_resolve_skips_recent.clear() + now = time.time() + s_in = dict(self._s_in(["a1"], n_nodes=3), adsb_hex="abc123") + solver_mod._record_resolve_skip(s_in, now) + assert state.solver_resolve_skips == 1 + assert state.solver_resolve_skips_dark == 0 + assert state.solver_resolve_skips_recent[0]["lane"] == "adsb" + + def test_skips_never_enter_the_solve_history(self): + """One skip per solve-history record would evict the solves the same + investigation needs — live, skips outrun dark records two to one.""" + _reset_state() + state.mlat_solve_history.clear() + s_in = self._s_in(["a1", "b1"]) + solver_mod._record_resolve_skip(s_in, time.time()) + assert not state.mlat_solve_history + assert not state.mlat_solve_history_known + class TestSolveBestAltitude: """Altitude-sweep helpers: n_nodes >= 3 uses a layer sweep, n_nodes = 2 uses initial_guess directly.""" diff --git a/docs/solverflow.md b/docs/solverflow.md index c626e5bf..c80c4673 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -11,10 +11,12 @@ beyond what publication needs, see [`pipeline.md`](pipeline.md) (its own §3 is stale on the known lane and pool fallback — this doc is the current source for those two topics). -File:line references are repo-relative to `backend/`, except the `libs/*` -paths, which are already fully qualified (those are separate submodule repos -vendored under `libs/`). All references were checked against `main` at -`0a1d30f`. +References name a **file and a symbol**, never a line number: paths are +repo-relative to `backend/`, except the `libs/*` ones, which are already fully +qualified (those are separate submodule repos vendored under `libs/`). Line +numbers were what this document used to carry, and they were stale within two +weeks of being written — every one of them had drifted by the time anyone +followed it. A symbol survives an edit above it, so grep for the name. ## Legend @@ -74,13 +76,13 @@ lane rides the solver loop's idle cycles rather than owning workers of its own. Everything that reaches a solve passes through one gate stack (`_process_solver_item`) before publication. -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py:358-359` | -| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py:365-366` | -| `FRAME_WORKERS` | 4 (compose sets 6) | `main.py:164`, `docker-compose.yml:54` | -| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py:31,67` | -| `KNOWN_LANE_MODE` default | `binding` | `core/state.py:72-74` | +| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py` | +| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py` | +| `FRAME_WORKERS` | 4 (compose sets 6) | `core/state.py` (`FRAME_WORKERS`), `docker-compose.yml` | +| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py` (`_N_SOLVER_WORKERS`, `_make_solver_pool`) | +| `KNOWN_LANE_MODE` default | `binding` | `core/state.py` (`KNOWN_LANE_MODE`) | --- @@ -89,11 +91,11 @@ own. Everything that reaches a solve passes through one gate stack ```mermaid flowchart TD subgraph producers["Five producers"] - p1["TCP (primary)
tcp_handler.py:326"] - p2["blah2 bridge
blah2_bridge.py:289"] - p3["v1 node HTTP API
node_stream.py:250"] - p4["Legacy HTTP radar routes
routes/radar.py:151,202"] - p5["Startup priming
node_pipeline.py:139"] + p1["TCP (primary)
tcp_handler._enqueue_detection"] + p2["blah2 bridge
blah2_bridge.blah2_bridge_task"] + p3["v1 node HTTP API
node_stream._file_frame"] + p4["Legacy HTTP radar routes
radar.ingest_detections(_bulk)"] + p5["Startup priming
node_pipeline.prime_pipeline"] end p1 --> gA{"Gate A: timestamp present?"} @@ -131,25 +133,25 @@ flowchart TD classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` -The ordering inside `process_one_frame` (`services/frame_processor.py:294`) is +The ordering inside `process_one_frame` (`services/frame_processor.py`) is load-bearing, not incidental: claiming (2.3) runs **before** ADS-B seeding (2.4) so a node-supplied `adsb` field is still distinguishable from a claim, and both run **before** the tracker (2.5) so that, in `binding` mode, a claimed detection never reaches the dark-lane tracker or association at all -— see the ordering comment at `services/frame_processor.py:327-337`. +— see the ordering comment at the head of `process_one_frame`'s claiming step. Frame-level gates (A/B/C on TCP, plus the connected-node check on the v1 API) sit ahead of everything else; nothing downstream sees a frame that failed one of them. -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| Gate A: timestamp required | — | `tcp_handler.py:513-516` | -| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node | `tcp_handler.py:495,527-532` | -| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler.py:536-552` | -| `process_one_frame` entry | — | `services/frame_processor.py:294` | -| Ordering rationale (claim → seed → tracker) | — | `services/frame_processor.py:327-337` | -| Gate 2.10: `n_nodes < 2` skip | — | `services/frame_processor.py:409-426` | -| blah2 poll interval | 1.0 s | `config/constants.py:266` | +| Gate A: timestamp required | — | `tcp_handler._enqueue_detection` | +| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node, counted as `node_frames_rate_limited` | `tcp_handler` (`_NODE_MIN_INTERVAL_S`, `_enqueue_detection`) | +| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler._enqueue_detection` | +| `process_one_frame` entry | — | `services/frame_processor.py` | +| Ordering rationale (claim → seed → tracker) | — | `frame_processor.process_one_frame` | +| Gate 2.10: `n_nodes < 2` skip | — | `frame_processor.process_one_frame` | +| blah2 poll interval | 1.0 s | `config/constants.py` (`BLAH2_POLL_INTERVAL_S`) | --- @@ -205,17 +207,17 @@ also reject — a differential property test in `test_known_claiming.py` failure increments the same `known_claims_visibility_rejects` counter as a gate failure: same event, same meaning, just caught cheaper. -**Mode semantics** (`KNOWN_LANE_MODE`, read once at `core/state.py:72-74`, +**Mode semantics** (`KNOWN_LANE_MODE`, read once in `core/state.py`, default `binding`; an unrecognized value falls back to `shadow`, not to the default — a typo should degrade to the inert mode, not the acting one): | Mode | Claiming | Frame the dark lane sees | Known-lane solver | Publication | |---|---|---|---|---| -| `off` | never runs | untouched | returns 0 immediately (`known_lane.py:391-392`); worker never even calls it (`solver.py:1961`) | none | +| `off` | never runs | untouched | returns 0 immediately (`known_lane.run_known_lane_pass`); worker never even calls it (`solver._run_solver_worker`) | none | | `shadow` | runs, records claims + residuals + counters | untouched | runs: solves, classifies, records accuracy samples | never | -| `binding` | runs | `strip_claimed_detections` removes claimed indices (`frame_processor.py:347`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish | +| `binding` | runs | `strip_claimed_detections` removes claimed indices (called from `frame_processor.process_one_frame`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish | -`strip_claimed_detections` (`services/known_claiming.py:343`) returns a copy +`strip_claimed_detections` (`services/known_claiming.py`) returns a copy with claimed indices removed from `delay`/`doppler`/`snr`/`adsb`; the original frame still feeds the archive and ADS-B extraction (steps 2.11-2.12) unchanged. @@ -224,7 +226,7 @@ unchanged. ```mermaid flowchart TD - arm["Solver worker loop arms known_lane
at thread start (solver.py:1961)"] + arm["Solver worker loop arms known_lane
at thread start (solver._run_solver_worker)"] arm --> drain["After every queue-drain iteration,
call maybe_run_pass"] drain --> gm{"mode == off?"} gm -->|"yes"| ret1["return"]:::inert @@ -261,7 +263,7 @@ flowchart TD gpub -->|"no"| noop["accuracy sample only,
no feed entry"]:::inert ``` -The docstring at `services/tasks/known_lane.py:19-27` calls this the "free +The module docstring of `services/tasks/known_lane.py` calls this the "free solve invariant": the ADS-B fix seeds the initial guess and pins altitude, nothing else — no regularization pulls the solve toward the truth position, so the residual (`err_km`) is a genuine measurement of radar accuracy, not a @@ -269,21 +271,21 @@ circular check. One more intentional-by-omission detail: known-lane measurements carry `snr = 0.0` (claim records have no `snr` key), which the LM's SNR weighting maps to a uniform weight of 1.0. -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `ADSB_SEED_MAX_DR_AGE_S`, `association.py:106`) | -| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `ADSB_SEED_*`, `association.py:98,99`) | -| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py:79` | -| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py:205` | -| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py:89`, applied in `known_claiming.py:_dark_global_projections` | -| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py:73,77` | -| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py:80` | -| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py:85,86` | -| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py:274-275` | -| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py:105` | -| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py:91,99` | -| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py:110` | -| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py:205` | +| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `association.ADSB_SEED_MAX_DR_AGE_S`) | +| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `association.ADSB_SEED_DELAY_GATE_US` / `_DOPPLER_GATE_HZ`) | +| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py` | +| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py` | +| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py`, applied in `known_claiming._dark_global_projections` | +| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py` | +| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py` | +| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py` | +| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py` | +| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py` | +| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py` | +| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py` | +| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py` | --- @@ -291,7 +293,7 @@ LM's SNR weighting maps to a uniform weight of 1.0. ```mermaid flowchart TD - frame["pipeline.process_frame
passive_radar.py:672"] + frame["PassiveRadarPipeline.process_frame
pipeline/passive_radar.py"] frame --> tracker["retina_tracker
Kalman + GNN"] tracker --> geo["_run_geolocation per track
with new data"] @@ -345,7 +347,7 @@ flowchart TD classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` -`compute_overlap_zone` (`libs/retina-analytics/.../association.py:578`) +`compute_overlap_zone` (`libs/retina-analytics/.../association.py`) underlies both the confirmed-track association round and the overlap-grid cache: it fast-prunes non-overlapping node pairs by receiver separation, grids the shared coverage at `ASSOC_GRID_STEP_KM` on six altitude layers that @@ -353,22 +355,22 @@ must match the solver's `_SOLVER_ALT_LAYERS_KM`, and requires each grid column to fall in **both** beams (`_point_in_beam`, FOV-aware only when `FOV_MODE=active`). -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py:194` | -| Single-node min detections | 3 | `passive_radar.py:356-361` | -| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py:59` | -| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py:220` | -| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py:22` | -| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py:23` | -| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py:31`, `association.py:1043` | -| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.py:883` | -| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.py:884` | -| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py:166` | -| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py:57-58` | -| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.py:2160` | -| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py:21` | -| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py:111` | +| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py` (applied as `_GEO_INTERVAL_S` in `_run_geolocation`) | +| Single-node min detections | 3 | `pipeline/passive_radar.py` (`_geolocate_track_event`, `min_det`) | +| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py` | +| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py` | +| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py` | +| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py` | +| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py`, `association.py` | +| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.compute_overlap_zone` (default arg) | +| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.compute_overlap_zone` (default arg) | +| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py` | +| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py` | +| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.InterNodeAssociator.format_track_pairs_for_solver` (local) | +| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py` | +| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py` | --- @@ -376,7 +378,7 @@ column to fall in **both** beams (`_point_in_beam`, FOV-aware only when The centerpiece: every candidate from either lane, once dequeued from `solver_queue`, runs through `_process_solver_item` -(`services/tasks/solver.py:1344`) as a strict, ordered chain. A failure at +(`services/tasks/solver.py`) as a strict, ordered chain. A failure at any gate stops the chain, bumps a counter, and (from 6.5 onward) writes a named record to solve history. @@ -430,14 +432,14 @@ flowchart TD ``` `SOLVER_CONSENSUS_MODE` is `off` in production (see the mode-flag table in -[`architecture.md:94-110`](architecture.md#feature-gates)), so in practice +[`architecture.md`](architecture.md#feature-gates)), so in practice this sub-branch never reaches `active` outside staging. ### The LM itself -`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py:518`, +`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py`, invoked through the process pool via `_pool_solve_multinode` -(`services/tasks/solver.py:1915`). +(`services/tasks/solver.py`). ```mermaid flowchart TD @@ -459,19 +461,19 @@ flowchart TD classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py:701` | -| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py:744` | -| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py:160-162` | -| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py:132` | -| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py:173` | -| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py:205` | -| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py:56` | -| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py:807` | -| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py:154` | -| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py:51,52` | -| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py:57,63` | +| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py` | +| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py` (`_SOLVER_RESOLVE_INTERVAL_S`) | +| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py` | +| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py` (`_SOLVER_RMS_DELAY_MAX_US`) | +| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py` | +| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py` | +| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py` | +| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py` | +| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py` | +| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py` | +| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py` | --- @@ -525,26 +527,61 @@ flowchart TD | Value | Set at | Meaning | |---|---|---| -| `multinode_solve` | `aircraft_feed.py:132` | published multi-node solve | -| `solver_adsb_seed` | `track_gates.py:330` | single-node LM with fresh ADS-B fix | -| `solver_single_node` | `track_gates.py:330` | single-node LM, no ADS-B | -| `single_node_ellipse_arc` | `track_gates.py:378` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint | -| `adsb_single_node` | `aircraft_feed.py:_claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` | -| `known_lane_truth_match` / `known_lane_ghost` | `known_lane.py:260` | accuracy-sample-only, not a feed entry | - -| Constant | Value | File:line | +| `multinode_solve` | `aircraft_feed.multinode_to_aircraft` | published multi-node solve | +| `solver_adsb_seed` | `track_gates.track_entry` | single-node LM with fresh ADS-B fix | +| `solver_single_node` | `track_gates.track_entry` | single-node LM, no ADS-B | +| `single_node_ellipse_arc` | `track_gates.track_entry` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint | +| `adsb_single_node` | `aircraft_feed._claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` | +| `known_lane_truth_match` / `known_lane_ghost` | `known_lane._record_accuracy` | accuracy-sample-only, not a feed entry | + +| Constant | Value | Defined in | |---|---|---| | `_MN_ASSOC_MAX_DIST_KM` / `_MN_ASSOC_MAX_AGE_S` (identity step 2/3) | 6.0 km / 60.0 s | `services/tasks/solver.py` | | `_MN_ASSOC_DRIFT_KM_PER_S` / `_MN_ASSOC_MAX_DIST_CAP_KM` (step 3 only — the gate grows with the matched entry's age) | 0.13 km/s / 12.0 km | `services/tasks/solver.py` | | Supersession gate (`_supersession_match`) — the same age-scaled `_mn_assoc_gate_km` and `_MN_ASSOC_MAX_AGE_S` as step 3, applied to the solve's RAW position | 6.0 + 0.13·dt km, cap 12.0 / 60.0 s | `services/tasks/solver.py` | -| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py:77` | -| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py:63` | -| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py:66` | -| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py:37-43` | -| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py:131-139` | -| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py:49-50` | -| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py:167` | -| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py:206,213` | +| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py` | +| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py` | +| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py` | +| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py` | +| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py` | +| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py` (`_DEDUP_PROXIMITY_KM`, `_DEDUP_ALT_GATE_FT`) | +| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py` | +| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py` | + +--- + +## 7. Reading the pipeline from outside + +Three endpoints answer questions about the two lanes, and each has a shape +worth knowing before it is trusted. + +**`/api/test/mlat-history`** dumps solve records. Both lanes write their own +deque (`state.mlat_solve_history`, `state.mlat_solve_history_known`) and every +reader merges them. `?lane=dark|known|adsb|all` narrows the answer; +`?limit=` (default 1 000, max 5 000) is applied **per lane**, so a known-lane +burst can never push dark records out of the response — the flat cap that +preceded it left a 30 min request holding only the newest ~6 min of dark +records, which reads exactly like a quiet dark lane. `lane_counts` is +reported pre-cap so a truncated `records` list is legible. +`?kind=resolve_skips` dumps a different store entirely — see below. + +**`/api/test/solver-stats`** is the Solver Report panel's source. Its funnel, +error percentiles, ghosts, fragmentation, `contamination` and `resolve_skips` +are all the DARK lane; `lane_split` gives the per-lane record counts and +`known_lane` that lane's own numbers. + +| Block | Says | Watch for | +|---|---|---| +| `contamination` | Of the dark records that matched ground truth, how many carried a node that could not see the aircraft (`foreign_node_ids` on the record; verdict is the associator's own `_point_in_beam`, the same gate known-lane claiming uses) | `pct` is the live version of the offline ~60 % the cluster-splitting work exists to move. Records with no GT match, or no registered geometry for any contributing node, are **out of the denominator** — abstention, not innocence | +| `resolve_skips` | Candidates the re-solve suppression refused in this window, from `state.solver_resolve_skips_recent`, with the claims that blocked each one | `attempts_ratio` is all-lane skips over DARK attempts (live baseline ~2.4). The deque holds 500 entries against ~50 skips/min, so read `window_effective_minutes` before reading `total` as a window count | +| `counters.resolve_skips_dark` | Dark share of the since-boot skip counter | — | +| `counters.node_frames_rate_limited` | Frames `NODE_FRAME_MIN_INTERVAL_S` refused before the tracker saw them (Gate B in §2) | Not the same event as `/api/admin/metrics`' `frames_dropped`, which is `frame_queue` saturation and normally reads zero | + +A skip is deliberately **not** a solve-history record: skips outrun dark +records roughly two to one on the live fleet, so writing them into +`mlat_solve_history` would evict exactly the solves an investigation needs. +They are also not counted as attempts or rejects — a skipped candidate never +reached a solve. --- @@ -566,15 +603,15 @@ flowchart TD (`fragmentation`) and `superseded_keys` / `superseded_blocked` on each published `mlat_solve_history` record are how this is watched. - **Node-trust residuals are measure-only.** `node_bias.py` computes them but - nothing in the solver consumes them yet (`node_bias.py:33-40` docstring). + nothing in the solver consumes them yet (`node_bias.py` module docstring). - **`docs/pipeline.md` §3 is stale.** It predates the known lane and the process-pool inline fallback; this doc supersedes it for both topics. - **The bottom-up doppler gate is inert.** `doppler_gate_hz` in the dark lane's coarse pairing step is defined but the grid gate is delay-only in - practice (`libs/retina-analytics/.../association.py:884`). + practice (`association.compute_overlap_zone`'s `doppler_gate_hz`). - **Production runs with every mode flag off** except `KNOWN_LANE_MODE`, which is `binding` everywhere by code default and is set in no environment's `.env`. The in-repo statement of what each environment sets is - [`architecture.md:94-110`](architecture.md#feature-gates); the actual + [`architecture.md`](architecture.md#feature-gates); the actual values live in the gitignored `backend/.env` on each host, not in this repo. From 3a69d2dd0d1ecb95b3718a6c3cc3f966c5867097 Mon Sep 17 00:00:00 2001 From: jehanazad Date: Sat, 5 Sep 2026 07:08:01 +0000 Subject: [PATCH 06/25] Claim a re-solve slot on publication, not on admission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suppression rule reads "this aircraft is already on the map at this width, at every track it is built from". _claim_resolve_slot recorded the claim BEFORE the solve, so a candidate that never reached the map still made that statement — and never released it. Two consequences, both live: * A rejected candidate blacked out its own identical twin for the full 12 s. 24% of dark attempts are rejected, and the retry that would have published was suppressed by the failure. * Tracker track ids are shared across the association candidates of DIFFERENT aircraft (74 of 178 ids in a 6 min window appeared in solves of more than one ground-truth aircraft — the finding that forced _supersession_match's spatial guard in #290). So a contaminated superset that the gates sank also suppressed the clean subsets behind it, including its neighbour's only candidate. Measured on the test droplet: ~1,537 skips against 646 dark attempts per 30 min. The mechanism refused more than twice as many candidates as it solved, for aircraft it had put nowhere. _claim_resolve_slot splits into a pure _resolve_slot_covered (read-only, run before the solve, also returning the blocking claims for the skip record) and _record_resolve_slot, called only on the publish path with the POST-TRIM survivors — result["source_track_ids"], which _filter_s_in_to_nodes rebuilds from the surviving track_ids_by_node. A trimmed node's track is deliberately left unclaimed: it contributed nothing to the published position and was probably another aircraft's, so claiming it would suppress that aircraft on the strength of a measurement this solve threw away. The rule itself is unchanged: every track covered at no fewer nodes within _SOLVER_RESOLVE_INTERVAL_S, widest claim wins, same pruning sweep. No negative claim for rejects — measure first. The cost is that the check no longer claims under the same lock, so two workers can both solve duplicates that arrived together. That is one extra solve, arbitrated downstream by keying and supersession, against the starvation above. Co-Authored-By: Claude Fable 5.1 --- backend/core/state.py | 15 +-- backend/routes/test.py | 2 +- backend/services/tasks/solver.py | 108 ++++++++++++------- backend/tests/test_mlat_history.py | 7 +- backend/tests/test_solver_trimming.py | 73 +++++++++++++ backend/tests/test_solver_worker.py | 150 +++++++++++++++++++++----- docs/solverflow.md | 28 ++++- 7 files changed, 306 insertions(+), 77 deletions(-) diff --git a/backend/core/state.py b/backend/core/state.py index 246907c9..54c14ea4 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -544,12 +544,15 @@ def _adsb_for_seeding() -> dict[str, dict]: solver_stale_drops: int = 0 # Candidates dequeued and skipped because every single-node track they carry -# was already solved within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes (see -# solver.py's _claim_resolve_slot). Association is per-node and rate-limited -# per node, so one aircraft arrives as one candidate per node that can see it; -# this counts the copies that were never worth solving. High against -# solver_successes is normal and is the mechanism working — it is -# solver_stale_drops that means work was lost. +# was already PUBLISHED within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes +# (see solver.py's _resolve_slot_covered). Association is per-node and +# rate-limited per node, so one aircraft arrives as one candidate per node +# that can see it; this counts the copies that were never worth solving. High +# against solver_successes is normal and is the mechanism working — it is +# solver_stale_drops that means work was lost. Read it against +# solver_successes, not against attempts: while the claim was taken on +# ADMISSION rather than on publication, a rejected candidate blacked out every +# later one sharing a track id and this counter ran at ~2.4x attempts. solver_resolve_skips: int = 0 # The dark-lane share of the counter above, split out because the two lanes diff --git a/backend/routes/test.py b/backend/routes/test.py index 5909ff9d..c9431cd2 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -210,7 +210,7 @@ def _build_dashboard_data() -> bytes: # not the queue size. "stale_drops": state.solver_stale_drops, # Duplicate candidates for an aircraft already solved this - # window (see solver.py's _claim_resolve_slot). Read it + # window (see solver.py's _resolve_slot_covered). Read it # against stale_drops: skips are work correctly not done, # stale drops are work lost. "resolve_skips": state.solver_resolve_skips, diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 5faf7bc5..dbdf3ce8 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -986,6 +986,34 @@ def _supersession_match( # Sized against the map, not the association cadence: multinode_tracks expire # at 60 s, so refreshing an aircraft every 12 s leaves four solves' worth of # margin. 0 disables the suppression entirely. +# +# The claim is recorded ON PUBLICATION, not on admission, and from the +# POST-TRIM survivors. Claiming on admission made a candidate that never +# reached the map suppress every later candidate sharing any of its track ids +# for the full window — including other aircraft's, since tracker track ids +# are shared across the association candidates of different aircraft (74 of +# 178 ids in a 6 min live window appeared in solves of more than one +# ground-truth aircraft; the same finding that forced _supersession_match's +# spatial guard). A rejected candidate, or a contaminated superset that the +# gates sank, therefore blacked out the clean subsets behind it for 12 s and +# nothing was refreshed at all. Live that cost ~1 537 skips per 646 dark +# attempts per 30 min — more candidates suppressed than solved, by a factor +# of two. The rule this suppression is FOR is "an aircraft already on the map +# at this width does not need re-solving yet", and only a publication puts an +# aircraft on the map. +# +# Two consequences, both accepted deliberately: +# * the check no longer claims under the same lock, so two workers can now +# both solve duplicates of one aircraft that arrived together. The pair +# costs one extra solve and is resolved downstream by keying and +# supersession, which already handle exactly this; the alternative is the +# starvation above. +# * trimmed nodes' track ids are NOT claimed (_filter_s_in_to_nodes rebuilds +# track_ids from the surviving track_ids_by_node, so result's +# source_track_ids are the survivors). A node dropped for a bad residual +# was probably another aircraft's — claiming its track would suppress that +# aircraft's own candidate on the strength of a measurement this solve +# threw away. _SOLVER_RESOLVE_INTERVAL_S = float(os.getenv("SOLVER_RESOLVE_INTERVAL_S", "12")) _RECENT_SOLVES: dict[str, tuple[float, int]] = {} # track_id → (solved_at, n_nodes) _RECENT_SOLVES_LOCK = threading.Lock() @@ -1003,61 +1031,60 @@ def _sweep_recent_solves(now_s: float) -> None: del _RECENT_SOLVES[tid] -def _claim_resolve_slot(s_in, now_s: float) -> bool: - """False when this candidate re-solves tracks another candidate just took. +def _resolve_slot_covered(s_in, now_s: float) -> tuple[bool, list[dict]]: + """Is every track this candidate carries already ON THE MAP at this width? - Records the claim as a side effect, under one lock with the test, so two - workers cannot both admit the same aircraft's duplicates. An input with no - track provenance (detection-level, or an anchored input carrying none) is - always admitted — there is nothing to match it against. + Pure: it reads the claims and mutates nothing, so a candidate that is + admitted here and then rejected by the gate stack leaves no trace. The + claim is made afterwards by _record_resolve_slot, from the publish path + only — see the block comment above for why, and for what the loss of + atomic test-and-claim costs. + + Returns (covered, blocking). ``blocking`` is the claims that covered it, + for the skip record; it is empty whenever ``covered`` is False. An input + with no track provenance (detection-level, or an anchored input carrying + none) is never covered — there is nothing to match it against. """ if _SOLVER_RESOLVE_INTERVAL_S <= 0 or not isinstance(s_in, dict): - return True + return False, [] track_ids = s_in.get("track_ids") if not track_ids: - return True + return False, [] n_nodes = int(s_in.get("n_nodes") or 0) cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S + blocking: list[dict] = [] with _RECENT_SOLVES_LOCK: - covered = True for tid in track_ids: held = _RECENT_SOLVES.get(tid) if held is None or held[0] <= cutoff or held[1] < n_nodes: - covered = False - break - if covered: - return False - for tid in track_ids: - held = _RECENT_SOLVES.get(tid) - # Keep the widest claim of the window: a narrow candidate admitted - # after a wide one must not lower the bar the next copy is tested - # against. - held_nodes = held[1] if held is not None and held[0] > cutoff else 0 - _RECENT_SOLVES[tid] = (now_s, max(n_nodes, held_nodes)) - _sweep_recent_solves(now_s) - return True + return False, [] + blocking.append({"track_id": tid, "held_ts": round(held[0], 3), "held_n": held[1]}) + return True, blocking -def _resolve_slot_blockers(track_ids, now_s: float) -> list[dict]: - """The live claims covering ``track_ids``, for a skip record. +def _record_resolve_slot(track_ids, n_nodes: int, now_s: float) -> None: + """Record that ``track_ids`` are covered by a PUBLISHED solve at n_nodes. - Read-only, and taken after the refusal rather than during it: the check - itself must stay one atomic test-and-claim, and a skip is rare enough - (relative to the queue drain rate) that a second lock acquisition on that - path costs nothing. Any claim that moves between the two is a claim the - diagnosis would have wanted to name anyway. + Called from the publish path alone, with the post-trim survivors + (``result["source_track_ids"]``). Nothing else may call it: a claim is a + statement that this aircraft is on the map, and a rejected solve puts + nothing there. """ + if _SOLVER_RESOLVE_INTERVAL_S <= 0 or not track_ids: + return + n_nodes = int(n_nodes or 0) cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S - out: list[dict] = [] with _RECENT_SOLVES_LOCK: for tid in track_ids: held = _RECENT_SOLVES.get(tid) - if held is not None and held[0] > cutoff: - out.append({"track_id": tid, "held_ts": round(held[0], 3), "held_n": held[1]}) - return out + # Keep the widest claim of the window: a narrow publish after a + # wide one must not lower the bar the next copy is tested against. + held_nodes = held[1] if held is not None and held[0] > cutoff else 0 + _RECENT_SOLVES[tid] = (now_s, max(n_nodes, held_nodes)) + _sweep_recent_solves(now_s) -def _record_resolve_skip(s_in, now_s: float, blocking: list[dict] | None = None) -> None: +def _record_resolve_skip(s_in, now_s: float, blocking: list[dict]) -> None: """Count and remember one resolve-slot refusal. The counter alone could not answer the question the suppression rule @@ -1089,7 +1116,7 @@ def _record_resolve_skip(s_in, now_s: float, blocking: list[dict] | None = None) "lane": "dark" if dark else "adsb", "track_ids": track_ids, "n_nodes": int(s.get("n_nodes") or 0), - "blocking": _resolve_slot_blockers(track_ids, now_s) if blocking is None else blocking, + "blocking": blocking, "guess_lat": round(float(ig["lat"]), 6) if ig.get("lat") else None, "guess_lon": round(float(ig["lon"]), 6) if ig.get("lon") else None, } @@ -1850,8 +1877,9 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus # and a copy that queued before its twin was solved can only be recognised # once it reaches a worker. _now_s = time.time() - if not _claim_resolve_slot(s_in, _now_s): - _record_resolve_skip(s_in, _now_s) + _covered, _blocking = _resolve_slot_covered(s_in, _now_s) + if _covered: + _record_resolve_skip(s_in, _now_s, _blocking) return None n_nodes = s_in.get("n_nodes", 0) if isinstance(s_in, dict) else 0 consensus_meta: dict | None = None @@ -2417,6 +2445,12 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus archive_record = dict(result) archive_record["solve_ts_ms"] = int(time.time() * 1000) state.track_archive_buffer.append(archive_record) + # The re-solve claim, taken here and nowhere else: this aircraft is now + # on the map at this width, which is the only thing that makes a + # duplicate not worth solving. Survivors only — source_track_ids is + # rebuilt from the post-trim node set. Outside _MN_TRACKS_LOCK on + # purpose, so _RECENT_SOLVES_LOCK is never nested inside it. + _record_resolve_slot(result.get("source_track_ids"), result.get("n_nodes"), time.time()) _record_solve_history( "published", s_in, diff --git a/backend/tests/test_mlat_history.py b/backend/tests/test_mlat_history.py index e8e91bbe..a406d1b4 100644 --- a/backend/tests/test_mlat_history.py +++ b/backend/tests/test_mlat_history.py @@ -1031,9 +1031,10 @@ def _client(self): def _skip(self, track_ids=("a1", "b1"), n_nodes=3, **s_in): now = time.time() s = dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids), **s_in) - solver_mod._claim_resolve_slot(s, now) - assert solver_mod._claim_resolve_slot(dict(s), now) is False - solver_mod._record_resolve_skip(dict(s), now) + solver_mod._record_resolve_slot(list(track_ids), n_nodes, now) + covered, blocking = solver_mod._resolve_slot_covered(dict(s), now) + assert covered is True + solver_mod._record_resolve_skip(dict(s), now, blocking) def test_skip_records_the_blocking_claim(self): self._skip() diff --git a/backend/tests/test_solver_trimming.py b/backend/tests/test_solver_trimming.py index 93504a5a..0f529848 100644 --- a/backend/tests/test_solver_trimming.py +++ b/backend/tests/test_solver_trimming.py @@ -827,3 +827,76 @@ def solve_fn(_s_in, _cfgs): assert state.fov_shadow_agree == 1 assert state.fov_shadow_would_pass == 0 assert state.fov_shadow_would_reject == 0 + + +class TestTrimmedTracksAreNotClaimed(_TrimmingTestBase): + """A trimmed node's tracks must not take a re-solve claim. + + The claim says "this aircraft is on the map at this width". A node + dropped for a bad residual contributed nothing to the published position + and its track was probably a different aircraft's — claiming it would + suppress that aircraft's own candidate on the strength of a measurement + this solve threw away. + """ + + _FULL = ["n1", "n2", "n3", "n4", "bad"] + _TRIM = ["n1", "n2", "n3", "n4"] + + def test_the_dropped_nodes_track_is_left_unclaimed(self): + table = { + frozenset(self._FULL): _stub_result( + self._FULL, + rms_delay=8.0, + per_node={"n1": 0.5, "n2": 0.5, "n3": 0.5, "n4": 0.5, "bad": 12.0}, + ), + frozenset(self._TRIM): _stub_result( + self._TRIM, + rms_delay=0.8, + per_node={"n1": 0.3, "n2": 0.3, "n3": 0.3, "n4": 0.3}, + ), + } + s_in = _s_in( + self._FULL, + track_ids=["t1", "t2", "t3", "t4", "tbad"], + track_ids_by_node={ + "n1": ["t1"], + "n2": ["t2"], + "n3": ["t3"], + "n4": ["t4"], + "bad": ["tbad"], + }, + ) + result = self._run(s_in, _stub_solve_fn(table)) + assert result is not None and result["success"] + assert result["source_track_ids"] == ["t1", "t2", "t3", "t4"] + assert set(solver_mod._RECENT_SOLVES) == {"t1", "t2", "t3", "t4"} + + def test_a_candidate_built_on_the_dropped_track_still_runs(self): + """The other half of the same claim: whoever "tbad" really belongs to + keeps its slot.""" + table = { + frozenset(self._FULL): _stub_result( + self._FULL, + rms_delay=8.0, + per_node={"n1": 0.5, "n2": 0.5, "n3": 0.5, "n4": 0.5, "bad": 12.0}, + ), + frozenset(self._TRIM): _stub_result( + self._TRIM, + rms_delay=0.8, + per_node={"n1": 0.3, "n2": 0.3, "n3": 0.3, "n4": 0.3}, + ), + } + s_in = _s_in( + self._FULL, + track_ids=["t1", "t2", "t3", "t4", "tbad"], + track_ids_by_node={ + "n1": ["t1"], + "n2": ["t2"], + "n3": ["t3"], + "n4": ["t4"], + "bad": ["tbad"], + }, + ) + self._run(s_in, _stub_solve_fn(table)) + neighbour = {"n_nodes": 2, "track_ids": ["tbad", "tother"]} + assert solver_mod._resolve_slot_covered(neighbour, time.time())[0] is False diff --git a/backend/tests/test_solver_worker.py b/backend/tests/test_solver_worker.py index a4a6c8e9..6e3a2035 100644 --- a/backend/tests/test_solver_worker.py +++ b/backend/tests/test_solver_worker.py @@ -335,56 +335,153 @@ class TestResolveSuppression: emits its own candidate for it inside one association window. Solving all of them starves aircraft that have no solve at all — the queue ages out behind work whose result is superseded the moment it lands. + + The claim that suppresses a duplicate is taken on PUBLICATION + (_record_resolve_slot), not on admission: the rule is "this aircraft is + already on the map at this width", and only a publish puts it there. + _resolve_slot_covered is the pure test run before the solve. """ def _s_in(self, track_ids, n_nodes=2): return dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids)) - def test_a_second_copy_of_the_same_tracks_is_skipped(self): + def _covered(self, track_ids, n_nodes=2, now=None): + return solver_mod._resolve_slot_covered(self._s_in(track_ids, n_nodes), now or time.time())[0] + + def _publish(self, track_ids, n_nodes=2, now=None): + solver_mod._record_resolve_slot(list(track_ids), n_nodes, now or time.time()) + + def test_a_second_copy_of_a_published_candidate_is_skipped(self): + now = time.time() + assert self._covered(["a1", "b1"], now=now) is False + self._publish(["a1", "b1"], now=now) + assert self._covered(["a1", "b1"], now=now) is True + + def test_the_check_alone_claims_nothing(self): + """The whole point of the split: a candidate that is admitted and then + rejected by the gate stack must leave no trace.""" now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is False + assert self._covered(["a1", "b1"], now=now) is False + assert self._covered(["a1", "b1"], now=now) is False - def test_a_candidate_carrying_an_unsolved_track_runs(self): + def test_a_candidate_carrying_an_unpublished_track_runs(self): """An aircraft entering coverage must never be suppressed.""" now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b2"]), now) is True + self._publish(["a1", "b1"], now=now) + assert self._covered(["a1", "b2"], now=now) is False def test_a_wider_view_of_the_same_tracks_runs(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=2), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True + self._publish(["a1", "b1"], n_nodes=2, now=now) + assert self._covered(["a1", "b1"], n_nodes=5, now=now) is False def test_a_narrower_view_after_a_wider_one_is_skipped(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=2), now) is False + self._publish(["a1", "b1"], n_nodes=5, now=now) + assert self._covered(["a1", "b1"], n_nodes=2, now=now) is True - def test_a_narrow_admission_does_not_lower_the_bar(self): + def test_a_narrow_publish_does_not_lower_the_bar(self): """The window holds the widest claim, not the most recent one.""" now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b2"], n_nodes=2), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=3), now) is False + self._publish(["a1", "b1"], n_nodes=5, now=now) + self._publish(["a1", "b2"], n_nodes=2, now=now) + assert self._covered(["a1", "b1"], n_nodes=3, now=now) is True def test_claims_expire(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True + self._publish(["a1", "b1"], now=now) later = now + solver_mod._SOLVER_RESOLVE_INTERVAL_S + 1.0 - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), later) is True + assert self._covered(["a1", "b1"], now=later) is False def test_an_input_without_track_provenance_always_runs(self): """Detection-level inputs carry no track ids — nothing to match on.""" now = time.time() - assert solver_mod._claim_resolve_slot({"n_nodes": 2}, now) is True - assert solver_mod._claim_resolve_slot({"n_nodes": 2}, now) is True + assert solver_mod._resolve_slot_covered({"n_nodes": 2}, now)[0] is False + solver_mod._record_resolve_slot(None, 2, now) + assert solver_mod._resolve_slot_covered({"n_nodes": 2}, now)[0] is False def test_zero_interval_disables_suppression(self, monkeypatch): monkeypatch.setattr(solver_mod, "_SOLVER_RESOLVE_INTERVAL_S", 0.0) now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True + self._publish(["a1", "b1"], now=now) + assert self._covered(["a1", "b1"], now=now) is False + + def test_the_check_names_every_blocking_claim(self): + now = time.time() + self._publish(["a1", "b1"], n_nodes=4, now=now) + covered, blocking = solver_mod._resolve_slot_covered(self._s_in(["a1", "b1"], n_nodes=3), now) + assert covered is True + assert {b["track_id"]: b["held_n"] for b in blocking} == {"a1": 4, "b1": 4} + + def test_an_admitted_candidate_reports_no_blockers(self): + covered, blocking = solver_mod._resolve_slot_covered(self._s_in(["a1", "b1"]), time.time()) + assert (covered, blocking) == (False, []) + + def _solve_fn(self, calls, rms_delay=0.5, lat=37.5, lon=-122.1): + def fn(s_in, cfgs): + calls.append(s_in) + return { + "success": True, + "lat": lat, + "lon": lon, + "alt_m": 9000.0, + "rms_delay": rms_delay, + "rms_doppler": 5.0, + "timestamp_ms": int(time.time() * 1000), + "contributing_node_ids": ["n1", "n2"], + "n_nodes": s_in.get("n_nodes", 2), + } + + return fn + + def test_a_rejected_candidate_does_not_block_an_identical_twin(self, monkeypatch): + """The bug this split exists to fix. A candidate the gate stack sank + put nothing on the map, so the next copy of the same aircraft is its + first real chance — and used to be blacked out for the full 12 s.""" + _reset_state() + monkeypatch.setattr(state, "node_analytics", _StubAnalytics()) + calls: list = [] + s_in = self._s_in(["a1", "b1"]) + + # rms_delay past the gate: solves, then rejected, publishes nothing. + solver_mod._process_solver_item((dict(s_in), {}, time.time()), self._solve_fn(calls, rms_delay=10.0)) + assert state.solver_fail_rms_delay == 1 + assert not state.multinode_tracks + + solver_mod._process_solver_item((dict(s_in), {}, time.time()), self._solve_fn(calls)) + assert len(calls) == 2, "the twin must not be suppressed by a reject" + assert state.multinode_tracks + assert state.solver_resolve_skips == 0 + + def test_a_subset_for_another_aircraft_survives_a_rejected_superset(self, monkeypatch): + """Tracker track ids are shared across the candidates of DIFFERENT + aircraft, so a contaminated superset that the gates sank used to take + every clean subset behind it down with it — including its neighbour's + only candidate.""" + _reset_state() + monkeypatch.setattr(state, "node_analytics", _StubAnalytics()) + calls: list = [] + superset = self._s_in(["a1", "b1", "c1"], n_nodes=3) + solver_mod._process_solver_item((superset, {}, time.time()), self._solve_fn(calls, rms_delay=10.0)) + assert not state.multinode_tracks + + # The neighbour: fewer nodes, sharing one contaminated track id. + subset = self._s_in(["a1", "b1"], n_nodes=2) + solver_mod._process_solver_item((subset, {}, time.time()), self._solve_fn(calls)) + assert len(calls) == 2 + assert state.multinode_tracks + + def test_only_the_published_width_is_claimed(self, monkeypatch): + """A publish claims at the width it published, so a later narrower + copy is suppressed and a wider one still runs.""" + _reset_state() + monkeypatch.setattr(state, "node_analytics", _StubAnalytics()) + calls: list = [] + solver_mod._process_solver_item((self._s_in(["a1", "b1"], n_nodes=3), {}, time.time()), self._solve_fn(calls)) + assert state.multinode_tracks + now = time.time() + assert self._covered(["a1", "b1"], n_nodes=2, now=now) is True + assert self._covered(["a1", "b1"], n_nodes=4, now=now) is False def test_a_skipped_item_never_reaches_the_solver(self, monkeypatch): _reset_state() @@ -408,6 +505,8 @@ def solve_fn(s_in, cfgs): solver_mod._process_solver_item((s_in, {}, time.time()), solve_fn) assert len(solve_calls) == 1 assert state.solver_successes == 1 + # The first item PUBLISHED, which is what makes the second redundant. + assert state.multinode_tracks assert solver_mod._process_solver_item((dict(s_in), {}, time.time()), solve_fn) is None assert len(solve_calls) == 1, "the duplicate must not be solved" @@ -425,9 +524,10 @@ def test_a_skip_is_recorded_with_the_claim_that_blocked_it(self): state.solver_resolve_skips_recent.clear() now = time.time() s_in = dict(self._s_in(["a1", "b1"], n_nodes=4), initial_guess={"lat": 35.0, "lon": -82.0}) - assert solver_mod._claim_resolve_slot(dict(s_in), now) is True - assert solver_mod._claim_resolve_slot(dict(s_in), now) is False - solver_mod._record_resolve_skip(dict(s_in), now) + solver_mod._record_resolve_slot(["a1", "b1"], 4, now) + covered, blocking = solver_mod._resolve_slot_covered(dict(s_in), now) + assert covered is True + solver_mod._record_resolve_skip(dict(s_in), now, blocking) assert state.solver_resolve_skips == 1 assert state.solver_resolve_skips_dark == 1 @@ -444,7 +544,7 @@ def test_a_tagged_candidate_is_counted_but_not_as_dark(self): state.solver_resolve_skips_recent.clear() now = time.time() s_in = dict(self._s_in(["a1"], n_nodes=3), adsb_hex="abc123") - solver_mod._record_resolve_skip(s_in, now) + solver_mod._record_resolve_skip(s_in, now, []) assert state.solver_resolve_skips == 1 assert state.solver_resolve_skips_dark == 0 assert state.solver_resolve_skips_recent[0]["lane"] == "adsb" @@ -455,7 +555,7 @@ def test_skips_never_enter_the_solve_history(self): _reset_state() state.mlat_solve_history.clear() s_in = self._s_in(["a1", "b1"]) - solver_mod._record_resolve_skip(s_in, time.time()) + solver_mod._record_resolve_skip(s_in, time.time(), []) assert not state.mlat_solve_history assert not state.mlat_solve_history_known diff --git a/docs/solverflow.md b/docs/solverflow.md index c80c4673..d22fc152 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -382,13 +382,30 @@ The centerpiece: every candidate from either lane, once dequeued from any gate stops the chain, bumps a counter, and (from 6.5 onward) writes a named record to solve history. +**6.2 claims on publication, not on admission.** The suppression rule is +"this aircraft is already on the map at this width, at every track it is +built from" — so `_resolve_slot_covered` only *reads* the claims, and +`_record_resolve_slot` takes them from the publish path, with the +**post-trim survivors** (`result["source_track_ids"]`, rebuilt from the +surviving `track_ids_by_node`). Claiming on admission instead meant a +candidate that never reached the map still blacked out every later candidate +sharing any of its track ids for the full 12 s — including *other aircraft's*, +since tracker track ids are shared across the association candidates of +different aircraft (the same finding behind `_supersession_match`'s spatial +guard; see Caveats). Live that ran at ~1 537 skips per 646 dark attempts per +30 min: more candidates suppressed than solved, by a factor of two. The price +of the split is that the check no longer claims under the same lock, so two +workers can now both solve duplicates that arrived together; that costs one +extra solve and is resolved downstream by keying and supersession, which +handle exactly this case already. + ```mermaid flowchart TD deq["Dequeue (s_in, node_cfgs, enqueued_at)"] deq --> g61{"6.1 Staleness
age_s > _SOLVER_MAX_QUEUE_AGE_S 45.0s?"} g61 -->|"yes"| f61["solver_stale_drops
(no history record)"]:::inert - g61 -->|"no"| g62{"6.2 Re-solve suppression
_claim_resolve_slot False?"} - g62 -->|"yes"| f62["solver_resolve_skips"]:::inert + g61 -->|"no"| g62{"6.2 Re-solve suppression
_resolve_slot_covered (pure)?"} + g62 -->|"yes"| f62["solver_resolve_skips (+_dark)
+ skip record with blockers"]:::inert g62 -->|"no"| g63["6.3 Solve dispatch:
no guess -> bare solve_fn;
n>=3 -> consensus? then
_solve_best_altitude (sweep);
n=2 -> _solve_best_altitude_n2
(single altitude)"] g63 -->|"exception"| f63["solver_failures +
solver_fail_exception,
result=None"]:::inert g63 --> g64{"6.4 Trim & resolve (recovery):
guess AND n>=4 AND
rms_delay > 3.0us?"} @@ -464,7 +481,7 @@ flowchart TD | Constant | Value | Defined in | |---|---|---| | `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py` | -| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py` (`_SOLVER_RESOLVE_INTERVAL_S`) | +| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py` (`_SOLVER_RESOLVE_INTERVAL_S`, `_resolve_slot_covered`, `_record_resolve_slot`) | | `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py` | | `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py` (`_SOLVER_RMS_DELAY_MAX_US`) | | `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py` | @@ -509,7 +526,8 @@ flowchart TD popped --> store["state.multinode_tracks[key] = result"] blocked --> store store --> archive["track-archive buffer append"] - archive --> histpub["_record_solve_history: published"] + archive --> claimslot["_record_resolve_slot:
claim the POST-TRIM survivors
for _SOLVER_RESOLVE_INTERVAL_S"] + claimslot --> histpub["_record_solve_history: published"] histpub --> feed["build_combined_aircraft_json
(1 Hz flush)"] feed --> gN2{"n=2 display gate:
solve_count < MN_N2_MIN_SOLVES 2?"} @@ -573,7 +591,7 @@ are all the DARK lane; `lane_split` gives the per-lane record counts and | Block | Says | Watch for | |---|---|---| | `contamination` | Of the dark records that matched ground truth, how many carried a node that could not see the aircraft (`foreign_node_ids` on the record; verdict is the associator's own `_point_in_beam`, the same gate known-lane claiming uses) | `pct` is the live version of the offline ~60 % the cluster-splitting work exists to move. Records with no GT match, or no registered geometry for any contributing node, are **out of the denominator** — abstention, not innocence | -| `resolve_skips` | Candidates the re-solve suppression refused in this window, from `state.solver_resolve_skips_recent`, with the claims that blocked each one | `attempts_ratio` is all-lane skips over DARK attempts (live baseline ~2.4). The deque holds 500 entries against ~50 skips/min, so read `window_effective_minutes` before reading `total` as a window count | +| `resolve_skips` | Candidates the re-solve suppression refused in this window, from `state.solver_resolve_skips_recent`, with the claims that blocked each one | `attempts_ratio` is all-lane skips over DARK attempts. It read ~2.4 while 6.2 claimed on admission; with the claim on publication it should sit at or below 0.5. The deque holds 500 entries against a live rate of tens per minute, so read `window_effective_minutes` before reading `total` as a window count | | `counters.resolve_skips_dark` | Dark share of the since-boot skip counter | — | | `counters.node_frames_rate_limited` | Frames `NODE_FRAME_MIN_INTERVAL_S` refused before the tracker saw them (Gate B in §2) | Not the same event as `/api/admin/metrics`' `frames_dropped`, which is `frame_queue` saturation and normally reads zero | From 964029ec5ece8a513839c4a588dc1a890bdc9cac Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 07:09:23 +0000 Subject: [PATCH 07/25] Score contamination at the publish point as well as the candidate point The candidate-level rate has a denominator association itself moves. Splitting one contaminated cluster into two clean sub-clusters plus the false pairing it was hiding emits three inputs where there was one, and the false pairing on its own scores as 100% contaminated -- so the rate rises while what reaches the map gets cleaner. Measured on the 50-node scatter scene: 20.3% -> 25.7% of candidates, with the ghost rate by solve falling 3.2% -> 2.4% over the same change. So the same score is now also taken at the publish point, over solves that cleared every gate and bound to a real aircraft. That is the population the live audit sampled (45% of published dark solves carried a foreign node), and its denominator is fixed by the aircraft in the sky rather than by how many hypotheses association chose to emit. Co-Authored-By: Claude Fable 5.1 --- backend/scripts/association_bench.py | 66 +++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py index 9e5792db..eac7048a 100644 --- a/backend/scripts/association_bench.py +++ b/backend/scripts/association_bench.py @@ -322,7 +322,7 @@ def _index_detection_truth(det_truth: dict, geo, node_id: str, frame: dict, airc det_truth[key] = oid if (prev is None or prev == oid) else _TRUTH_AMBIGUOUS -def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) -> None: +def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) -> int | None: """Count the nodes in one solver input that are not looking at its aircraft. The input's own aircraft is the plurality of its measurements' true @@ -335,16 +335,23 @@ def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) A node is foreign when its measurement belongs to a different aircraft, or to no aircraft at all (clutter that survived the tracker's M-of-N and the delay grid). Ambiguous attributions are counted in neither. + + Returns the foreign-node count, so the caller can score the same input + again at the publish point (see the PUBLISHED counters on Result: the + candidate-level rate has a denominator the association layer itself moves, + and a change that emits more, cleaner candidates reads as a regression on + it while being an improvement on what actually reaches the map). None + when nothing could be attributed at all. """ oids = [ det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"]))) for m in s_in.get("measurements") or [] ] if not oids: - return + return None counts = Counter(o for o in oids if o is not None and o != _TRUTH_AMBIGUOUS) if not counts: - return + return None top_n = max(counts.values()) contenders = sorted(o for o, c in counts.items() if c == top_n) if len(contenders) > 1: @@ -366,6 +373,7 @@ def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) res.foreign_nodes += foreign if foreign: res.inputs_contaminated += 1 + return foreign def _strip_adsb(frame: dict) -> dict: @@ -667,6 +675,16 @@ class Result: inputs_contaminated: int = 0 foreign_nodes: int = 0 input_nodes: int = 0 + # The same score restricted to inputs that cleared every gate and bound + # to a real aircraft — what actually reached the map. Reported next to + # the candidate rate because the two answer different questions: the + # candidate rate's denominator is the number of candidates association + # chooses to emit, so splitting one contaminated cluster into several + # clean ones plus the false pairing it was hiding *raises* it while + # lowering this one. + published_inputs: int = 0 + published_contaminated: int = 0 + published_foreign_nodes: int = 0 # Stone-Soup GOSPA/SIAP scalars for this one run (--ss-metrics), or None # when it was off, stonesoup wasn't available, or the recorder had @@ -775,6 +793,9 @@ def keys_per_object(self): "inputs_contaminated", "foreign_nodes", "input_nodes", + "published_inputs", + "published_contaminated", + "published_foreign_nodes", "cluster_splits", ) _EXTEND_FIELDS = ( @@ -835,6 +856,14 @@ def contaminated_inputs_pct(self): def foreign_nodes_per_input(self): return self.foreign_nodes / self.inputs_scored if self.inputs_scored else 0.0 + @property + def published_contaminated_pct(self): + return 100.0 * self.published_contaminated / self.published_inputs if self.published_inputs else 0.0 + + @property + def published_foreign_per_solve(self): + return self.published_foreign_nodes / self.published_inputs if self.published_inputs else 0.0 + def build_scene( seed: int, @@ -1165,11 +1194,13 @@ def _geo_key(nid): res.cluster_sizes[(_k, len(s_in.get("track_ids") or []))] += 1 if s_in.get("n_nodes", 0) < 2: continue + _foreign = None if mode == "track": # Scored here, ahead of the solve and every gate below: # this measures what association emitted, which is the - # thing the cluster-merge rework changes. - _score_contamination(res, s_in, det_truth, truth) + # thing the cluster-merge rework changes. Re-scored at + # the publish point further down, on the same number. + _foreign = _score_contamination(res, s_in, det_truth, truth) try: _t0 = time.perf_counter() out = solve_fn(s_in, node_cfgs) @@ -1273,6 +1304,17 @@ def _geo_key(nid): (_keys_real if d <= MATCH_KM else _keys_ghost).add(_key) if d <= MATCH_KM: res.matched += 1 + if _foreign is not None: + # Same input, scored again now that every gate has + # accepted it and it has bound to a real aircraft: + # this is the population the live audit sampled (45% + # of published dark solves carried a foreign node), + # and unlike the candidate rate its denominator is + # not something association can inflate. + res.published_inputs += 1 + res.published_foreign_nodes += _foreign + if _foreign: + res.published_contaminated += 1 res.errors_km.append(d) res.n_nodes_matched[nn] += 1 # Broken out because the whole dual-site hypothesis is @@ -1506,6 +1548,12 @@ def report(label: str, r: Result, truth_max_kt: float | None = None): f"foreign nodes/input {r.foreign_nodes_per_input:.2f}" f" ({r.foreign_nodes}/{r.input_nodes} nodes)" ) + if r.published_inputs: + print( + f" CONTAMINATION (published): {r.published_contaminated}/{r.published_inputs} matched solves" + f" -> {r.published_contaminated_pct:5.1f}% " + f"foreign nodes/solve {r.published_foreign_per_solve:.2f}" + ) if r.cluster_splits: print(f" cluster splits (same-node track conflict): {r.cluster_splits}") if r.gate_gated: @@ -1846,7 +1894,7 @@ def main(): solve_fn = _ESTIMATORS[estimator_name] rates, solve_rates, reals, fakes, speed_errs = [], [], [], [], [] n2_rates = [] - contam_rates, foreign_rates, med_errs = [], [], [] + contam_rates, foreign_rates, med_errs, pub_contam_rates = [], [], [], [] agg = Result() last = None for k in range(args.repeat): @@ -1896,6 +1944,7 @@ def main(): speed_errs.append(statistics.median(last.speed_err_ms)) contam_rates.append(last.contaminated_inputs_pct) foreign_rates.append(last.foreign_nodes_per_input) + pub_contam_rates.append(last.published_contaminated_pct) med_errs.append(statistics.median(last.errors_km) if last.errors_km else float("nan")) label = f"assoc_interval={interval:g}s" if chi2_max is not None: @@ -1930,6 +1979,11 @@ def main(): f"{', '.join(f'{x:.2f}' for x in foreign_rates)}" f" mean {statistics.mean(foreign_rates):.2f}" ) + print( + f" published contaminated per seed: " + f"{', '.join(f'{x:.0f}%' for x in pub_contam_rates)}" + f" mean {statistics.mean(pub_contam_rates):.1f}%" + ) print( f" median matched error per seed: " f"{', '.join(f'{x:.2f}' for x in med_errs)} km" From ed1a023b229071997c233d989766478d13080434 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 07:12:20 +0000 Subject: [PATCH 08/25] Report superseded and cluster_splits as live association counters /api/radar/association/status filed track_pairs_superseded under "track_pairs_inline_only", documented as permanently zero in production because the only exclusivity stage ran on a chi2 that cv_fit=None never computes. The deferred path now has an exclusivity stage of its own, so the counter moves on a live fleet and that grouping would be a lie. Moved to the live block alongside the new cluster_splits. Co-Authored-By: Claude Fable 5.1 --- backend/routes/analytics.py | 14 ++++++++++--- backend/tests/test_analytics_routes.py | 29 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/backend/routes/analytics.py b/backend/routes/analytics.py index 07d83a9c..8cb8c836 100644 --- a/backend/routes/analytics.py +++ b/backend/routes/analytics.py @@ -131,20 +131,28 @@ async def association_status(): # Track-pairing outcomes since boot. gated is everything past the # coarse delay grid; unfitted counts the pairings handed to the solver # worker (which runs the fit and the n=2 gate); deferred counts rounds - # a budget cut short. Those three are the live production surface. + # a budget cut short; superseded counts pairings dropped because a + # better-ranked one claiming the same track implied a velocity theirs + # contradicts; cluster_splits counts position clusters that held two + # tracks of one node and were emitted as one solver input each. All + # five are the live production surface. "track_pairs": { "gated": getattr(_a, "track_pairs_gated", 0), "unfitted": getattr(_a, "track_pairs_unfitted", 0), "deferred": getattr(_a, "track_pairs_deferred", 0), + "superseded": getattr(_a, "track_pairs_superseded", 0), + "cluster_splits": getattr(_a, "cluster_splits", 0), }, # Inline-fit counters — permanently zero in production BY DESIGN # (state.py builds the associator with cv_fit=None; only the offline - # bench's inline mode exercises stage-2 selection). Split out so + # bench's inline mode exercises the chi2 threshold). Split out so # nobody reads a structural zero as "no rejections happening". + # superseded used to live here too, and no longer can: the deferred + # path now has an exclusivity stage of its own, so the counter moves + # in production. "track_pairs_inline_only": { "accepted": getattr(_a, "track_pairs_accepted", 0), "rejected": getattr(_a, "track_pairs_rejected", 0), - "superseded": getattr(_a, "track_pairs_superseded", 0), }, # Top-down claiming (ASSOC_CLAIM_MODE) since boot. rounds/matched/ # conflicts/anchored_inputs are all live in shadow too — _claim_round diff --git a/backend/tests/test_analytics_routes.py b/backend/tests/test_analytics_routes.py index 22a821e0..1778cdf8 100644 --- a/backend/tests/test_analytics_routes.py +++ b/backend/tests/test_analytics_routes.py @@ -129,6 +129,35 @@ def test_status_returns_expected_fields(self, client): assert "overlap_zones" in body assert "overlaps" in body + def test_track_pairs_block_reports_the_live_counters(self, client): + """superseded and cluster_splits belong to the LIVE block, not the + inline-only one. + + They used to be structurally zero in production because the only + exclusivity stage ran on a chi2 nothing computes with cv_fit=None. + The deferred path now prunes on implied-velocity conflict and splits + clusters that hold two tracks of one node, so both counters move on a + live fleet and reading them as "inline only" would be wrong. + """ + _a = state.node_associator + _a.track_pairs_superseded += 5 + _a.cluster_splits += 3 + try: + body = client.get("/api/radar/association/status").json() + assert body["track_pairs"].keys() == { + "gated", + "unfitted", + "deferred", + "superseded", + "cluster_splits", + } + assert body["track_pairs"]["superseded"] == 5 + assert body["track_pairs"]["cluster_splits"] == 3 + assert body["track_pairs_inline_only"].keys() == {"accepted", "rejected"} + finally: + _a.track_pairs_superseded -= 5 + _a.cluster_splits -= 3 + def test_status_includes_claiming_block(self, client): """Top-down claiming (ASSOC_CLAIM_MODE) since boot — off by default in tests, so this pins the shape rather than any particular mode.""" From ca748c39e46c7b65af429b1ed3851c7b34704a6a Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 07:46:33 +0000 Subject: [PATCH 09/25] Bump retina-analytics: bound the cluster diameter Picks up offworldlabs/retina-analytics@7e5414a, which extends the cluster partition from same-node conflicts only to every group, bounding each emitted solver input at the merge distance. Foreign nodes per published solve 0.80 -> 0.32 on the 15-node ring bench scene. Co-Authored-By: Claude Fable 5.1 --- libs/retina-analytics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/retina-analytics b/libs/retina-analytics index 4ff48fca..7e5414ac 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 4ff48fcaa3abdaad1d42eb576f8573342addad4e +Subproject commit 7e5414ac1671bb66c160a425363b1d19c889eea4 From 3702fec563e601ab56a4d8f077e98c8205da7bf4 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 08:22:57 +0000 Subject: [PATCH 10/25] Exclude stale coasting tracks and align measurement epochs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent corrections to the same failure: legitimately in-cone nodes being trimmed out of dark solves because their measurement did not describe the moment the solve claims to. Stale-track exclusion (TRACK_MAX_STALE_S, default 3.0 s). A COASTING track survives N_DELETE=10 frames past its last association, and at the fleet's 0.74-1 Hz per-node cadence that is up to ~13 s of dead reckoning. confirmed_track_views hands association the track's newest REAL sample and association hands the solver that sample as the node's current measurement, so an aircraft that has flown out of a node's beam keeps contributing seconds-old delay to n>=3 solves. Measured on the test droplet: 230 out-of-cone nodes reached published dark solves in 20 min, median 4 deg outside the beam edge (p90 22 deg) and 1.8 km beyond max range. The freshness test reads the newest entry from get_recent_detections, which returns only ASSOCIATED samples, so it is the last real detection's time rather than a coast count — and it is compared against the frame timestamp being processed, never wall clock, so replays and backfills are unaffected. Epoch alignment (SOLVER_EPOCH_ALIGN, default on). The solver's residual model evaluates every measurement against one target state; nodes sample at independent phases, so it is fitting a set that spans up to a frame interval. align_measurement_epochs dead-reckons each delay onto the newest sample's time along its own measured Doppler, using d(delay_us)/dt = -doppler_hz * 1e6 / fc_hz — the rate implied by the geolocator's own residual model, in which positive Doppler is a closing target whose delay is decreasing. All-or-nothing per input: a partially aligned set just relocates the error, so a missing t_s, doppler_hz or fc_hz skips the input and counts solver_epoch_align_skipped. Both flags exist to be turned off live rather than rolled back. The known lane reuses the same helper: it does NOT have one epoch either (_CLAIM_SPREAD_S admits claims up to 5 s apart, ~1.5 km of motion at 300 m/s), and it is the lane whose residual is the fleet's accuracy measurement. Pins retina-analytics feat/measurement-epochs (PR #27), which carries t_s on each solver-input measurement. Co-Authored-By: Claude Fable 5.1 --- backend/config/constants.py | 13 ++++ backend/core/state.py | 30 ++++++++- backend/routes/admin.py | 2 + backend/routes/test.py | 9 +++ backend/scripts/association_bench.py | 2 +- backend/services/frame_processor.py | 33 ++++++++-- backend/services/tasks/known_lane.py | 19 +++++- backend/services/tasks/solver.py | 96 ++++++++++++++++++++++++++++ libs/retina-analytics | 2 +- 9 files changed, 197 insertions(+), 9 deletions(-) diff --git a/backend/config/constants.py b/backend/config/constants.py index ea788abf..2d1cd19c 100644 --- a/backend/config/constants.py +++ b/backend/config/constants.py @@ -84,6 +84,19 @@ def as_num(v) -> float: N2_CONFIRM_MIN_EPOCHS = 4 # Floor on samples; span is the real gate N2_TRACK_HISTORY_MAX = 20 # Per-node track samples fed to the fit +# How old a track's newest REAL detection may be before the track stops being +# offered to association (see services/frame_processor.confirmed_track_views). +# A COASTING track is kept alive for N_DELETE=10 frames after its last +# association, and at the fleet's 0.74-1 Hz per-node cadence that is up to ~13 s +# of dead reckoning. confirmed_track_views hands association the track's last +# real sample, and association hands the solver that sample as if it were +# current — so an aircraft that has flown out of a node's beam keeps +# contributing a seconds-old delay to n>=3 solves. Measured on the test +# droplet: 230 out of-cone nodes survived into published dark solves in 20 min, +# median 4 deg outside the beam edge (p90 22 deg) and 1.8 km beyond max range, +# and they are the nodes the rms trim then throws away. 0 disables the filter. +TRACK_MAX_STALE_S = float(os.getenv("TRACK_MAX_STALE_S", "3.0")) + # A 2-node track needs this many solves before it renders a plane; 1 # disables the gate. One-shot n=2 solves were the dominant ghost source. MN_N2_MIN_SOLVES = int(os.getenv("MN_N2_MIN_SOLVES", "2")) diff --git a/backend/core/state.py b/backend/core/state.py index 634aa212..f9e4b6f3 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -74,6 +74,18 @@ if KNOWN_LANE_MODE not in ("off", "shadow", "binding"): KNOWN_LANE_MODE = "shadow" +# Measurement epoch alignment (see services/tasks/solver.align_measurement_epochs). +# on/off rather than the off/shadow/active triple its neighbours use: there is +# nothing to shadow — the correction is a closed-form dead-reckoning of each +# delay along its own measured Doppler, so a dry run would produce the same +# number the acting run applies and observe nothing extra. Default "on", +# because leaving it off is the bug: nodes sample at independent phases and the +# solver treats their measurements as simultaneous, so a 250 m/s target charges +# up to ~1 us of delay error per second of skew (measured ~0.3 us rms at 2 s +# skew on the fleet) straight to the 3 us rms gate. The flag exists so the +# alignment can be turned off live without a rollback if it ever misbehaves. +SOLVER_EPOCH_ALIGN = os.getenv("SOLVER_EPOCH_ALIGN", "on").strip().lower() != "off" + node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE) @@ -526,6 +538,21 @@ def _adsb_for_seeding() -> dict[str, dict]: # fleet's trigger rate — constraints are then converging slower than the # coverage they follow, which no rebuild counter can show. coverage_rebuild_backlog: int = 0 + +# Confirmed tracks withheld from association because their newest REAL +# detection was older than TRACK_MAX_STALE_S at the frame being processed — +# see services/frame_processor.confirmed_track_views. These are aircraft that +# have left a node's beam and whose track is dead-reckoning toward deletion; +# their last real sample used to reach the solver as a current measurement. +tracks_stale_skipped: int = 0 + +# Solver inputs whose measurements could not be aligned to a common epoch +# because at least one lacked t_s, doppler_hz, or a node config with fc_hz — +# see services/tasks/solver.align_measurement_epochs. Counted only when +# SOLVER_EPOCH_ALIGN is on; a nonzero value against solver_successes says how +# much of the fleet is still emitting untimed measurements. +solver_epoch_align_skipped: int = 0 + solver_queue_drops: int = 0 # Queue items discarded unsolved because they aged past _SOLVER_MAX_QUEUE_AGE_S # waiting for a worker. Was only a DEBUG log, which staging does not emit — @@ -750,7 +777,7 @@ def _reset_for_tests() -> None: global known_claims_made, known_claim_contentions, known_claims_bound global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes - global coverage_rebuild_backlog + global coverage_rebuild_backlog, tracks_stale_skipped, solver_epoch_align_skipped global solver_queue_drops, solver_stale_drops, solver_resolve_skips global mn_superseded, mn_superseded_blocked, solver_trimmed global solver_consensus_selected, solver_consensus_filtered @@ -840,6 +867,7 @@ def _reset_for_tests() -> None: known_claims_world_rejects = 0 coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0 coverage_rebuild_backlog = 0 + tracks_stale_skipped = solver_epoch_align_skipped = 0 solver_stale_drops = 0 solver_resolve_skips = 0 mn_superseded = mn_superseded_blocked = 0 diff --git a/backend/routes/admin.py b/backend/routes/admin.py index 2d0eca6d..4d11fb0b 100644 --- a/backend/routes/admin.py +++ b/backend/routes/admin.py @@ -590,6 +590,8 @@ async def system_metrics(_user=Depends(require_admin)): "solver_queue_drops": state.solver_queue_drops, "solver_stale_drops": state.solver_stale_drops, "solver_resolve_skips": state.solver_resolve_skips, + "tracks_stale_skipped": state.tracks_stale_skipped, + "solver_epoch_align_skipped": state.solver_epoch_align_skipped, "mn_superseded": state.mn_superseded, "solver_trimmed": state.solver_trimmed, "solver_last_latency_s": round(state.solver_last_latency_s, 3), diff --git a/backend/routes/test.py b/backend/routes/test.py index e4018b36..afc57dc7 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -214,6 +214,13 @@ def _build_dashboard_data() -> bytes: # against stale_drops: skips are work correctly not done, # stale drops are work lost. "resolve_skips": state.solver_resolve_skips, + # Confirmed tracks withheld from association because their + # newest real detection had aged past TRACK_MAX_STALE_S, and + # solver inputs the epoch alignment could not correct because a + # measurement carried no sample time (see frame_processor's + # confirmed_track_views and solver's align_measurement_epochs). + "tracks_stale_skipped": state.tracks_stale_skipped, + "epoch_align_skipped": state.solver_epoch_align_skipped, # Multinode entries replaced because a later solve consumed # the same source tracks under a new key (fragmented re-solve). "mn_superseded": state.mn_superseded, @@ -1192,6 +1199,8 @@ def _solver_window_stats(minutes: float) -> dict: "solver_trimmed": state.solver_trimmed, "stale_drops": state.solver_stale_drops, "resolve_skips": state.solver_resolve_skips, + "tracks_stale_skipped": state.tracks_stale_skipped, + "epoch_align_skipped": state.solver_epoch_align_skipped, "queue_drops": state.solver_queue_drops, "worker_errors": state.solver_worker_errors, "vel_untrusted_published": state.solver_vel_untrusted_published, diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py index e08999bb..acc7196a 100644 --- a/backend/scripts/association_bench.py +++ b/backend/scripts/association_bench.py @@ -978,7 +978,7 @@ def _geo_key(nid): # frame is what association pairs against — but only let a node # *trigger* a round on its own cadence. if mode == "track": - assoc._pending_tracks[nid] = confirmed_track_views(trackers[nid], history_n) + assoc._pending_tracks[nid] = confirmed_track_views(trackers[nid], history_n, ts_ms) else: assoc._pending_frames[nid] = frame if (t - last_assoc.get(nid, -1e9)) < assoc_interval: diff --git a/backend/services/frame_processor.py b/backend/services/frame_processor.py index 0fb2bbd5..9490e8ca 100644 --- a/backend/services/frame_processor.py +++ b/backend/services/frame_processor.py @@ -18,6 +18,7 @@ ARCHIVE_BATCH_MAX, ARCHIVE_FLUSH_INTERVAL_S, N2_TRACK_HISTORY_MAX, + TRACK_MAX_STALE_S, ) from core import state from pipeline.passive_radar import PassiveRadarPipeline @@ -254,7 +255,11 @@ def _view_adsb_hex(track, hist) -> str | None: return hexn -def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> list[dict]: +def confirmed_track_views( + tracker, + history_n: int = N2_TRACK_HISTORY_MAX, + now_ts_ms: int | None = None, +) -> list[dict]: """A tracker's confirmed tracks, in the shape submit_tracks takes. TENTATIVE tracks are excluded, the same filter the arc builder applies: they @@ -264,9 +269,26 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis reason arcs keep it — at 22 fps a single missed frame flips ACTIVE → COASTING and the next flips it back. + But COASTING is kept only while its newest REAL detection is fresh. What + travels downstream is ``history[-1]``, and association hands that sample to + the solver as the node's current measurement — so a track coasting toward + its N_DELETE deletion point contributes a delay from wherever the aircraft + was several seconds ago. That is the out-of-cone node the rms trim then + has to discard (see TRACK_MAX_STALE_S). The staleness test reads + ``hist[-1]["timestamp"]`` rather than the track's coast count because + get_recent_detections returns only ASSOCIATED samples — mark_missed appends + None to ``history["measurements"]`` and the reverse scan skips those — so + that timestamp IS the last real detection's, exactly the honest signal, + while n_missed only counts frames the node happened to process. Compared + against *now_ts_ms*, the frame timestamp being processed, never wall clock: + the fleet replays and backfills, and a filter keyed on wall clock would + silently empty every view in those runs. Skipped when the caller supplies + no frame time, or when TRACK_MAX_STALE_S is 0. + Shared with scripts/association_bench.py (which carried a near-verbatim copy) so the bench feeds association exactly what production does. """ + max_stale_ms = TRACK_MAX_STALE_S * 1000.0 if now_ts_ms is not None else 0.0 views = [] for tr in tracker.tracks: if tr.state_status == TrackState.TENTATIVE: @@ -274,6 +296,9 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis hist = tr.get_recent_detections(history_n) if len(hist) < 2: continue + if max_stale_ms > 0 and (now_ts_ms - hist[-1]["timestamp"]) > max_stale_ms: + state.bump_counter("tracks_stale_skipped") + continue views.append( { "track_id": tr.id or f"tmp-{id(tr)}", @@ -292,8 +317,8 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis return views -def _node_track_views(pipeline: PassiveRadarPipeline) -> list[dict]: - return confirmed_track_views(pipeline.tracker) +def _node_track_views(pipeline: PassiveRadarPipeline, now_ts_ms: int | None = None) -> list[dict]: + return confirmed_track_views(pipeline.tracker, now_ts_ms=now_ts_ms) def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarPipeline): @@ -397,7 +422,7 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP # Track-level association. The detection-level path it replaced now lives # in retina_analytics.detection_association, reachable only from the # offline bench, which keeps it as the A/B baseline. - _track_views = _node_track_views(pipeline) + _track_views = _node_track_views(pipeline, _ts_ms_assoc or None) # Feed the per-node distinct-track counters — total_tracks / # geolocated_tracks were exported (and read by the admin API) but never # written anywhere. diff --git a/backend/services/tasks/known_lane.py b/backend/services/tasks/known_lane.py index 6ffe29ad..8d335bbd 100644 --- a/backend/services/tasks/known_lane.py +++ b/backend/services/tasks/known_lane.py @@ -245,6 +245,12 @@ def _build_solver_input(hexn: str, claims: dict[str, dict]) -> dict | None: "delay_us": float(c["delay_us"]), "doppler_hz": float(c["doppler_hz"]), "snr": _num(c.get("snr")), + # This lane does NOT have one epoch: _CLAIM_SPREAD_S admits + # claims up to 5 s apart, which at 300 m/s is ~1.5 km of target + # motion charged straight to the residual this lane exists to + # measure. Carrying each claim's own capture time lets + # _attempt reuse the regular lane's epoch alignment. + "t_s": int(c["ts_ms"]) / 1000.0, } for nid, c in sorted(claims.items()) ], @@ -362,6 +368,15 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non record's displacement_km and the accuracy error are the same number. """ state.bump_counter("known_lane_attempts") + # Same correction, same flag, same helper as the regular lane — see + # solver.align_measurement_epochs. Applied here rather than in + # _build_solver_input because the alignment needs the node configs, and + # because the accuracy classification below compares the solve against an + # initial guess already dead-reckoned to the newest claim's epoch, which is + # exactly the t0 the helper aligns onto. + epoch_meta: dict = {"epoch_aligned": False} + if state.SOLVER_EPOCH_ALIGN: + s_in, epoch_meta = solver_mod.align_measurement_epochs(s_in, node_cfgs) try: # Single solve at the pinned ADS-B altitude — no layer sweep. The # sweep exists to DISCOVER an unknown altitude; here identity already @@ -379,7 +394,7 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non "known_no_converge", s_in, result if isinstance(result, dict) else None, - extra={"known_lane": True, "label": "no_converge", "published": False}, + extra={"known_lane": True, "label": "no_converge", "published": False, **epoch_meta}, ) return @@ -419,7 +434,7 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non raw_lat=raw_lat, raw_lon=raw_lon, displacement_km=err_km, - extra={"known_lane": True, "label": label, "published": published}, + extra={"known_lane": True, "label": label, "published": published, **epoch_meta}, ) diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 7c6b947c..4c1f56ea 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -279,6 +279,91 @@ def _dark_displacement_cap_km() -> float: # It lived here while the frame path had its own, looser, unstated one. +# ── Measurement epoch alignment ────────────────────────────────────────────── +# The solver's residual model evaluates every measurement against ONE target +# state: the measurement set is assumed simultaneous. It is not. Each node +# samples on its own free-running cadence (~0.74-1 Hz on the fleet), so the +# delays in one solver input were captured at times spread over up to a frame +# interval, and association hands over each track's newest sample regardless of +# when that was. A 250 m/s target moves ~250 m per second of skew, which shows +# up as up to ~1 us of bistatic delay error per second — charged in full to the +# 3 us rms_delay gate, where it is indistinguishable from a contaminated node +# and drives the trim to throw away legitimately in-cone nodes. +# +# The correction is closed-form and needs nothing the measurement does not +# already carry. Writing d_tx / d_rx for the TX->target and target->RX ranges, +# the bistatic delay is (d_tx + d_rx - baseline)/c and the bistatic Doppler is +# (fc/c)(v_tx + v_rx), where v_tx / v_rx are the target's velocity components +# along the unit vectors pointing FROM the target TOWARD the TX and the RX +# (retina_geolocator.multinode_solver._residual_function; the simulator's +# _bistatic_delay / _bistatic_doppler in retina_simulation.world use the +# identical convention). Moving toward a site shortens that leg, so +# d(d_tx)/dt = -v_tx and d(d_rx)/dt = -v_rx, and therefore +# +# d(delay_us)/dt = -(v_tx + v_rx) / C_KM_US +# = -doppler_hz * (C_KM_S / fc_hz) / C_KM_US +# = -doppler_hz * 1e6 / fc_hz +# +# i.e. positive Doppler is a closing target and its delay is DECREASING. The +# unit test test_epoch_alignment.py checks the sign against a target flown +# through the simulator's own geometry helpers at two times, rather than +# against this derivation. +_DELAY_RATE_HZ_TO_US_PER_S = 1e6 + + +def align_measurement_epochs(s_in: dict, node_cfgs: dict) -> tuple[dict, dict]: + """Dead-reckon every measurement's delay onto the newest one's epoch. + + Pure: returns a new solver input (shallow copy, fresh measurement dicts) + and a metadata dict for the history record; *s_in* is never mutated, so a + caller can drop the result and keep the untouched input. + + Alignment is all-or-nothing per input. A partially aligned set is worse + than an unaligned one — the residual model has no way to know which + measurements share an epoch, so mixing corrected and uncorrected delays + just moves the error onto a different node. Any measurement missing t_s + or doppler_hz, or whose node has no config to read fc_hz from, therefore + skips the whole input and counts solver_epoch_align_skipped. + + Returns (s_in, meta) where meta carries epoch_aligned and, when it ran, + epoch_skew_s — the widest gap the correction closed. + """ + meas = s_in.get("measurements") or [] + if len(meas) < 2: + return s_in, {"epoch_aligned": False} + + rates = [] + for m in meas: + t_s = m.get("t_s") + doppler = m.get("doppler_hz") + cfg = node_cfgs.get(m.get("node_id")) or {} + # Same fallback chain the geolocator uses when it builds its NodeSetup, + # so a node whose config spells the carrier "FC" aligns on exactly the + # frequency the solve will predict against. + fc_hz = cfg.get("fc_hz", cfg.get("FC")) + if t_s is None or doppler is None or not fc_hz: + state.bump_counter("solver_epoch_align_skipped") + return s_in, {"epoch_aligned": False} + rates.append((float(t_s), -float(doppler) * _DELAY_RATE_HZ_TO_US_PER_S / float(fc_hz))) + + # The newest sample, not the input's timestamp_ms: t0 has to be a time some + # measurement was actually taken, or every delay is extrapolated and the + # freshest node — the one that needed no correction — acquires an error. + t0 = max(t for t, _ in rates) + skew_s = t0 - min(t for t, _ in rates) + + aligned = dict(s_in) + aligned["measurements"] = [ + {**m, "delay_us": float(m["delay_us"]) + rate * (t0 - t_s)} for m, (t_s, rate) in zip(meas, rates) + ] + if "timestamp_ms" in aligned: + # The set now describes t0, so everything downstream that ages this + # solve (multinode expiry, the dead-reckoning gates, the history + # record's measurement_ts_ms) should date it from t0 too. + aligned["timestamp_ms"] = int(round(t0 * 1000.0)) + return aligned, {"epoch_aligned": True, "epoch_skew_s": round(skew_s, 3)} + + def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[float], metric: str) -> dict | None: """Try each altitude layer; return the result with lowest value of `metric`. @@ -1728,6 +1813,13 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus state.bump_counter("solver_resolve_skips") return None n_nodes = s_in.get("n_nodes", 0) if isinstance(s_in, dict) else 0 + # Before anything reads a delay: the nodes did not sample simultaneously, + # and every gate below (rms_delay first among them) assumes they did. Runs + # ahead of consensus and the altitude sweep so both judge the same aligned + # numbers the published solve is fitted to. + epoch_meta: dict = {"epoch_aligned": False} + if state.SOLVER_EPOCH_ALIGN and isinstance(s_in, dict): + s_in, epoch_meta = align_measurement_epochs(s_in, node_cfgs) consensus_meta: dict | None = None try: if "initial_guess" not in s_in: @@ -1771,6 +1863,10 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus _extra: dict | None = dict(trim_meta) if trim_meta else {} if consensus_meta is not None: _extra["consensus_meta"] = consensus_meta + # Always stamped, aligned or not: "this solve was not aligned" is the + # fact /api/test/mlat-history needs to separate a residual the + # correction could not have helped from one it was applied to. + _extra.update(epoch_meta) _extra = _extra or None rms_delay = result.get("rms_delay", 0) or 0 diff --git a/libs/retina-analytics b/libs/retina-analytics index 14504176..d9dc9e00 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 145041767422723d89d98ec003f843347ddbb880 +Subproject commit d9dc9e00f97d3788f5319109e0832cb270ed2fec From cb38610b1eae394ee8472491fc2e7759d6b48ddc Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 08:33:35 +0000 Subject: [PATCH 11/25] solver: one free-altitude start by default, SOLVER_FREE_ALT_STARTS for more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The free mode shipped with three start altitudes on the reasoning that freeing z removes the ladder's quantisation but not the LM's locality, so several starts are still what keeps a solve off the wrong side of a bistatic ellipse. On this fleet's geometry there was almost nothing for them to keep it off. Over a 20-minute window on test, 1019 free-mode solves: the three starts' rms_delay differed by more than 0.1 us in 13 of them (1.3%), and the nearest-layer start — the one the sweep would have pinned at — was more than 0.5 us worse than the best start in 2. So the extra two starts bought ~0.2% of solves a rounding error at three times the solver CPU, and solver CPU is now the constraint: ~1.7 attempts/s against a 2.0 s average latency on two pool workers. The count becomes SOLVER_FREE_ALT_STARTS, default 1, read per call beside the mode flag and clamped into [1, len(layers)] by _free_alt_starts against the ladder that module owns. One start is the layer nearest the association guess, or the guess altitude itself when the input carries a non-layer one (ADS-B) — the same splice the sweep does, so the one exact altitude a candidate has is still what it starts from. Above 1 the window is unchanged, so 3 reproduces what was measured; the knob stays because locality is a property of the geometry, not of this fleet, and a deployment whose nodes sit nearer an ellipse should not need a code change to buy the starts back. alt_starts_km / alt_start_rms_us keep being recorded — a one-element list rather than three — so the live comparison channel is unchanged. Co-Authored-By: Claude Fable 5.1 --- backend/.env.example | 9 +++ backend/core/state.py | 21 +++++- backend/services/tasks/solver.py | 34 ++++++--- backend/tests/test_solver_alt_mode.py | 101 +++++++++++++++++++++++--- docs/solverflow.md | 28 +++++-- 5 files changed, 162 insertions(+), 31 deletions(-) diff --git a/backend/.env.example b/backend/.env.example index 488ecbdc..51a1c782 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -138,10 +138,19 @@ MENDER_PAT= # multi-start helper, which solves altitude as a sixth unknown. Both modes # stamp altitude_mode on the solve-history record, so /api/test/mlat-history # can compare them across a deploy of each. See docs/solverflow.md. +# +# SOLVER_FREE_ALT_STARTS is how many start altitudes free mode gives that +# helper, clamped to the number of layers. 1 (the default) starts at the layer +# nearest the association guess, or at the guess altitude itself when it comes +# from ADS-B; more is a window around it, at one LM run each. Measured on test +# over 1019 free solves, three starts changed rms_delay by more than 0.1 us in +# 13 of them, so the extra runs are off by default and worth turning on only +# where the geometry sends a single start to the wrong side of an ellipse. # SOLVER_WORKERS=2 # SOLVER_RESOLVE_INTERVAL_S=12 # SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0 # SOLVER_ALT_MODE=sweep +# SOLVER_FREE_ALT_STARTS=1 # Detection mirror. Production only. Every accepted v1 detection frame is # forwarded to another environment's /api/radar/detections/bulk, batched once a diff --git a/backend/core/state.py b/backend/core/state.py index f1239459..962c9006 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -84,7 +84,8 @@ # systematically up to 1 km wrong and that error lands in the # residual the reject gate reads. # free — one pool call to retina_geolocator's multi-start helper, which -# solves altitude as a sixth unknown from three start layers. +# solves altitude as a sixth unknown, started from +# SOLVER_FREE_ALT_STARTS of those layers. # Not off/shadow/active: there is no shadow here, because the two modes # produce the same shape of result and the history record carries # altitude_mode either way — running both would double the solver's cost to @@ -94,6 +95,24 @@ if SOLVER_ALT_MODE not in ("sweep", "free"): SOLVER_ALT_MODE = "sweep" +# How many start altitudes the free mode hands that helper. Read here beside +# the mode it qualifies; _free_alt_starts in services/tasks/solver.py clamps it +# into [1, len(layers)] against the ladder that module owns. 1 starts at the +# layer nearest the association guess — where the sweep would have pinned; +# more is a window around it. +# +# The default is 1 because three starts did not pay for themselves: over 1019 +# free-mode solves on test, the three starts' rms_delay differed by more than +# 0.1 us in 13 of them, and the nearest-layer start was more than 0.5 us worse +# than the best start in 2. That is ~0.2% of solves helped for 3x the solver +# CPU, and the pool — not the altitude ladder — is what this deployment is +# short of (~1.7 attempts/s against a 2.0 s average latency on two workers). +# The knob stays because the reason for several starts is the LM's locality, +# which is a property of the geometry rather than of this fleet: nodes lying +# nearer a bistatic ellipse than these can send a single start to the wrong +# side of it, and finding that out should not need a code change. +SOLVER_FREE_ALT_STARTS = max(1, int(os.getenv("SOLVER_FREE_ALT_STARTS", "1"))) + node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE) diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 41bb53ec..df890941 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -338,12 +338,6 @@ def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[floa return best_result -# How many start altitudes the free mode hands the multi-start helper: the -# layer nearest the guess and its two neighbours. Freeing z removes the -# ladder's quantisation but not the LM's locality, so the starts are still -# what stops a solve settling on the wrong side of a bistatic ellipse — but -# three of them, in one pool call, rather than the sweep's six. -_FREE_ALT_N_STARTS = 3 # Fewest measurements the free mode is used at. Below this altitude is not # observable and retina_geolocator pins it anyway; the sweep is left in place # so the n=2 path keeps its documented behaviour exactly. @@ -351,17 +345,33 @@ def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[floa def _free_alt_starts(ig_alt_km, layers: list[float]) -> list[float]: - """The layer nearest ``ig_alt_km`` and its neighbours — _FREE_ALT_N_STARTS - of them, clamped to the ends of the ladder so the count never shrinks - there (the top and bottom layers are where a wrong start is least + """The start altitudes the free mode hands the multi-start helper. + + state.SOLVER_FREE_ALT_STARTS of them, clamped into [1, len(layers)] — read + per call, like the mode flag, so a test and a config reload both see what + they set. One start is the layer nearest ``ig_alt_km``, which is the + altitude spliced into ``layers`` when the input carries a non-layer one of + its own (ADS-B), exactly as the sweep treats it. Several are a window + centred on that layer, clamped to the ends of the ladder so the count never + shrinks there (the top and bottom layers are where a wrong start is least recoverable, not most). + + Freeing z removes the ladder's quantisation but not the LM's locality, and + the extra starts are what would stop a solve settling on the wrong side of + a bistatic ellipse. On this fleet's geometry they had almost nothing to + stop: over 1019 free-mode solves on test, three starts' rms_delay differed + by more than 0.1 µs in 13 of them, and the nearest-layer start was more + than 0.5 µs worse than the best in 2 — so the default is one start and the + other two are bought explicitly, by a deployment whose geometry shows it + needs them. See core/state.py for the numbers and the trade. """ if not layers: return [] + n = max(1, min(int(state.SOLVER_FREE_ALT_STARTS), len(layers))) alt = float(ig_alt_km) if ig_alt_km is not None else 7.0 nearest = min(range(len(layers)), key=lambda i: abs(layers[i] - alt)) - lo = max(0, min(nearest - 1, len(layers) - _FREE_ALT_N_STARTS)) - return layers[lo : lo + _FREE_ALT_N_STARTS] + lo = max(0, min(nearest - (n - 1) // 2, len(layers) - n)) + return layers[lo : lo + n] def _solve_best_altitude( @@ -399,7 +409,7 @@ def _solve_best_altitude( n_meas = len({m.get("node_id") for m in (s_in.get("measurements") or [])}) if state.SOLVER_ALT_MODE == "free" and n_meas >= _FREE_ALT_MIN_NODES: # No fall back to the sweep when this returns None: a helper that got - # no solve out of three starts is reporting the same thing the sweep + # no solve out of its starts is reporting the same thing the sweep # reports when every layer fails, and sweeping anyway would cost the # six round trips this mode exists to avoid on exactly the candidates # that are least likely to repay them. diff --git a/backend/tests/test_solver_alt_mode.py b/backend/tests/test_solver_alt_mode.py index 8a374dbc..2c9eec02 100644 --- a/backend/tests/test_solver_alt_mode.py +++ b/backend/tests/test_solver_alt_mode.py @@ -4,7 +4,8 @@ lowest rms_delay — six process-pool round trips, and an altitude quantised to a ladder 2 km wide, which puts up to 1 km of error into the residual the reject gate reads. free makes ONE call to the geolocator's multi-start -helper, which solves altitude as a sixth unknown from three start layers. +helper, which solves altitude as a sixth unknown from SOLVER_FREE_ALT_STARTS +start layers — one by default, the layer nearest the association guess. These tests are about the routing, not the physics: the geolocator's own suite (tests/test_free_altitude.py there) measures what the free solve @@ -78,7 +79,32 @@ def teardown_method(self): class TestFreeAltStarts: - """The three starts handed to the multi-start helper.""" + """The starts handed to the multi-start helper: SOLVER_FREE_ALT_STARTS of + them, one by default.""" + + @pytest.mark.parametrize( + "alt_km,expected", + [ + (9.0, [9.0]), + (7.0, [7.0]), + (8.2, [9.0]), + (1.5, [1.5]), + # Off the ends of the ladder: still the nearest layer, not nothing. + (0.4, [1.5]), + (40.0, [11.0]), + ], + ) + def test_one_start_at_the_nearest_layer_by_default(self, alt_km, expected): + assert state.SOLVER_FREE_ALT_STARTS == 1 + assert solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) == expected + + def test_an_adsb_altitude_in_the_ladder_is_the_start(self): + """_solve_best_altitude splices a non-layer altitude (ADS-B) into the + layers, and the starts are taken over that spliced list — so the single + default start is that exact altitude, which is what the sweep would + have pinned too.""" + layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4])) + assert solver_mod._free_alt_starts(8.4, layers) == [8.4] @pytest.mark.parametrize( "alt_km,expected", @@ -94,18 +120,31 @@ class TestFreeAltStarts: (40.0, [7.0, 9.0, 11.0]), ], ) - def test_window_around_the_nearest_layer(self, alt_km, expected): + def test_three_starts_are_the_window_around_the_nearest_layer(self, alt_km, expected, monkeypatch): + """The pre-default behaviour, still reachable by configuration.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3) starts = solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) assert starts == expected - assert len(starts) == solver_mod._FREE_ALT_N_STARTS + assert len(starts) == 3 - def test_an_adsb_altitude_in_the_ladder_is_a_start(self): - """_solve_best_altitude splices an ADS-B altitude into the layers, and - the window is taken over that spliced list — otherwise the one exact - altitude available would never be started from.""" + def test_three_starts_window_the_spliced_adsb_altitude(self, monkeypatch): + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3) layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4])) assert solver_mod._free_alt_starts(8.4, layers) == [7.0, 8.4, 9.0] + @pytest.mark.parametrize("configured", [0, -3]) + def test_fewer_than_one_start_still_starts_somewhere(self, configured, monkeypatch): + """A count below one would leave the LM no start at all, so it clamps + rather than raises: a mis-set env degrades to a working solve.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", configured) + assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == [9.0] + + def test_more_starts_than_layers_is_every_layer(self, monkeypatch): + """The other clamp: a count past the end of the ladder would slice + short of it, quietly dropping starts that were asked for.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 99) + assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == solver_mod._SOLVER_ALT_LAYERS_KM + def test_no_layers_gives_no_starts(self): assert solver_mod._free_alt_starts(9.0, []) == [] @@ -135,10 +174,10 @@ def teardown_method(self): state.SOLVER_ALT_MODE = self._saved_mode super().teardown_method() - def test_one_multistart_call_with_three_starts(self): + def test_one_multistart_call_with_one_start(self): nodes = ["n1", "n2", "n3"] solve = _Recorder(lambda n, s, *r: pytest.fail("sweep ran in free mode")) - multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[1.2, 0.4, 0.9])) + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4])) result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) @@ -146,7 +185,29 @@ def test_one_multistart_call_with_three_starts(self): assert solve.calls == [] assert len(multistart.calls) == 1 (_, _, rest) = multistart.calls[0] - assert rest == ([7.0, 9.0, 11.0],) + assert rest == ([9.0],) + + def test_an_adsb_guess_altitude_is_the_start(self): + """A non-layer initial_guess altitude is spliced into the ladder and + becomes the start itself — the free-mode analogue of the sweep's extra + layer, and the one exact altitude the candidate has.""" + nodes = ["n1", "n2", "n3"] + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free")) + + solver_mod._solve_best_altitude(_s_in(nodes, alt_km=8.437), {}, lambda s, c: None, multistart) + + assert multistart.calls[0][2] == ([8.437],) + + def test_the_start_count_is_configurable(self, monkeypatch): + """SOLVER_FREE_ALT_STARTS buys back the neighbour window for a geometry + whose single start lands on the wrong side of an ellipse.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3) + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free")) + + solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, lambda s, c: None, multistart) + + assert len(multistart.calls) == 1 + assert multistart.calls[0][2] == ([7.0, 9.0, 11.0],) def test_n2_keeps_the_sweep(self): """Altitude is unobservable at n=2 — the free path is not entered even @@ -188,6 +249,24 @@ def test_history_carries_the_mode_and_the_per_start_residuals(self): assert rec["alt_starts_km"] == [5.0, 7.0, 9.0] assert rec["alt_start_rms_us"] == [1.234, None, 0.432] + def test_a_single_start_still_records_its_residual(self): + """The comparison channel does not depend on there being several + starts: one start records a one-element list, not a bare number or + nothing at all.""" + nodes = ["n1", "n2", "n3"] + multistart = _Recorder( + lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4321], alt_starts_km=[9.0]) + ) + solver_mod._process_solver_item( + (_s_in(nodes), {}, time.time()), + lambda s, c: pytest.fail("sweep ran in free mode"), + multistart_fn=multistart, + ) + + rec = state.mlat_solve_history[-1] + assert rec["alt_starts_km"] == [9.0] + assert rec["alt_start_rms_us"] == [0.432] + def test_z_saturation_reaches_the_history(self): nodes = ["n1", "n2", "n3"] multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", z_saturated=True)) diff --git a/docs/solverflow.md b/docs/solverflow.md index 50101e85..3908611b 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -81,7 +81,8 @@ own. Everything that reaches a solve passes through one gate stack | `FRAME_WORKERS` | 4 (compose sets 6) | `main.py:164`, `docker-compose.yml:54` | | `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py:31,67` | | `KNOWN_LANE_MODE` default | `binding` | `core/state.py:72-74` | -| `SOLVER_ALT_MODE` default | `sweep` | `core/state.py:77-95` | +| `SOLVER_ALT_MODE` default | `sweep` | `core/state.py:77-96` | +| `SOLVER_FREE_ALT_STARTS` default | 1 | `core/state.py:98-114` | --- @@ -456,7 +457,7 @@ flowchart TD m7 --> alt{"n_nodes >= 3?"} alt -->|"yes"| mode{"SOLVER_ALT_MODE"} mode -->|"sweep (default)"| sweep["_solve_best_altitude:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"] - mode -->|"free"| freealt["_solve_best_altitude:
ONE pool call to
solve_multinode_multistart,
3 start layers, z solved"] + mode -->|"free"| freealt["_solve_best_altitude:
ONE pool call to
solve_multinode_multistart,
SOLVER_FREE_ALT_STARTS start
layers (1 by default), z solved"] alt -->|"no, n=2"| single["_solve_best_altitude_n2:
one LM call at the
association altitude"] classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 @@ -475,13 +476,26 @@ trimming (6.4) drops measurements that were never the problem. `free` instead calls `solve_multinode_multistart`, which runs the LM with altitude as a sixth unknown (state `[x, y, z, vx, vy, vz]`, z bounded -0.05–20 km, the `vz` bound unchanged) from three start layers — the one nearest -the association guess and its two neighbours — and keeps the lowest `rms_delay`. -Freeing z removes the ladder's quantisation but not the LM's locality, which is -what the several starts are still for. It is also cheaper: **one** process-pool +0.05–20 km, the `vz` bound unchanged) from `SOLVER_FREE_ALT_STARTS` start +layers, keeping the lowest `rms_delay`. It is also cheaper: **one** process-pool round trip per candidate instead of six, each of which pickles the node configs the input needs. +`SOLVER_FREE_ALT_STARTS` defaults to **1** — the layer nearest the association +guess, or the guess altitude itself when that came from ADS-B and was spliced +into the ladder (the same splice the sweep does). Freeing z removes the +ladder's quantisation but not the LM's locality, and extra starts are what +would stop a solve settling on the wrong side of a bistatic ellipse; on this +fleet's geometry they had almost nothing to stop. Over a 20-minute window of +1019 free-mode solves on test, the three starts' `rms_delay` differed by more +than 0.1 us in **13** of them, and the nearest-layer start was more than 0.5 us +worse than the best start in **2** — ~0.2% of solves helped, at three times the +solver CPU, while the pool is the binding constraint (~1.7 attempts/s against a +2.0 s average latency on two workers). Set it above 1 for a geometry where that +locality does bite; `_free_alt_starts` clamps it into `[1, len(layers)]` and +values above 1 give the same neighbour window as before, so `3` restores the +original behaviour exactly. + At n=2 the mode is inert — four residuals cannot support six unknowns, so the geolocator pins altitude regardless and `_solve_best_altitude_n2` is unchanged. Trimming re-solves through `_solve_best_altitude`, so a trim round inherits @@ -498,7 +512,7 @@ mode per environment and read the two lanes' `rms_delay` and `gt_error_km` off | Mode | Pool calls per n>=3 candidate | Altitude | |---|---|---| | `sweep` (default) | 6 (one per layer) | quantised to the nearest layer | -| `free` | 1 (three starts inside it) | solved, 0.05–20 km | +| `free` | 1 (`SOLVER_FREE_ALT_STARTS` starts inside it, 1 by default) | solved, 0.05–20 km | | Constant | Value | File:line | |---|---|---| From fcccd03fecce25f12857d6f14ba99fe0fef33440 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 08:47:55 +0000 Subject: [PATCH 12/25] Test stale-track exclusion and measurement epoch alignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sign of the Doppler-to-delay-rate conversion is the part of this that cannot be checked by reading it, so it is checked against the simulator's own _bistatic_delay / _bistatic_doppler: a target is flown in a straight line, sampled at two times, and the correction applied to the older sample must land on the newer sample's true delay. Three velocities (inbound, outbound, crossing-with-climb), plus a direct assertion that the correction and the true delay change share a sign, so a failure says "the sign is wrong" rather than "the error did not shrink enough". The staleness tests express the coast as a gap between the newest ASSOCIATED sample and the frame time, which is exactly what the tracker produces — get_recent_detections skips the None measurements mark_missed appends — and pin that TENTATIVE exclusion is unchanged and does not feed the stale counter, so the counter keeps meaning "an aircraft left this node's cone". Co-Authored-By: Claude Fable 5.1 --- backend/tests/test_epoch_alignment.py | 274 ++++++++++++++++++++++++++ backend/tests/test_frame_processor.py | 93 +++++++++ backend/tests/test_solver_stats.py | 4 + 3 files changed, 371 insertions(+) create mode 100644 backend/tests/test_epoch_alignment.py diff --git a/backend/tests/test_epoch_alignment.py b/backend/tests/test_epoch_alignment.py new file mode 100644 index 00000000..a805dc22 --- /dev/null +++ b/backend/tests/test_epoch_alignment.py @@ -0,0 +1,274 @@ +"""Measurement epoch alignment (SOLVER_EPOCH_ALIGN) — solver.align_measurement_epochs. + +The solver's residual model evaluates every measurement against ONE target +state, so a solver input is implicitly a claim that its measurements were +simultaneous. Nodes sample on independent free-running cadences, so the claim +is false by up to a frame interval, and the resulting delay error is charged to +the rms_delay gate. These tests pin the correction, and — more importantly — +pin its SIGN against the simulator's own geometry rather than against the +derivation the helper's comment gives, since a sign error there would silently +double the very error the correction exists to remove. +""" + +import pytest +from retina_simulation.world import _bistatic_delay, _bistatic_doppler + +from core import state +from services.tasks import solver as solver_mod +from services.tasks.solver import align_measurement_epochs + +_FC_HZ = 183e6 + +# One node's ENU geometry, km. Only fc_hz is read by the helper; the rest is +# here because the simulator's delay/Doppler helpers need a real bistatic +# triangle to produce numbers whose sign means anything. +_TX_ENU = (-20.0, 5.0, 0.05) +_RX_ENU = (0.0, 0.0, 0.3) + +_NODE_CFGS = { + "node-a": {"fc_hz": _FC_HZ}, + "node-b": {"fc_hz": _FC_HZ}, + "node-c": {"FC": _FC_HZ}, # the alternate spelling the geolocator accepts +} + + +def _s_in(measurements, **over): + base = { + "initial_guess": {"lat": 34.85, "lon": -82.4, "alt_km": 9.0}, + "measurements": measurements, + "n_nodes": len({m["node_id"] for m in measurements}), + "timestamp_ms": 1_700_000_000_000, + } + base.update(over) + return base + + +def _m(node_id, delay_us, doppler_hz, t_s, snr=15.0): + return { + "node_id": node_id, + "delay_us": delay_us, + "doppler_hz": doppler_hz, + "snr": snr, + "t_s": t_s, + } + + +@pytest.fixture(autouse=True) +def _zero_counter(): + state.solver_epoch_align_skipped = 0 + yield + + +class TestPureHelper: + def test_newest_measurement_is_the_epoch_and_is_untouched(self): + """t0 is the newest SAMPLE time, not the input's timestamp_ms: the + freshest node needed no correction and must not acquire one.""" + s_in = _s_in( + [ + _m("node-a", 40.0, 100.0, 1000.0), + _m("node-b", 50.0, -80.0, 1001.5), + _m("node-c", 60.0, 0.0, 1002.0), + ] + ) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + by_id = {m["node_id"]: m for m in out["measurements"]} + assert by_id["node-c"]["delay_us"] == 60.0 + assert meta["epoch_aligned"] is True + assert meta["epoch_skew_s"] == pytest.approx(2.0) + + def test_each_delay_moves_by_its_own_doppler_rate(self): + """d(delay_us)/dt = -doppler_hz * 1e6 / fc_hz, applied over that + measurement's own gap to t0.""" + s_in = _s_in( + [ + _m("node-a", 40.0, 100.0, 1000.0), + _m("node-b", 50.0, -80.0, 1001.5), + _m("node-c", 60.0, 0.0, 1002.0), + ] + ) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + by_id = {m["node_id"]: m for m in out["measurements"]} + assert by_id["node-a"]["delay_us"] == pytest.approx(40.0 + (-100.0 * 1e6 / _FC_HZ) * 2.0) + assert by_id["node-b"]["delay_us"] == pytest.approx(50.0 + (80.0 * 1e6 / _FC_HZ) * 0.5) + + def test_zero_doppler_measurement_is_unchanged(self): + """A tangential target's bistatic range is stationary, so no amount of + skew moves its delay — the rate is the only thing that can.""" + s_in = _s_in([_m("node-a", 40.0, 0.0, 1000.0), _m("node-b", 50.0, 20.0, 1004.0)]) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + assert out["measurements"][0]["delay_us"] == 40.0 + + def test_input_is_not_mutated(self): + """Pure: a caller must be able to drop the result and keep the + original, which is exactly what the flag-off path does.""" + meas = [_m("node-a", 40.0, 100.0, 1000.0), _m("node-b", 50.0, -80.0, 1002.0)] + s_in = _s_in(meas) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + assert s_in["measurements"][0]["delay_us"] == 40.0 + assert s_in["measurements"] is not out["measurements"] + assert s_in["timestamp_ms"] == 1_700_000_000_000 + + def test_timestamp_ms_is_restamped_to_the_epoch(self): + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), _m("node-b", 50.0, -80.0, 1002.25)]) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + assert out["timestamp_ms"] == 1_002_250 + + def test_fc_spelled_FC_is_accepted(self): + """Same fallback chain the geolocator uses to build its NodeSetup, so + a node aligns on exactly the carrier its solve predicts against.""" + s_in = _s_in([_m("node-c", 40.0, 100.0, 1000.0), _m("node-b", 50.0, 0.0, 1001.0)]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert meta["epoch_aligned"] is True + assert out["measurements"][0]["delay_us"] == pytest.approx(40.0 - 100.0 * 1e6 / _FC_HZ) + + def test_single_measurement_input_is_a_no_op(self): + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0)]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert out is s_in + assert meta == {"epoch_aligned": False} + assert state.solver_epoch_align_skipped == 0 + + +class TestSkipPath: + @pytest.mark.parametrize( + "broken", + [ + {"t_s": None}, + {"doppler_hz": None}, + ], + ) + def test_missing_field_skips_the_whole_input(self, broken): + """All-or-nothing: a partially aligned set has no marker saying which + measurements share an epoch, so it just relocates the error.""" + good = _m("node-a", 40.0, 100.0, 1000.0) + bad = {**_m("node-b", 50.0, -80.0, 1002.0), **broken} + s_in = _s_in([good, bad]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert out is s_in + assert meta == {"epoch_aligned": False} + assert state.solver_epoch_align_skipped == 1 + + def test_missing_t_s_key_entirely_skips(self): + """The pre-upgrade measurement shape: no t_s key at all.""" + untimed = {"node_id": "node-b", "delay_us": 50.0, "doppler_hz": -80.0, "snr": 9.0} + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), untimed]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert out is s_in + assert meta["epoch_aligned"] is False + assert state.solver_epoch_align_skipped == 1 + + def test_unknown_node_config_skips(self): + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), _m("node-zzz", 50.0, -80.0, 1002.0)]) + out, meta = align_measurement_epochs(s_in, {"node-a": {"fc_hz": _FC_HZ}}) + assert out is s_in + assert meta["epoch_aligned"] is False + assert state.solver_epoch_align_skipped == 1 + + +class TestSignAgainstSimulatorGeometry: + """The sign check, run against the simulator's own delay/Doppler model. + + A target is flown in a straight line and sampled at two times using + _bistatic_delay / _bistatic_doppler. The older sample plus the correction + must land on the newer sample's true delay — which is a statement about the + sign of the Doppler-to-delay-rate conversion that no amount of algebra in a + comment can substitute for. + """ + + _POS0 = (10.0, 15.0, 9.0) # km ENU + _DT_S = 2.0 + + @staticmethod + def _truth(vel_kms, dt_s): + pos0 = TestSignAgainstSimulatorGeometry._POS0 + pos1 = tuple(pos0[i] + vel_kms[i] * dt_s for i in range(3)) + return ( + _bistatic_delay(pos0, _TX_ENU, _RX_ENU), + _bistatic_delay(pos1, _TX_ENU, _RX_ENU), + _bistatic_doppler(pos0, vel_kms, _TX_ENU, _RX_ENU, _FC_HZ), + ) + + @pytest.mark.parametrize( + "vel_kms", + [ + (-0.20, -0.15, 0.0), # inbound: bistatic range shrinking + (0.20, 0.15, 0.0), # outbound: bistatic range growing + (0.05, -0.24, 0.01), # mostly crossing, with a climb + ], + ) + def test_alignment_moves_the_stale_delay_toward_the_truth(self, vel_kms): + delay0, delay1, doppler0 = self._truth(vel_kms, self._DT_S) + + # node-a sampled _DT_S seconds ago; node-b is the newest sample and + # therefore defines t0. node-b's own numbers are irrelevant to the + # assertion — it is only here to set the epoch. + s_in = _s_in( + [ + _m("node-a", delay0, doppler0, 1000.0), + _m("node-b", 77.0, 0.0, 1000.0 + self._DT_S), + ] + ) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert meta["epoch_aligned"] is True + aligned = out["measurements"][0]["delay_us"] + + err_before = abs(delay0 - delay1) + err_after = abs(aligned - delay1) + # A wrong sign would double err_before rather than shrink it, so the + # margin here is the sign test. Over 2 s of straight-line flight the + # first-order term dominates; the residual is the trajectory's + # curvature in bistatic range, not a modelling disagreement. + assert err_after < err_before * 0.2 + assert err_before > 0.05 # the case would prove nothing otherwise + + def test_correction_and_truth_share_a_sign(self): + """Stated directly, so a failure says 'the sign is wrong' rather than + 'the error did not shrink enough'.""" + vel_kms = (-0.20, -0.15, 0.0) + delay0, delay1, doppler0 = self._truth(vel_kms, self._DT_S) + s_in = _s_in( + [ + _m("node-a", delay0, doppler0, 1000.0), + _m("node-b", 77.0, 0.0, 1000.0 + self._DT_S), + ] + ) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + correction = out["measurements"][0]["delay_us"] - delay0 + assert correction * (delay1 - delay0) > 0 + + +class TestProcessSolverItemWiring: + """The flag, and that the aligned numbers are what the solve actually sees.""" + + @staticmethod + def _item(): + s_in = _s_in( + [ + _m("node-a", 40.0, 300.0, 1000.0), + _m("node-b", 50.0, 0.0, 1004.0), + ] + ) + return (s_in, _NODE_CFGS, None) + + def test_flag_off_leaves_the_input_untouched(self, monkeypatch): + monkeypatch.setattr(state, "SOLVER_EPOCH_ALIGN", False) + seen = {} + + def _solve(s_in, node_cfgs): + seen["delays"] = [m["delay_us"] for m in s_in["measurements"]] + return None + + solver_mod._process_solver_item(self._item(), _solve) + assert seen["delays"] == [40.0, 50.0] + + def test_flag_on_hands_the_solver_aligned_delays(self, monkeypatch): + monkeypatch.setattr(state, "SOLVER_EPOCH_ALIGN", True) + seen = {} + + def _solve(s_in, node_cfgs): + seen["delays"] = [m["delay_us"] for m in s_in["measurements"]] + return None + + solver_mod._process_solver_item(self._item(), _solve) + assert seen["delays"][0] == pytest.approx(40.0 + (-300.0 * 1e6 / _FC_HZ) * 4.0) + assert seen["delays"][1] == 50.0 diff --git a/backend/tests/test_frame_processor.py b/backend/tests/test_frame_processor.py index c5d31e92..56a1409a 100644 --- a/backend/tests/test_frame_processor.py +++ b/backend/tests/test_frame_processor.py @@ -6,15 +6,19 @@ import queue import time +import types import pytest +from retina_tracker.track import TrackState from config.constants import GT_DISPLAY_STALE_S from core import state from pipeline.passive_radar import DEFAULT_NODE_CONFIG, PassiveRadarPipeline +from services import frame_processor from services.frame_processor import ( append_track_history, build_combined_aircraft_json, + confirmed_track_views, dedup_aircraft, flush_all_archive_buffers, get_node_configs, @@ -277,6 +281,95 @@ def test_claiming_anchored_inputs_reach_the_solver_queue(self, monkeypatch): # ── Multinode result conversion ────────────────────────────────────────────── +class TestConfirmedTrackViewsStaleness: + """TRACK_MAX_STALE_S: a coasting track stops being offered to association + once its newest REAL detection has aged out. + + The freshness signal is the newest entry get_recent_detections returns, + which by construction is an ASSOCIATED sample (mark_missed appends None to + history["measurements"] and the reverse scan skips those) — so these fakes + hand back only real detections, exactly as the tracker does, and the coast + is expressed as a gap between that newest sample and the frame time. + """ + + @staticmethod + def _track(newest_ts_ms: int, status=TrackState.COASTING, track_id="trk-stale"): + hist = [ + {"timestamp": newest_ts_ms - 1000, "delay": 40.0, "doppler": 5.0, "snr": 12.0, "adsb": None}, + {"timestamp": newest_ts_ms, "delay": 41.0, "doppler": 5.0, "snr": 12.0, "adsb": None}, + ] + return types.SimpleNamespace( + id=track_id, + state_status=status, + adsb_hex=None, + get_recent_detections=lambda n: hist[-n:], + ) + + def _tracker(self, *tracks): + return types.SimpleNamespace(tracks=list(tracks)) + + def test_five_second_old_coasting_track_is_excluded_at_three(self, monkeypatch): + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms - 5000)) + assert confirmed_track_views(tracker, now_ts_ms=now_ms) == [] + assert state.tracks_stale_skipped == 1 + + def test_same_track_is_included_at_ten(self, monkeypatch): + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 10.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms - 5000)) + views = confirmed_track_views(tracker, now_ts_ms=now_ms) + assert [v["track_id"] for v in views] == ["trk-stale"] + assert state.tracks_stale_skipped == 0 + + def test_zero_disables_the_filter(self, monkeypatch): + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 0.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms - 600_000)) + assert len(confirmed_track_views(tracker, now_ts_ms=now_ms)) == 1 + assert state.tracks_stale_skipped == 0 + + def test_no_frame_time_disables_the_filter(self, monkeypatch): + """The bench and the ADS-B-seeding tests call without a frame time; + wall clock is not a substitute, so those callers stay unfiltered.""" + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + tracker = self._tracker(self._track(0)) + assert len(confirmed_track_views(tracker)) == 1 + assert state.tracks_stale_skipped == 0 + + def test_fresh_track_survives_beside_a_stale_one(self, monkeypatch): + """Staleness is per track, not per tracker — the node keeps + contributing whatever it can still actually see.""" + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker( + self._track(now_ms - 5000, track_id="gone"), + self._track(now_ms - 500, track_id="here"), + ) + views = confirmed_track_views(tracker, now_ts_ms=now_ms) + assert [v["track_id"] for v in views] == ["here"] + assert state.tracks_stale_skipped == 1 + + def test_tentative_is_still_excluded_regardless_of_freshness(self, monkeypatch): + """The TENTATIVE filter is unchanged and independent: a brand-new + TENTATIVE track's newest detection is as fresh as it gets, and it must + still not reach association.""" + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms, status=TrackState.TENTATIVE)) + assert confirmed_track_views(tracker, now_ts_ms=now_ms) == [] + # Skipped as TENTATIVE, not as stale — the counter must stay clean so + # it only ever means "an aircraft left this node's cone". + assert state.tracks_stale_skipped == 0 + + class TestMultinodeToAircraft: def test_basic_conversion(self): r = { diff --git a/backend/tests/test_solver_stats.py b/backend/tests/test_solver_stats.py index 67bc599c..35c80e99 100644 --- a/backend/tests/test_solver_stats.py +++ b/backend/tests/test_solver_stats.py @@ -257,6 +257,8 @@ def test_consensus_and_counters_reflect_state(self): state.solver_consensus_fallback = 9 state.solver_consensus_shadow = 10 state.solver_vel_untrusted_published = 11 + state.tracks_stale_skipped = 13 + state.solver_epoch_align_skipped = 14 out = _solver_window_stats(10.0) assert out["counters"] == { "successes": 5, @@ -265,6 +267,8 @@ def test_consensus_and_counters_reflect_state(self): "solver_trimmed": 3, "stale_drops": 4, "resolve_skips": 12, + "tracks_stale_skipped": 13, + "epoch_align_skipped": 14, "queue_drops": 6, "worker_errors": 0, "vel_untrusted_published": 11, From c66b55f2ca9b3e49ac4e5a09e3c10581ae60ff48 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 08:55:29 +0000 Subject: [PATCH 13/25] Bump retina-analytics: merge distance down to one grid step Picks up offworldlabs/retina-analytics@2b83871. The 6.0 / 4.5 / 3.0 km sweep on the association bench is monotone with the real-track count flat at every point, so 3.0 is taken: published contamination 32.0% -> 25.1%, foreign nodes per published solve 0.76 -> 0.52. Co-Authored-By: Claude Fable 5.1 --- libs/retina-analytics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/retina-analytics b/libs/retina-analytics index 7e5414ac..2b83871e 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 7e5414ac1671bb66c160a425363b1d19c889eea4 +Subproject commit 2b83871e38cb17d9a2e5ceda3a55ceeaf1492f42 From 6f84343ee842f189fa8e1b9ac09de897846d4c08 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 09:13:00 +0000 Subject: [PATCH 14/25] Bump retina-analytics: lint fix Picks up offworldlabs/retina-analytics@67f1488 (ruff-format only, no behaviour change). Co-Authored-By: Claude Fable 5.1 --- libs/retina-analytics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/retina-analytics b/libs/retina-analytics index 2b83871e..67f1488d 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 2b83871e38cb17d9a2e5ceda3a55ceeaf1492f42 +Subproject commit 67f1488d5d06dd99dfa247cd7deb92bb5b59e01b From a6cff9f8d7a2b187533f2837d721bdd3a3e31de9 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 17:16:48 +0000 Subject: [PATCH 15/25] Tighten multinode DR budget and refresh the selected track's solve dots Two map complaints, both traceable to numbers chosen when a dark aircraft was re-solved every ~12 s. Dark solves now land every 1-3 s while a track is held, which changes what a gap means: it is a lost track, not a cadence gap, and every budget sized as an allowance for the old cadence now buys wrong pixels instead of coverage. Measured on the test droplet over 20 minutes (dark multinode feed entries against ground truth), median position error by solve age: 1.05 km under 3 s, 1.21 km at 3-8 s, 1.50 km at 8-15 s, 2.02 km at 15-30 s (7% more than 5 km off), 3.99 km at 30-60 s (12% of all displayed dark entries, 32% more than 5 km off). Dead reckoning is cut at that curve's two knees. MN_DR_CAP_S (15 s, was a hardcoded 30) is how far an entry is extrapolated before it holds its last point; MN_DARK_EXPIRY_S (30 s) is when a dark entry leaves the feed entirely. mn-adsb-* entries keep the 60 s expiry -- a transponder hex anchors them, so the same gap there is the ADS-B feed breathing. The frontend's matching budgets move with them: the dark icon-hide distance 6 km -> 3 km (6 km was 90%-drawable at the 12 s cadence; now it is the width of the error it hides) and the uncertainty disc's growth cap 60 s -> 30 s, which is the age past which a dark entry cannot exist. The selected track's per-solve dots came from a 30 s poll, so at a 1-3 s solve cadence they showed a decomposition of the track that was mostly missing. The poll is now MLAT_HISTORY_REFRESH_MS (3 s), and a fall in the selected entry's `seen` -- the feed announcing a fresh solve -- refetches inside it. newSolveArrived is the pure predicate for that, unit-tested. Co-Authored-By: Claude Fable 5.1 --- backend/.env.example | 13 +++- backend/config/constants.py | 34 +++++++- backend/services/aircraft_feed.py | 29 ++++--- backend/tests/test_feed_multinode.py | 78 ++++++++++++++++--- backend/tests/test_mn_lifetime.py | 14 ++-- docs/arc-display.md | 6 +- .../2026-09-05-solve-uncertainty-disc.md | 14 +++- docs/runbook.md | 3 +- docs/solverflow.md | 23 +++++- frontend/src/components/LiveAircraftMap.tsx | 33 ++++++-- frontend/src/components/map/constants.ts | 30 +++---- frontend/src/components/map/icons.test.ts | 8 +- frontend/src/components/map/index.ts | 1 + .../src/components/map/mlatHistory.test.ts | 43 ++++++++++ frontend/src/components/map/mlatHistory.ts | 52 +++++++++++++ .../src/components/map/uncertainty.test.ts | 14 ++-- frontend/src/components/map/uncertainty.ts | 15 +++- 17 files changed, 340 insertions(+), 70 deletions(-) create mode 100644 frontend/src/components/map/mlatHistory.test.ts create mode 100644 frontend/src/components/map/mlatHistory.ts diff --git a/backend/.env.example b/backend/.env.example index 20a107b4..ccf5ba44 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -121,7 +121,8 @@ MENDER_PAT= # SOLVER_RESOLVE_INTERVAL_S is how long an aircraft is not re-solved for. # Association is rate-limited per node, so every node that sees an aircraft # emits its own candidate for it and most arrivals are duplicates; this is the -# window that suppresses them. Do not exceed the 60 s multinode_tracks expiry — +# window that suppresses them. Do not exceed the multinode_tracks expiry +# (MN_DARK_EXPIRY_S below for dark tracks, 60 s for ADS-B-assisted ones) — # tracks would lapse between solves. 0 turns the suppression off. See # services/tasks/solver.py and docs/runbook.md's solver_latency_high entry. # @@ -129,9 +130,19 @@ MENDER_PAT= # from its association guess before it is rejected; the default is 6 km = 2 x # the 3 km association grid step, since a dark guess is a quantised grid point # rather than an ADS-B fix. ADS-B-anchored solves keep the fixed 2 km cap. +# +# MN_DR_CAP_S is how far a multinode entry is dead-reckoned past its last +# solve before it holds position, and MN_DARK_EXPIRY_S is when a DARK +# (mn-dark-*) entry is dropped from the feed entirely. Both are position-error +# budgets: measured against ground truth, dark entries run 1.5 km median error +# at 8-15 s of solve age but 2.0 km at 15-30 s and 4.0 km at 30-60 s. ADS-B- +# assisted (mn-adsb-*) entries keep the 60 s expiry — a transponder hex anchors +# them. Raise both to trade position accuracy for track continuity. # SOLVER_WORKERS=2 # SOLVER_RESOLVE_INTERVAL_S=12 # SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0 +# MN_DR_CAP_S=15.0 +# MN_DARK_EXPIRY_S=30.0 # Detection mirror. Production only. Every accepted v1 detection frame is # forwarded to another environment's /api/radar/detections/bulk, batched once a diff --git a/backend/config/constants.py b/backend/config/constants.py index ea788abf..a7576ff2 100644 --- a/backend/config/constants.py +++ b/backend/config/constants.py @@ -88,7 +88,8 @@ def as_num(v) -> float: # disables the gate. One-shot n=2 solves were the dominant ghost source. MN_N2_MIN_SOLVES = int(os.getenv("MN_N2_MIN_SOLVES", "2")) # One-shot display lifetime for n>=3 solves, seconds. A track confirmed by -# a second solve gets the normal 60 s entry expiry / 30 s DR cap. +# a second solve gets the normal MN_DARK_EXPIRY_S/60 s entry expiry and the +# MN_DR_CAP_S dead-reckoning cap below. # # The window has to outlive the wait for the confirmation it is waiting for, # or it is not a preview window — it is a guaranteed disappearance. A second @@ -97,11 +98,36 @@ def as_num(v) -> float: # cadence is a 9 s median and a 25 s p90. At 5 s the great majority of # genuine n>=3 one-shots blinked out before the round that would have # confirmed them ever ran, which reads on the map as flicker, not as caution. -# 15 s covers the median and most of the p90 while still being well short of -# the 60 s entry expiry, so an unconfirmed one-shot is still withdrawn long -# before a confirmed track would be. +# 15 s covers the median and most of the p90 while still being short of the +# entry expiry (MN_DARK_EXPIRY_S 30 s dark, 60 s assisted), so an unconfirmed +# one-shot is still withdrawn before a confirmed track would be. MN_ONESHOT_TTL_S = float(os.getenv("MN_ONESHOT_TTL_S", "15.0")) +# How far a multinode entry may be dead-reckoned past its last solve, seconds. +# Beyond this it holds its last dead-reckoned point until the entry expires. +# +# The cap is a position-error budget, not a cadence allowance. Measured on +# the test droplet over 20 minutes (dark multinode feed entries vs ground +# truth): median error 1.05 km under 3 s of solve age, 1.21 km at 3–8 s, +# 1.50 km at 8–15 s, then 2.02 km at 15–30 s (7% of entries more than 5 km +# off). The knee is at 15 s, which is where the KF's learned velocity error +# starts to dominate the solve error it is extrapolating. The old 30 s cap +# was set when a dark aircraft was re-solved every 12 s and the extra window +# bought coverage; dark solves now land every 1–3 s, so a 15 s gap is a lost +# track rather than a cadence gap, and extrapolating it only invents motion. +MN_DR_CAP_S = float(os.getenv("MN_DR_CAP_S", "15.0")) + +# Entry expiry for DARK multinode tracks (mn-dark-*), seconds. ADS-B-assisted +# entries (mn-adsb-*) keep the 60 s expiry: they are anchored to a transponder +# fix, so a gap there is the ADS-B feed breathing rather than a lost target. +# +# Same 20-minute capture: the 30–60 s age band was 12% of all displayed dark +# entries, with a 3.99 km median error and 32% more than 5 km off — an icon +# that reads as a live target while sitting kilometres from any aircraft. At +# the current 1–3 s dark solve cadence an entry that has not re-solved in 30 s +# is a lost track, and withdrawing it is more honest than holding it. +MN_DARK_EXPIRY_S = float(os.getenv("MN_DARK_EXPIRY_S", "30.0")) + # Quality gate for adopting the constant-velocity fit's velocity into a # published solve, in place of the single-epoch Doppler solution (see # solver.py's _resolve_cv_fit / velocity adoption in _process_solver_item). diff --git a/backend/services/aircraft_feed.py b/backend/services/aircraft_feed.py index 1befcf04..8acb85ad 100644 --- a/backend/services/aircraft_feed.py +++ b/backend/services/aircraft_feed.py @@ -17,6 +17,8 @@ ARC_REFRESH_S, CLAIMED_DISPLAY_FRESH_S, GT_REFRESH_S, + MN_DARK_EXPIRY_S, + MN_DR_CAP_S, MN_N2_MIN_SOLVES, MN_ONESHOT_TTL_S, STALE_TRACK_S, @@ -342,16 +344,24 @@ def build_combined_aircraft_json(default_pipeline: PassiveRadarPipeline) -> dict stale_mn = [] for key, r in list(state.multinode_tracks.items()): age_s = now - r.get("timestamp_ms", 0) / 1000 - if age_s > 60: + # Lane-aware expiry, on the same key-prefix truth multinode_to_aircraft + # reads adsb_assisted off. An assisted entry is anchored to a + # transponder hex, so a long gap is the ADS-B feed breathing and the + # historic 60 s still fits it. A dark entry has nothing holding it in + # place: at the current 1–3 s dark solve cadence a 30 s gap is a lost + # track, and the 30–60 s band measured 3.99 km median error (32% over + # 5 km) — a confident icon kilometres from any aircraft. + _expiry_s = 60.0 if key.startswith(_MN_ADSB_PREFIX) else MN_DARK_EXPIRY_S + if age_s > _expiry_s: stale_mn.append(key) continue # Display gates below are NOT staleness — a gated entry stays in # state.multinode_tracks so the next solve can confirm it (n=2) or - # supersede it, and only the age_s > 60 branch above discards its + # supersede it, and only the expiry branch above discards its # anomaly hex. A one-shot solve renders nothing at all: a 2-node # track needs a second solve to prove it isn't a mirror-point ghost, # and a 3+-node one-shot gets a short preview window instead of the - # full 60 s entry lifetime before it either confirms or expires. + # full entry lifetime before it either confirms or expires. solve_count = int(r.get("solve_count") or 1) if r.get("n_nodes") == 2 and solve_count < MN_N2_MIN_SOLVES: continue @@ -359,19 +369,18 @@ def build_combined_aircraft_json(default_pipeline: PassiveRadarPipeline) -> dict continue ac = multinode_to_aircraft(key, r) # Dead-reckon position using solver velocity (vel_east/vel_north in - # m/s), capped at 30 s: beyond that a velocity error dominates any - # solve accuracy (a 15 m/s error is already 450 m of drift at the - # cap), so an old solve holds its last dead-reckoned point until the - # 60 s entry expiry rather than drifting further. + # m/s), capped at MN_DR_CAP_S: beyond that a velocity error dominates + # any solve accuracy, so an old solve holds its last dead-reckoned + # point until the entry expiry above rather than drifting further. ts_fix = r.get("timestamp_ms", 0) / 1000.0 - elapsed = min(now - ts_fix, 30.0) + elapsed = min(now - ts_fix, MN_DR_CAP_S) vel_east_m_s = r.get("vel_east", 0.0) vel_north_m_s = r.get("vel_north", 0.0) # TRACK_DR_SOURCE, read per call like TRACK_SMOOTHER: "kf" (default) # dead-reckons with the display filter's LEARNED velocity when one # exists — the solved velocity this block used to trust was measured - # (2026-08-09, n=93) at median 127 m/s vector error, i.e. ~3.8 km of - # drift at the 30 s cap below, worse than the solve error itself. + # (2026-08-09, n=93) at median 127 m/s vector error, i.e. ~1.9 km of + # drift at the MN_DR_CAP_S cap, worse than the solve error itself. # "solve" restores the old behaviour (rollback, env only). The KF # accessor returns None whenever the KF never saw this key (smoother # in ewma/off mode, first solve, TTL-swept) so the fallback below is diff --git a/backend/tests/test_feed_multinode.py b/backend/tests/test_feed_multinode.py index 9fb4ab28..2cfe0d0a 100644 --- a/backend/tests/test_feed_multinode.py +++ b/backend/tests/test_feed_multinode.py @@ -1,9 +1,11 @@ """Feed-side handling of multinode solver tracks. Covers the dead-reckoning of mn-* entries in build_combined_aircraft_json: -position is advanced with the solved velocity, but only up to a 30 s horizon — +position is advanced with the solved velocity, but only up to MN_DR_CAP_S — past that a velocity error dominates any solve accuracy, so an old solve holds -its last dead-reckoned point until the 60 s entry expiry. +its last dead-reckoned point until the entry expires. Expiry is lane-aware: +MN_DARK_EXPIRY_S for a dark (mn-dark-*) entry, the historic 60 s for an +ADS-B-assisted (mn-adsb-*) one, whose transponder hex anchors it. """ import math @@ -16,6 +18,7 @@ os.environ.setdefault("RETINA_ENV", "test") os.environ.setdefault("RADAR_API_KEY", "test-key-abc123") +from config.constants import MN_DARK_EXPIRY_S, MN_DR_CAP_S # noqa: E402 from core import state # noqa: E402 from services import track_filter # noqa: E402 from services.geo import offset_latlon_m # noqa: E402 @@ -68,15 +71,72 @@ def test_fresh_entry_is_dead_reckoned_fully(self): exp_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * 10.0) assert ac["lat"] == pytest.approx(exp_lat, abs=2e-4) - def test_dr_horizon_is_capped_at_30s(self): - # 45 s old (younger than the 60 s expiry): advanced 30 s worth of - # motion, not 45. - state.multinode_tracks["mn-dark-x"] = _mn_entry(age_s=45.0, vel_north=100.0) + def test_dr_horizon_is_capped_at_mn_dr_cap_s(self): + # 20 s old (still inside the 30 s dark expiry, so this exercises the + # DR cap and not staleness): advanced MN_DR_CAP_S worth of motion, + # not 20 s worth. + state.multinode_tracks["mn-dark-x"] = _mn_entry(age_s=20.0, vel_north=100.0) ac = self._build_mn() - capped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * 30.0) - uncapped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * 45.0) + capped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * MN_DR_CAP_S) + uncapped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * 20.0) assert ac["lat"] == pytest.approx(capped_lat, abs=2e-4) - assert abs(ac["lat"] - uncapped_lat) > 5e-3 + # The 5 s the cap withheld is 500 m of northing, ~0.0045 deg of lat. + assert abs(ac["lat"] - uncapped_lat) > 3e-3 + + def test_assisted_entry_past_the_cap_holds_its_dead_reckoned_point(self): + # The assisted lane lives 60 s, so it reaches ages the dark lane never + # does — and past MN_DR_CAP_S it must hold, not keep extrapolating. + state.multinode_tracks["mn-adsb-abc123"] = _mn_entry(age_s=45.0, vel_north=100.0) + ac = self._build_mn() + capped_lat, _ = offset_latlon_m(LAT, LON, east_m=0.0, north_m=100.0 * MN_DR_CAP_S) + assert ac["lat"] == pytest.approx(capped_lat, abs=2e-4) + + +class TestMultinodeLaneExpiry: + """Dark entries expire at MN_DARK_EXPIRY_S, assisted ones at 60 s. + + A dark solve now lands every 1-3 s while the aircraft is tracked, so an + entry that has not re-solved in 30 s is a lost track rather than a cadence + gap; holding it drew an icon a measured 3.99 km (median) from any + aircraft. An mn-adsb-* entry is anchored to a transponder hex, so the + same gap is the ADS-B feed breathing and it keeps the historic 60 s. + """ + + @pytest.fixture(autouse=True) + def _clean_state(self): + state.multinode_tracks.clear() + state.track_histories.clear() + yield + state.multinode_tracks.clear() + state.track_histories.clear() + + def _mn_hexes(self): + from services.frame_processor import build_combined_aircraft_json + + pipeline = types.SimpleNamespace(geolocated_tracks={}, config={}) + result = build_combined_aircraft_json(pipeline) + return [a["hex"] for a in result["aircraft"] if a.get("multinode")] + + def test_dark_entry_inside_the_dark_expiry_is_rendered(self): + state.multinode_tracks["mn-dark-x"] = _mn_entry(age_s=MN_DARK_EXPIRY_S - 5.0, vel_north=100.0) + assert len(self._mn_hexes()) == 1 + assert "mn-dark-x" in state.multinode_tracks + + def test_dark_entry_past_the_dark_expiry_is_dropped(self): + state.multinode_tracks["mn-dark-x"] = _mn_entry(age_s=MN_DARK_EXPIRY_S + 1.0, vel_north=100.0) + assert self._mn_hexes() == [] + # Expiry, not a display gate: the entry leaves the store entirely. + assert "mn-dark-x" not in state.multinode_tracks + + def test_assisted_entry_at_the_same_age_survives(self): + state.multinode_tracks["mn-adsb-abc123"] = _mn_entry(age_s=MN_DARK_EXPIRY_S + 1.0, vel_north=100.0) + assert len(self._mn_hexes()) == 1 + assert "mn-adsb-abc123" in state.multinode_tracks + + def test_assisted_entry_still_expires_at_60s(self): + state.multinode_tracks["mn-adsb-abc123"] = _mn_entry(age_s=61.0, vel_north=100.0) + assert self._mn_hexes() == [] + assert "mn-adsb-abc123" not in state.multinode_tracks class TestMultinodeDeadReckonSource: diff --git a/backend/tests/test_mn_lifetime.py b/backend/tests/test_mn_lifetime.py index 5b648ca1..d3f8bd49 100644 --- a/backend/tests/test_mn_lifetime.py +++ b/backend/tests/test_mn_lifetime.py @@ -2,8 +2,8 @@ Two problems in how state.multinode_tracks renders as mn-* aircraft: -- a track solved exactly once dead-reckons for the full 60 s entry lifetime - (30 s DR cap) even though nothing has confirmed it is a real aircraft and +- a track solved exactly once dead-reckons for the full entry lifetime + (MN_DR_CAP_S DR cap) even though nothing has confirmed it is a real aircraft and not a mirror-point/wrong-frame ghost. solver.py now stamps a solve_count onto every published entry, and aircraft_feed.py withholds rendering until a 2-node track has MN_N2_MIN_SOLVES solves, and caps an n>=3 one-shot's @@ -425,14 +425,18 @@ def test_n3_one_shot_within_ttl_is_rendered(self): assert len(self._mn_aircraft()) == 1 def test_n3_one_shot_past_ttl_is_not_rendered_but_not_expired(self): - # Past MN_ONESHOT_TTL_S (15 s) but well inside the 60 s entry expiry, - # so this exercises the display gate and not staleness. + # Past MN_ONESHOT_TTL_S (15 s) but inside the dark lane's + # MN_DARK_EXPIRY_S (30 s), so this exercises the display gate and not + # staleness. state.multinode_tracks["k"] = _mn_entry(n_nodes=3, age_s=20.0, solve_count=1) assert self._mn_aircraft() == [] assert "k" in state.multinode_tracks def test_n3_confirmed_old_entry_is_rendered(self): - state.multinode_tracks["k"] = _mn_entry(n_nodes=3, age_s=40.0, solve_count=2) + # 25 s: old enough to be past every display gate, young enough to sit + # inside the dark lane's MN_DARK_EXPIRY_S — the key here is bare "k", + # which is not mn-adsb-* and so expires on the dark budget. + state.multinode_tracks["k"] = _mn_entry(n_nodes=3, age_s=25.0, solve_count=2) assert len(self._mn_aircraft()) == 1 def test_n2_gate_disabled_via_min_solves_1(self, monkeypatch): diff --git a/docs/arc-display.md b/docs/arc-display.md index 3eb17ead..8afb87cd 100644 --- a/docs/arc-display.md +++ b/docs/arc-display.md @@ -90,9 +90,9 @@ Geometry is frozen at creation; only style refreshes. Opacity fades linearly over `ARC_FADE_MS` (5 s), after which the buffer entry is pruned. Arc-only tracks dead-reckon their reference position for at most -`ARC_DR_MAX_S` (10 s, vs 60 s for solved tracks) — the backend pins their -position to the arc midpoint, so a long glide walks the reference off the -measured locus. +`ARC_DR_MAX_S` (10 s, vs `MN_DR_CAP_S` 15 s for solved tracks) — the backend +pins their position to the arc midpoint, so a long glide walks the reference +off the measured locus. Selecting an arc track (from the list panel or by clicking the arc) highlights its arcs in amber, draws the detecting node's beam wedge, and centers the map diff --git a/docs/design-notes/2026-09-05-solve-uncertainty-disc.md b/docs/design-notes/2026-09-05-solve-uncertainty-disc.md index 4bc7a6a3..2121072b 100644 --- a/docs/design-notes/2026-09-05-solve-uncertainty-disc.md +++ b/docs/design-notes/2026-09-05-solve-uncertainty-disc.md @@ -90,14 +90,20 @@ gain only to let a genuinely ill-conditioned solve show a bigger disc. ### Growth while dead-reckoning -The feed dead-reckons a solve forward up to 30 s with the KF's learned -velocity, then holds; the frontend continues its own DR up to 60 s. The disc -grows accordingly: +The feed dead-reckons a solve forward up to `MN_DR_CAP_S` with the KF's +learned velocity, then holds; the frontend continues its own DR up to +`UNCERTAINTY_DR_CAP_S`. The disc grows accordingly: ``` -σ(t) = sqrt( σ_solve² + (σ_v · min(t, 60))² ) +σ(t) = sqrt( σ_solve² + (σ_v · min(t, UNCERTAINTY_DR_CAP_S))² ) ``` +Both caps were halved on 2026-09-05 (30 → 15 s and 60 → 30 s) once dark solves +started landing every 1–3 s: a dark entry no longer survives to 60 s at all +(`MN_DARK_EXPIRY_S` = 30 s), so the second half of the old growth curve +described entries that cannot exist. See `docs/solverflow.md` for the +error-vs-solve-age measurement the two budgets are cut from. + `t` is the age of the solve at draw time, `σ_v` the velocity sigma: `track_filter.learned_velocity(key)[2]` when the KF has state for the key (clamped to [5, 150] m/s), else `SOLVE_SIGMA_VEL_DEFAULT_MS = 25`. A fresh diff --git a/docs/runbook.md b/docs/runbook.md index c2d582b2..31406cdd 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -396,7 +396,8 @@ recognised as such and dropped without solving. `SOLVER_RESOLVE_INTERVAL_S` (default 12 s) is the window an aircraft is not re-solved in. Raising it trades map refresh rate for solver headroom — do not go -past the 60 s `multinode_tracks` expiry, or tracks will lapse between solves. `0` +past the `multinode_tracks` expiry (`MN_DARK_EXPIRY_S`, 30 s, for `mn-dark-*` +entries; 60 s for `mn-adsb-*` ones), or tracks will lapse between solves. `0` turns the suppression off. --- diff --git a/docs/solverflow.md b/docs/solverflow.md index c626e5bf..2dd51455 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -510,11 +510,13 @@ flowchart TD archive --> histpub["_record_solve_history: published"] histpub --> feed["build_combined_aircraft_json
(1 Hz flush)"] - feed --> gN2{"n=2 display gate:
solve_count < MN_N2_MIN_SOLVES 2?"} + feed --> gExpiry{"entry expiry:
mn-adsb-* age_s > 60s,
mn-dark-* age_s >
MN_DARK_EXPIRY_S 30.0s?"} + gExpiry -->|"yes"| dropExpired["popped from
state.multinode_tracks,
anomaly hex discarded"]:::inert + gExpiry -->|"no"| gN2{"n=2 display gate:
solve_count < MN_N2_MIN_SOLVES 2?"} gN2 -->|"yes"| retainN2["retained, not rendered"]:::inert gN2 -->|"no"| gOneshot{"n>=3 one-shot:
solve_count==1 AND
age_s > MN_ONESHOT_TTL_S 15.0s?"} gOneshot -->|"yes"| dropOneshot["not rendered"]:::inert - gOneshot -->|"no"| dr["dead reckoning,
capped 30s"] + gOneshot -->|"no"| dr["dead reckoning,
capped MN_DR_CAP_S 15.0s,
then holds"] dr --> dedup["dedup_aircraft:
rank by _DEDUP_SOURCE_RANK,
3.0km / 2000ft gate"] dedup --> out["aircraft.json + WebSocket -> map"] @@ -540,6 +542,8 @@ flowchart TD | `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py:77` | | `MN_N2_MIN_SOLVES` | 2 | `config/constants.py:63` | | `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py:66` | +| `MN_DR_CAP_S` (dead-reckoning horizon past the last solve; the entry then holds its last DR'd point) | 15.0 s | `config/constants.py` | +| `MN_DARK_EXPIRY_S` (entry expiry, `mn-dark-*` only — `mn-adsb-*` keeps 60 s) | 30.0 s | `config/constants.py` | | `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py:37-43` | | `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py:131-139` | | Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py:49-50` | @@ -565,6 +569,21 @@ flowchart TD `mn_superseded` / `mn_superseded_blocked` in `/api/test/solver-stats` (`fragmentation`) and `superseded_keys` / `superseded_blocked` on each published `mlat_solve_history` record are how this is watched. +- **A displayed dark entry is only as good as its solve age, and the budgets + say so.** Measured on the test droplet over 20 minutes (dark multinode feed + entries against ground truth): median position error 1.05 km under 3 s of + solve age, 1.21 km at 3–8 s, 1.50 km at 8–15 s, 2.02 km at 15–30 s (7% more + than 5 km off), and 3.99 km at 30–60 s (12% of all displayed dark entries, + 32% more than 5 km off). `MN_DR_CAP_S` (15 s) and `MN_DARK_EXPIRY_S` (30 s) + are cut at that curve's two knees. Both were looser — a 30 s cap and a flat + 60 s expiry — from when the solver refused to re-solve the same tracks inside + `SOLVER_RESOLVE_INTERVAL_S` = 12 s and the extra window bought coverage; a + dark aircraft now re-solves every 1–3 s while it is tracked, so a 15 s gap is + a lost track rather than a cadence gap and extrapolating it only invents + motion. `mn-adsb-*` entries keep the 60 s expiry: a transponder hex anchors + them, so the same gap is the ADS-B feed breathing. The frontend's matching + budgets are `DR_ICON_HIDE_DISTANCE_DARK_M` (3 km) and `UNCERTAINTY_DR_CAP_S` + (30 s). - **Node-trust residuals are measure-only.** `node_bias.py` computes them but nothing in the solver consumes them yet (`node_bias.py:33-40` docstring). - **`docs/pipeline.md` §3 is stale.** It predates the known lane and the diff --git a/frontend/src/components/LiveAircraftMap.tsx b/frontend/src/components/LiveAircraftMap.tsx index b0ef25e9..727b6e8e 100644 --- a/frontend/src/components/LiveAircraftMap.tsx +++ b/frontend/src/components/LiveAircraftMap.tsx @@ -21,6 +21,8 @@ import { GT_PRUNE_GRACE_MS, POSITION_SOURCE_ARC_ONLY, ARC_DR_MAX_S, + MLAT_HISTORY_REFRESH_MS, + newSolveArrived, groundTruthKey, applyGroundTruthFixes, pruneGroundTruthFixes, @@ -1520,12 +1522,16 @@ export default function LiveAircraftMap() { : null; // Per-solve history for the selected MLAT track (debug): fetched once per - // selection + refreshed on the backend's ~30 s recording cadence. Tagged - // with the hex it was fetched for so a selection change never shows the - // previous track's solves while the new fetch is in flight. + // selection, then polled every MLAT_HISTORY_REFRESH_MS. Tagged with the hex + // it was fetched for so a selection change never shows the previous track's + // solves while the new fetch is in flight. const selectedMnHex = selectedAc?.position_source === "multinode_solve" ? selectedAc.hex : null; + const selectedMnSeen = selectedMnHex ? selectedAc?.seen ?? null : null; const [mlatHistory, setMlatHistory] = useState(null); + // The poll's loader, published for the `seen` watcher below to call. A ref + // rather than a dependency so a refetch never restarts the interval. + const reloadMlatHistoryRef = useRef(null); useEffect(() => { if (!selectedMnHex) { setMlatHistory(null); @@ -1537,11 +1543,28 @@ export default function LiveAircraftMap() { if (!cancelled && d && d.hex === selectedMnHex) setMlatHistory(d); }); }; + reloadMlatHistoryRef.current = load; load(); - const interval = setInterval(load, 30000); - return () => { cancelled = true; clearInterval(interval); }; + const interval = setInterval(load, MLAT_HISTORY_REFRESH_MS); + return () => { + cancelled = true; + clearInterval(interval); + reloadMlatHistoryRef.current = null; + }; }, [selectedMnHex]); + // A fall in `seen` is the feed announcing a fresh solve for this track — the + // one event worth a fetch off the poll's schedule (see newSolveArrived). + // Dark solves arrive every 1-3 s while a track is held, faster than the + // poll, and the dots are the surface someone selected the aircraft to read. + const prevMnSeenRef = useRef({ hex: null, seen: null }); + useEffect(() => { + const prev = prevMnSeenRef.current; + const next = { hex: selectedMnHex, seen: selectedMnSeen }; + prevMnSeenRef.current = next; + if (newSolveArrived(prev, next)) reloadMlatHistoryRef.current?.(); + }, [selectedMnHex, selectedMnSeen]); + // Nodes with a live detection of the selected simulated object — read from // the detection-presence oracle (per-aircraft signals ∪ the detecting_nodes // feed key). trailTick advances on every ingest, so this refreshes at the diff --git a/frontend/src/components/map/constants.ts b/frontend/src/components/map/constants.ts index 15295401..84486a16 100644 --- a/frontend/src/components/map/constants.ts +++ b/frontend/src/components/map/constants.ts @@ -71,10 +71,10 @@ export const ADSB_SINGLE_ARC_ICON_MULTIPLE = 2.5; export const ARC_DR_MAX_S = 10; // Dead-reckoning drift budget (metres) past which the plane ICON stops being -// drawn as an ordinary live target. The backend keeps feeding an mn entry for -// 60 s after its last solve, dead-reckoned the whole way, so a target whose -// solves stop is drawn kilometres from where it actually is — the icon reads as -// a real target because nothing about it looks stale. +// drawn as an ordinary live target. The backend keeps feeding an mn entry past +// its last solve (MN_DARK_EXPIRY_S 30 s dark, 60 s ADS-B-assisted), so a target +// whose solves stop is drawn kilometres from where it actually is — the icon +// reads as a real target because nothing about it looks stale. // // The budget is LANE-AWARE, because the two multi-node lanes re-solve at very // different rates and one budget cannot describe both: @@ -86,24 +86,28 @@ export const ARC_DR_MAX_S = 10; // displacement gate (_MAX_DISPLACEMENT_KM) and sits under backend dedup's // 3 km proximity gate, so a second icon cannot appear at the true position // while the drifted one is still shown. -// * Dark lane (mn-dark-*) — 6 km. 2 km is a budget the dark lane cannot meet -// at all: the solver refuses to re-solve the same tracks inside -// SOLVER_RESOLVE_INTERVAL_S = 12 s, and 2000 m / 12 s is 324 kt while -// simulated dark speeds run 233–525 kt. Measured dark cadence is 9.0 s -// median / 24.9 s p90 with ~3 s of publish latency on top, and a 26-minute -// capture had 48% of published dark track-frames hidden by the 2 km gate — -// 80% would be drawable at 4 km, 90% at 6 km. +// * Dark lane (mn-dark-*) — 3 km. It was 6 km, sized when the solver refused +// to re-solve the same tracks inside SOLVER_RESOLVE_INTERVAL_S = 12 s and +// the measured dark cadence was 9.0 s median / 24.9 s p90: at that cadence +// 2 km hid 48% of published dark track-frames and only 6 km got 90% of them +// drawn. Dark solves now land every 1–3 s, so the drift a healthy track +// accrues is metres, and the budget stopped buying coverage and started +// buying wrong icons: measured against ground truth over 20 minutes, dark +// entries run 1.50 km median error at 8–15 s of solve age but 2.02 km at +// 15–30 s (7% over 5 km) and 3.99 km at 30–60 s (32% over 5 km). 3 km sits +// at that knee — it still tolerates the odd missed solve, and it no longer +// draws a confident icon 6 km from any aircraft. // // Exceeding the budget does not mean the same thing in both lanes, so neither // does the rendering (see drIconState in icons.ts): an assisted track over // budget is a genuine anomaly and loses its icon, while a dark track over -// budget is the normal consequence of a 12 s re-solve floor and is drawn in a +// budget is the normal consequence of a missed solve and is drawn in a // degraded "stale solve" style instead — "solved but stale" has to stay // distinguishable from "not solved". Either way the TRACK stays alive — // stores, trails, list, selection — so a new solve restores the normal icon on // the next 2 Hz render. export const DR_ICON_HIDE_DISTANCE_M = 2000; -export const DR_ICON_HIDE_DISTANCE_DARK_M = 6000; +export const DR_ICON_HIDE_DISTANCE_DARK_M = 3000; // Ground speed (knots) assumed when a multi-node entry carries no `gs` at all. // The backend deletes gs from entries whose velocity vector it does not trust diff --git a/frontend/src/components/map/icons.test.ts b/frontend/src/components/map/icons.test.ts index c9ab0047..b6f4cc11 100644 --- a/frontend/src/components/map/icons.test.ts +++ b/frontend/src/components/map/icons.test.ts @@ -139,15 +139,17 @@ describe("lane-aware drift budget", () => { expect(isDarkMultinodeSolve(null)).toBe(false); }); - it("keeps a dark solve inside budget across one 12 s re-solve floor", () => { + it("keeps a dark solve inside budget across a missed solve", () => { // 450 kt for 12 s is ~2.8 km: over the 2 km budget the old gate applied to - // every lane, comfortably inside the dark one. This is the 48%-hidden bug. + // every lane, still inside the dark one. This is the 48%-hidden bug. expect(hideDrIcon({ ...dark(), seen: 12 }, NOW)).toBe(false); expect(hideDrIcon({ ...assisted(), seen: 12 }, NOW)).toBe(true); }); it("still trips the dark budget on a long enough gap", () => { - // 450 kt for 30 s is ~6.9 km — past 6 km. + // 450 kt for 15 s is ~3.5 km — past the 3 km dark budget. Under the old + // 6 km one this same entry was drawn as an ordinary live target. + expect(hideDrIcon({ ...dark(), seen: 15 }, NOW)).toBe(true); expect(hideDrIcon({ ...dark(), seen: 30 }, NOW)).toBe(true); }); }); diff --git a/frontend/src/components/map/index.ts b/frontend/src/components/map/index.ts index b19b5bac..9604040d 100644 --- a/frontend/src/components/map/index.ts +++ b/frontend/src/components/map/index.ts @@ -19,6 +19,7 @@ export { solveSigmaM, solveUncertaintyRadiusM, } from "./uncertainty"; +export { MLAT_HISTORY_REFRESH_MS, newSolveArrived } from "./mlatHistory"; export { mergeTrailPositions, sampleTrailPositions, buildTrailSegments } from "./trails"; export { PLANE_PATH, getAircraftColor, altitudeColor, ALTITUDE_LEGEND, aircraftIconSize, makeAircraftIcon, makeDroneIcon, nodeIcon, drDriftM, drGsKt, drIconBudgetM, drIconState, hideDrIcon, isDarkMultinodeSolve, isMultinodeSolve } from "./icons"; export { FitBounds, ViewportTracker, MapClickClear } from "./MapControls"; diff --git a/frontend/src/components/map/mlatHistory.test.ts b/frontend/src/components/map/mlatHistory.test.ts new file mode 100644 index 00000000..be564f86 --- /dev/null +++ b/frontend/src/components/map/mlatHistory.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect } from "vitest"; +import { MLAT_HISTORY_REFRESH_MS, newSolveArrived } from "./mlatHistory"; + +const at = (hex: string | null, seen: number | null) => ({ hex, seen }); + +describe("MLAT_HISTORY_REFRESH_MS", () => { + it("keeps up with the 1-3 s dark solve cadence", () => { + expect(MLAT_HISTORY_REFRESH_MS).toBe(3_000); + }); +}); + +describe("newSolveArrived", () => { + it("fires when seen falls on the same track", () => { + // 8 s old, then 0.4 s old: a solve landed between the two flushes. + expect(newSolveArrived(at("mnabc123", 8), at("mnabc123", 0.4))).toBe(true); + }); + + it("does not fire while the same solve simply ages", () => { + expect(newSolveArrived(at("mnabc123", 2), at("mnabc123", 5))).toBe(false); + expect(newSolveArrived(at("mnabc123", 2), at("mnabc123", 2))).toBe(false); + }); + + it("does not compare ages across a selection change", () => { + // The new track's seen is a different clock, and the selection effect + // refetches from scratch anyway. + expect(newSolveArrived(at("mnabc123", 30), at("mndef456", 1))).toBe(false); + expect(newSolveArrived(at(null, null), at("mnabc123", 1))).toBe(false); + }); + + it("does not fire on a deselection", () => { + expect(newSolveArrived(at("mnabc123", 8), at(null, null))).toBe(false); + }); + + it("needs an age on both sides", () => { + expect(newSolveArrived(at("mnabc123", null), at("mnabc123", 1))).toBe(false); + expect(newSolveArrived(at("mnabc123", 8), at("mnabc123", null))).toBe(false); + }); + + it("treats a zeroed age as a new solve", () => { + // The backend clamps a negative age to 0, so 0 is a real value here. + expect(newSolveArrived(at("mnabc123", 4), at("mnabc123", 0))).toBe(true); + }); +}); diff --git a/frontend/src/components/map/mlatHistory.ts b/frontend/src/components/map/mlatHistory.ts new file mode 100644 index 00000000..7cac1b5f --- /dev/null +++ b/frontend/src/components/map/mlatHistory.ts @@ -0,0 +1,52 @@ +/* ------------------------------------------------------------------ */ +/* Refresh policy for the selected track's per-solve history */ +/* ------------------------------------------------------------------ */ + +/** + * When LiveAircraftMap should re-read `/api/test/mlat-history` for the + * multi-node aircraft the operator has selected. That payload draws + * MlatSolveHistoryLayer's per-solve dots and the detail panel's solve table — + * the surface someone selects an aircraft in order to read, so it lagging the + * marker is the whole bug. + * + * Two rules, because a poll alone cannot be both cheap and prompt: a floor + * interval that runs regardless, and an event that fires inside it. No + * Leaflet, no React — unit-tested on its own. + */ + +/** + * Poll interval for the selected track's solve history, in ms. + * + * It was 30 s, sized when the solver refused to re-solve the same tracks + * inside SOLVER_RESOLVE_INTERVAL_S = 12 s, so a poll could not miss much. + * Dark solves now land every 1–3 s while a track is held, and at 30 s the dots + * trailed the live marker by up to half a minute — a decomposition of the + * track that was mostly missing. 3 s tracks that cadence at ~20 requests a + * minute for ONE selected track, and only while something is selected. + */ +export const MLAT_HISTORY_REFRESH_MS = 3_000; + +/** The two fields that decide a refetch: which track, and how old its last + * solve was when the feed last said. */ +export interface SelectedSolveAge { + /** Feed hex of the selected multi-node entry, or null when none is. */ + hex: string | null; + /** `seen` from that entry — the backend's age of its last solve, seconds. */ + seen: number | null; +} + +/** + * True when the feed has just announced a fresh solve for the SAME selected + * track, so its history is worth refetching ahead of the next poll. + * + * `seen` is an age, so it climbs on every flush and only ever falls when a new + * solve replaced the one it was measuring — that fall is the event. A change + * of `hex` is not a comparison at all but a different clock, and the selection + * effect refetches from scratch anyway; a null on either side is a track that + * is not a multi-node solve, or a feed entry that never carried an age. + */ +export function newSolveArrived(prev: SelectedSolveAge, next: SelectedSolveAge): boolean { + if (prev.hex !== next.hex || next.hex == null) return false; + if (prev.seen == null || next.seen == null) return false; + return next.seen < prev.seen; +} diff --git a/frontend/src/components/map/uncertainty.test.ts b/frontend/src/components/map/uncertainty.test.ts index 0a3abfa5..ddb38af7 100644 --- a/frontend/src/components/map/uncertainty.test.ts +++ b/frontend/src/components/map/uncertainty.test.ts @@ -50,11 +50,13 @@ describe("solveSigmaM", () => { ); }); - it("stops growing at the 60 s dead-reckoning cap", () => { + it("stops growing at the dead-reckoning cap", () => { + // The cap tracks the backend's dark-lane entry expiry (MN_DARK_EXPIRY_S). + expect(UNCERTAINTY_DR_CAP_S).toBe(30); const ac = mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25 }); const atCap = solveSigmaM(ac, UNCERTAINTY_DR_CAP_S); expect(solveSigmaM(ac, 600)).toBe(atCap); - expect(atCap).toBeCloseTo(Math.sqrt(200 * 200 + 1500 * 1500), 6); + expect(atCap).toBeCloseTo(Math.sqrt(200 * 200 + 750 * 750), 6); }); it("clamps a negative age to the solve epoch", () => { @@ -106,21 +108,21 @@ describe("solveUncertaintyRadiusM", () => { ).toBeCloseTo(UNCERTAINTY_K95 * 200, 6); }); - it("grows while dead-reckoning, then holds at the 60 s cap", () => { + it("grows while dead-reckoning, then holds at the cap", () => { const ac = mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25, _updatedAt: NOW - 10_000 }); const fresh = solveUncertaintyRadiusM(mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25 }), NOW); const aged = solveUncertaintyRadiusM(ac, NOW); expect(aged).toBeGreaterThan(fresh); // Past the cap the radius stops moving. - const at60 = solveUncertaintyRadiusM( - mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25, _updatedAt: NOW - 60_000 }), + const atCap = solveUncertaintyRadiusM( + mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25, _updatedAt: NOW - UNCERTAINTY_DR_CAP_S * 1000 }), NOW, ); const at300 = solveUncertaintyRadiusM( mn({ pos_sigma_m: 200, pos_sigma_vel_ms: 25, _updatedAt: NOW - 300_000 }), NOW, ); - expect(at300).toBe(at60); + expect(at300).toBe(atCap); }); it("caps the radius at 10 km", () => { diff --git a/frontend/src/components/map/uncertainty.ts b/frontend/src/components/map/uncertainty.ts index 530e9bda..55383b1f 100644 --- a/frontend/src/components/map/uncertainty.ts +++ b/frontend/src/components/map/uncertainty.ts @@ -22,9 +22,16 @@ import type { Aircraft } from "../../types"; /** Rayleigh 95% radius factor (k_50 = 1.177 CEP, k_68 = 1.510). */ export const UNCERTAINTY_K95 = 2.4477; -/** Dead-reckoning growth is capped here: past 60 s the icon itself is stale - * and a disc that kept growing would just be a claim about nothing. */ -export const UNCERTAINTY_DR_CAP_S = 60; +/** Dead-reckoning growth is capped here: past this the icon itself is stale + * and a disc that kept growing would just be a claim about nothing. + * + * 30 s, down from 60 s, to match the backend's dark-lane entry expiry + * (`MN_DARK_EXPIRY_S`): a dark entry no longer survives to 60 s at all, so + * the second half of the old growth curve described entries that cannot + * exist. Assisted entries do live to 60 s, but they are anchored to a + * transponder fix rather than extrapolated, so growing their disc past the + * point the dark budget stops is not the honest reading either. */ +export const UNCERTAINTY_DR_CAP_S = 30; /** Hard ceiling on the drawn radius. A degenerate solve (near-parallel * baselines) can report an astronomically large formal sigma; without a cap @@ -58,7 +65,7 @@ export function solveAgeS(ac: UncertaintyEntry | null | undefined, nowMs: number /** * Per-axis position sigma in metres at age `ageS`: - * `sqrt(pos_sigma_m² + (pos_sigma_vel_ms · min(age, 60))²)`. + * `sqrt(pos_sigma_m² + (pos_sigma_vel_ms · min(age, UNCERTAINTY_DR_CAP_S))²)`. * * Returns null when the entry carries no usable `pos_sigma_m` — an older * backend, or a solve whose node count was unknown. A missing or non-finite From 3f7f97467aeb04c6d4abeb56a293a37a1b9872b7 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 17:17:40 +0000 Subject: [PATCH 16/25] Reject non-PSD solve covariance before it poisons the display filter The test droplet threw 91 ValueError: math domain error tracebacks in 40 minutes out of learned_velocity's sqrt, from two callers that each lose real work: solver.py's multinode_key_decision (that solve is dropped) and aircraft_feed's multinode_to_aircraft (the whole feed broadcast for the tick is dropped). One sick track was taking down the map. Root cause is upstream of the sqrt. cov_en_km2 is the top-left 2x2 of s2 * inv(JtJ) for the solver's 5-state fit, and the solver falls back to pinv only on an outright LinAlgError -- an ill-conditioned-but-singular JtJ (near-parallel baselines, the degenerate tail that puts the formal sigma's p99 at 3.8e6 km) inverts to numerical garbage that is INDEFINITE while still having both diagonals positive. _measurement_R checked only shape, finiteness and positive diagonals, so that matrix became R. An indefinite R is not survivable: _kf_correct's Joseph form preserves PSD for any gain, but only GIVEN PSD P and R -- its K R K^T term inherits R's negative eigenvalue -- and _init_entry seeds P's position block straight from R, so a sick covariance poisons the filter at birth as well as on every update. Once P's velocity diagonals go negative, learned_velocity's sqrt raises. Three changes, defence in depth: - _measurement_R now requires the cov to be positive-semidefinite (both diagonals > 0 AND determinant >= 0), falling back to the base floor otherwise. That is already this function's documented answer for a degenerate cov, and a covariance this broken carries no trustworthy relative weighting worth repairing by eigenvalue clipping. - The predict step re-symmetrises P, as _kf_correct already did. - learned_velocity clamps its sqrt at 0, so a pathological state degrades to "sigma 0" rather than taking out a solve or a broadcast. The feed's per-entry build is now failure-isolated for the same reason: one entry raising logs once a minute and skips that aircraft instead of aborting the whole tick. Measured on origin/main across a 180-point sweep of indefinite covariances (correlation 1.2-10, formal variance 0.01-100 km^2, 1.4/5/20 s cadence, 0/100/250 m/s): 89 of 180 reproduce the exact math domain error through the public API, the fastest in 3 solves. All 180 pass after this change, with the worst P diagonal over the sweep at +71. Co-Authored-By: Claude Fable 5.1 --- backend/services/aircraft_feed.py | 117 ++++++++++++++++++++++-------- backend/services/track_filter.py | 54 +++++++++++++- 2 files changed, 136 insertions(+), 35 deletions(-) diff --git a/backend/services/aircraft_feed.py b/backend/services/aircraft_feed.py index 1befcf04..a132ef15 100644 --- a/backend/services/aircraft_feed.py +++ b/backend/services/aircraft_feed.py @@ -7,6 +7,7 @@ in services.feed_gc. """ +import logging import math import os import time @@ -56,12 +57,18 @@ def _reset_for_tests() -> None: """ global _cached_pending_arcs, _cached_detecting_nodes, _arcs_last_ts global _cached_gt_snapshot, _cached_gt_meta, _gt_last_ts + global _mn_entry_fail_logged_at, _mn_entry_fail_count _cached_pending_arcs = [] _cached_detecting_nodes = {} _arcs_last_ts = 0.0 _cached_gt_snapshot = {} _cached_gt_meta = {} _gt_last_ts = 0.0 + # Same reason as the wall-clock caches above: the log throttle is + # monotonic-time state, so one test's skipped entry would otherwise + # silence the next test's. + _mn_entry_fail_logged_at = 0.0 + _mn_entry_fail_count = 0 _MN_ADSB_PREFIX = "mn-adsb-" @@ -177,6 +184,72 @@ def multinode_to_aircraft(key: str, r: dict) -> dict: return entry +# One multinode entry failing is a bug worth a log line, but the feed builds +# at 1 Hz over ~40 live keys, so an unguarded logger would turn one sick +# aircraft into thousands of identical lines an hour and bury everything else. +# Same shape of throttle detection_mirror.py uses for its dropped-frame line. +_MN_ENTRY_FAIL_LOG_INTERVAL_S = 60.0 +_mn_entry_fail_logged_at = 0.0 +_mn_entry_fail_count = 0 + + +def _note_multinode_entry_failure(key: str) -> None: + """Count a skipped multinode entry, logging at most once a minute.""" + global _mn_entry_fail_logged_at, _mn_entry_fail_count + _mn_entry_fail_count += 1 + now = time.monotonic() + if now - _mn_entry_fail_logged_at < _MN_ENTRY_FAIL_LOG_INTERVAL_S: + return + _mn_entry_fail_logged_at = now + logging.exception( + "Multinode feed entry failed for key=%s, skipping it (%d total since boot)", + key, + _mn_entry_fail_count, + ) + + +def _multinode_entry(key: str, r: dict, now: float) -> dict: + """One multinode solve as a feed entry, dead-reckoned to ``now``. + + Split out of build_combined_aircraft_json so the whole per-entry + computation — multinode_to_aircraft, the learned-velocity lookup, the + dead-reckon — sits behind one try/except there. Inline, an exception + from any of them took the entire flush with it. + """ + ac = multinode_to_aircraft(key, r) + # Dead-reckon position using solver velocity (vel_east/vel_north in + # m/s), capped at 30 s: beyond that a velocity error dominates any + # solve accuracy (a 15 m/s error is already 450 m of drift at the + # cap), so an old solve holds its last dead-reckoned point until the + # 60 s entry expiry rather than drifting further. + ts_fix = r.get("timestamp_ms", 0) / 1000.0 + elapsed = min(now - ts_fix, 30.0) + vel_east_m_s = r.get("vel_east", 0.0) + vel_north_m_s = r.get("vel_north", 0.0) + # TRACK_DR_SOURCE, read per call like TRACK_SMOOTHER: "kf" (default) + # dead-reckons with the display filter's LEARNED velocity when one + # exists — the solved velocity this block used to trust was measured + # (2026-08-09, n=93) at median 127 m/s vector error, i.e. ~3.8 km of + # drift at the 30 s cap below, worse than the solve error itself. + # "solve" restores the old behaviour (rollback, env only). The KF + # accessor returns None whenever the KF never saw this key (smoother + # in ewma/off mode, first solve, TTL-swept) so the fallback below is + # also the natural off-path, not a separate mode. + if (os.getenv("TRACK_DR_SOURCE", "kf") or "kf").strip().lower() != "solve": + _lv = track_filter.learned_velocity(key) + if _lv is not None: + vel_east_m_s, vel_north_m_s = _lv[0], _lv[1] + if elapsed > 0.0 and (vel_east_m_s != 0.0 or vel_north_m_s != 0.0): + _dr_lat, _dr_lon = offset_latlon_m( + ac["lat"], + ac["lon"], + east_m=vel_east_m_s * elapsed, + north_m=vel_north_m_s * elapsed, + ) + ac["lat"], ac["lon"] = round(_dr_lat, 5), round(_dr_lon, 5) + return ac + + def _claimed_single_node_entries(now: float) -> list[dict]: """Feed entries for hexes exactly ONE node is currently claiming. @@ -357,37 +430,19 @@ def build_combined_aircraft_json(default_pipeline: PassiveRadarPipeline) -> dict continue if r.get("n_nodes", 0) >= 3 and solve_count == 1 and age_s > MN_ONESHOT_TTL_S: continue - ac = multinode_to_aircraft(key, r) - # Dead-reckon position using solver velocity (vel_east/vel_north in - # m/s), capped at 30 s: beyond that a velocity error dominates any - # solve accuracy (a 15 m/s error is already 450 m of drift at the - # cap), so an old solve holds its last dead-reckoned point until the - # 60 s entry expiry rather than drifting further. - ts_fix = r.get("timestamp_ms", 0) / 1000.0 - elapsed = min(now - ts_fix, 30.0) - vel_east_m_s = r.get("vel_east", 0.0) - vel_north_m_s = r.get("vel_north", 0.0) - # TRACK_DR_SOURCE, read per call like TRACK_SMOOTHER: "kf" (default) - # dead-reckons with the display filter's LEARNED velocity when one - # exists — the solved velocity this block used to trust was measured - # (2026-08-09, n=93) at median 127 m/s vector error, i.e. ~3.8 km of - # drift at the 30 s cap below, worse than the solve error itself. - # "solve" restores the old behaviour (rollback, env only). The KF - # accessor returns None whenever the KF never saw this key (smoother - # in ewma/off mode, first solve, TTL-swept) so the fallback below is - # also the natural off-path, not a separate mode. - if (os.getenv("TRACK_DR_SOURCE", "kf") or "kf").strip().lower() != "solve": - _lv = track_filter.learned_velocity(key) - if _lv is not None: - vel_east_m_s, vel_north_m_s = _lv[0], _lv[1] - if elapsed > 0.0 and (vel_east_m_s != 0.0 or vel_north_m_s != 0.0): - _dr_lat, _dr_lon = offset_latlon_m( - ac["lat"], - ac["lon"], - east_m=vel_east_m_s * elapsed, - north_m=vel_north_m_s * elapsed, - ) - ac["lat"], ac["lon"] = round(_dr_lat, 5), round(_dr_lon, 5) + # One sick solve must not cost the whole broadcast. Everything from + # here to the append reads a single multinode entry, and an exception + # anywhere in it used to propagate out of the flush task ("Aircraft + # flush failed") and drop the ENTIRE tick's feed — every other + # aircraft with it — for one bad key. A skipped entry ages out of + # state.multinode_tracks on its own within 60 s, so degrading to + # "this one aircraft is missing for a few ticks" is strictly better + # than an empty map. + try: + ac = _multinode_entry(key, r, now) + except Exception: + _note_multinode_entry_failure(key) + continue if ac["hex"] not in seen_hex: seen_hex.add(ac["hex"]) append_track_history(ac["hex"], ac["lat"], ac["lon"], ac["alt_baro"], now) diff --git a/backend/services/track_filter.py b/backend/services/track_filter.py index 22accba3..aa705ea5 100644 --- a/backend/services/track_filter.py +++ b/backend/services/track_filter.py @@ -270,12 +270,21 @@ def learned_velocity(track_key: str) -> tuple[float, float, float, float] | None filter. _KF_LOCK is a leaf lock (see module docstring), so taking it here keeps the established solver.py -> track_filter order; callers must not hold it already. + + The sqrt is clamped for the same reason _smooth_kf's kf_pos_sigma_m one + is: this is a read-only accessor on a hot display path with two callers + that each lose real work when it throws — solver.py's + multinode_key_decision (drops that solve) and aircraft_feed's + multinode_to_aircraft (drops the whole broadcast) — so a pathological + filter state must degrade to "sigma 0", never to a ValueError. An + accessor is the wrong place to discover a covariance is sick; the + invariant is enforced upstream in _measurement_R and _kf_correct. """ with _KF_LOCK: entry = _KF_TRACKS.get(track_key) if entry is None: return None - vel_sigma = math.sqrt(0.5 * (entry.P[1, 1] + entry.P[3, 3])) + vel_sigma = math.sqrt(max(0.0, 0.5 * (entry.P[1, 1] + entry.P[3, 3]))) return float(entry.x[1]), float(entry.x[3]), float(vel_sigma), float(entry.last_ts_s) @@ -391,9 +400,13 @@ def _measurement_R(result: dict) -> np.ndarray: not one scaled version of the other, so they add rather than one replacing the other: - cov present and sane: R = (_KF_R_INFLATE**2) * cov_m2 + base + cov present and PSD: R = (_KF_R_INFLATE**2) * cov_m2 + base cov absent/degenerate: R = base (cov=0 limit) + "Degenerate" now includes a cov that is not positive-semidefinite, which + an ill-conditioned solve really does produce — see the determinant check + in the body for why that has to be rejected rather than passed through. + where base = diag(_KF_DEFAULT_POS_SIGMA_M**2, _KF_DEFAULT_POS_SIGMA_M**2) is added UNCONDITIONALLY — the no-cov fallback is not a separate branch, it is exactly this same formula evaluated at cov_m2 = 0. This is what @@ -413,8 +426,37 @@ def _measurement_R(result: dict) -> np.ndarray: cov = result.get("cov_en_km2") if cov is not None: arr = np.asarray(cov, dtype=float) * 1e6 # km^2 -> m^2 - if arr.shape == (2, 2) and np.all(np.isfinite(arr)) and arr[0, 0] > 0 and arr[1, 1] > 0: - r = (_KF_R_INFLATE**2) * arr + base # independent noise sources -> variances add + if arr.shape == (2, 2) and np.all(np.isfinite(arr)): + arr = (arr + arr.T) / 2.0 + # PSD, not just positive-diagonal. A 2x2 symmetric matrix is PSD + # iff both diagonals are >= 0 AND the determinant is >= 0; the + # determinant is the half that was missing, and it is not a + # theoretical gap. cov_en_km2 is the top-left 2x2 block of + # s2 * inv(JtJ) for the solver's 5-state fit, and the solver + # falls back to pinv only on an outright LinAlgError — an + # ill-conditioned-but-not-singular JtJ (near-parallel baselines, + # the same degenerate tail that puts the formal sigma's p99 at + # 3.8e6 km) inverts to numerical garbage that is INDEFINITE while + # still having both diagonals positive, so it passed both this + # check and the solver's own. + # + # An indefinite R is not survivable downstream. _kf_correct's + # Joseph form preserves positive-semidefiniteness for any gain, + # but only GIVEN PSD P and R — its K R K^T term inherits R's + # negative eigenvalue directly — and _init_entry seeds P's + # position block from this matrix, so a sick R poisons the filter + # at birth as well as on every update. Once P's velocity + # diagonals go negative, learned_velocity's sqrt raises: on the + # test droplet that was 91 tracebacks in 40 minutes, each one + # costing either a solve or an entire feed broadcast. + # + # Rejecting is the honest response rather than repairing by + # eigenvalue clipping: a covariance this degenerate carries no + # trustworthy relative weighting to preserve, and "R = base" is + # already this function's documented answer for a degenerate cov. + det = arr[0, 0] * arr[1, 1] - arr[0, 1] * arr[1, 0] + if arr[0, 0] > 0 and arr[1, 1] > 0 and det >= 0: + r = (_KF_R_INFLATE**2) * arr + base # independent noise sources -> variances add s = math.sqrt(0.5 * (r[0, 0] + r[1, 1])) if s < _KF_MIN_POS_SIGMA_M: @@ -510,6 +552,10 @@ def _smooth_kf(result: dict, track_key: str, adsb_hex: str | None) -> dict: f, q = _f_q(dt) x_pred = f @ entry.x p_pred = f @ entry.P @ f.T + q + # Symmetrize here too, not only in _kf_correct: F P F^T is symmetric + # in exact arithmetic but drifts by roundoff, and this filter composes + # predict and update thousands of times over a track's life. + p_pred = (p_pred + p_pred.T) / 2.0 z_e, z_n = _enu_offset_m(entry.ref_lat, entry.ref_lon, r_lat, r_lon) z = np.array([z_e, z_n]) From 41197651686a5a69050685064a5a8635026477b3 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 17:23:59 +0000 Subject: [PATCH 17/25] Follow established dark tracks top-down (DARK_FOLLOW_MODE) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dark aircraft are solved bottom-up all the way down: tracker tracks, delay grid, cluster, solve, and only then a key chosen by proximity to whatever multinode entry is nearest. Measured on the test droplet: consecutive solves of one aircraft <5 s apart land on a different key 15% of the time, any gap re-mints the key, neighbours 3 km apart share keys, and no solve is ever told where its aircraft is expected to be — the initial guess is a 3 km grid centroid. The known lane already inverted that order for ADS-B aircraft. This adds the missing half of the analogy for dark ones: an established mn-dark-* track with a Kalman state predicts (delay, Doppler) at any node exactly the way a dead-reckoned transponder fix does, so the same claiming machinery works against it. The follow solve then carries anchor_key = the followed key, so multinode_key_decision's anchor branch lands it back on the SAME track — continuity by construction rather than by the proximity scan happening to pick the right neighbour. Node-track ids were the obvious alternative and are not safe: attaching each solve to the newest key sharing a source_track_id linked the wrong aircraft 12% of the time in a dense metro cluster, for the same reason _supersession_match stopped trusting a bare shared id. Following is a feedback loop, so the ghost guard is load-bearing rather than tidy: two rejected follow-solves in a row, or a filter velocity sigma past the threshold, drop a key for a cooldown and hand the aircraft back to the bottom-up lane. Default mode is shadow — the lane decides what the map believes in with no transponder to check itself against. Co-Authored-By: Claude Fable 5.1 --- backend/core/state.py | 29 +++ backend/routes/test.py | 21 +- backend/services/dark_follow.py | 348 +++++++++++++++++++++++++++ backend/services/frame_processor.py | 18 +- backend/services/known_claiming.py | 170 ++++++++++++- backend/services/tasks/known_lane.py | 245 ++++++++++++++++++- backend/services/tasks/solver.py | 30 ++- backend/tests/conftest.py | 2 + 8 files changed, 845 insertions(+), 18 deletions(-) create mode 100644 backend/services/dark_follow.py diff --git a/backend/core/state.py b/backend/core/state.py index 634aa212..5be36271 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -74,6 +74,20 @@ if KNOWN_LANE_MODE not in ("off", "shadow", "binding"): KNOWN_LANE_MODE = "shadow" +# Dark track following (see services/dark_follow.py) — the same top-down claim +# as the known lane, against established mn-dark-* tracks instead of ADS-B +# identities. Same three-way vocabulary, and the same fallback discipline: an +# unrecognised value degrades to the inert-for-the-feed mode. Default +# "shadow", unlike KNOWN_LANE_MODE's "binding", because this lane decides which +# aircraft the map believes in without a transponder to check itself against — +# it earns its soak before it binds. Claiming rides the known lane's per-frame +# stage, so this is inert whenever KNOWN_LANE_MODE is off (an ADS-B aircraft +# must never lose a detection to a dark pseudo-state, and that precedence needs +# the ADS-B claims to have been made first). +DARK_FOLLOW_MODE = os.getenv("DARK_FOLLOW_MODE", "shadow").lower() +if DARK_FOLLOW_MODE not in ("off", "shadow", "binding"): + DARK_FOLLOW_MODE = "shadow" + node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE) @@ -508,6 +522,17 @@ def _adsb_for_seeding() -> dict[str, dict]: # Claiming-stage exceptions absorbed by frame_processor's fail-open guard. # Nonzero means the known lane is broken and silently contributing nothing. known_claims_errors: int = 0 +# Dark track following (DARK_FOLLOW_MODE) — see services/dark_follow.py. +# targets is a GAUGE (the size of the current pseudo-state list, assigned on +# every rebuild), the other four are since-boot counters. The funnel reads +# targets -> claims -> inputs -> published, with dropped the guard's own +# tally: a lane with targets but no claims is predicting positions no node +# agrees with, and one with rising dropped is the guard doing its job. +dark_follow_targets: int = 0 +dark_follow_claims: int = 0 +dark_follow_inputs: int = 0 +dark_follow_published: int = 0 +dark_follow_dropped: int = 0 # n=2 solves withheld from the map because their track pairing has not (yet) # passed the constant-velocity fit. Counted separately from solver_failures: # the solve succeeded, it simply has not earned publication, and a real target @@ -749,6 +774,8 @@ def _reset_for_tests() -> None: global adsb_seed_frames_autotagged, adsb_capture_ts_fallback global known_claims_made, known_claim_contentions, known_claims_bound global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects + global dark_follow_targets, dark_follow_claims, dark_follow_inputs + global dark_follow_published, dark_follow_dropped global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes global coverage_rebuild_backlog global solver_queue_drops, solver_stale_drops, solver_resolve_skips @@ -838,6 +865,8 @@ def _reset_for_tests() -> None: known_claims_made = known_claim_contentions = known_claims_bound = 0 known_claims_errors = known_claims_visibility_rejects = 0 known_claims_world_rejects = 0 + dark_follow_targets = dark_follow_claims = dark_follow_inputs = 0 + dark_follow_published = dark_follow_dropped = 0 coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0 coverage_rebuild_backlog = 0 solver_stale_drops = 0 diff --git a/backend/routes/test.py b/backend/routes/test.py index e4018b36..d9edcfe3 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -734,7 +734,8 @@ def _window_effective_minutes(records: list[dict], minutes: float) -> float: def _record_lane(rec: dict) -> str: - """Which solver lane produced one history record: "known", "dark" or "adsb". + """Which solver lane produced one history record: "known", "dark_follow", + "dark" or "adsb". ``known_lane`` is stamped by known_lane._attempt via ``extra``. For the regular pipeline the authority is the minted track key (mn-dark-* vs @@ -742,9 +743,16 @@ def _record_lane(rec: dict) -> str: key exists, so it falls back to the same predicate that key decision uses — whether the solver input carried a transponder-shaped identity, which is also what picked its displacement cap. + + ``lane`` is checked before the key, because a dark-follow record + (services/dark_follow.py) is keyed mn-dark-* by design — it is the same + aircraft the dark lane tracks, reached top-down — and would otherwise land + in the bottom-up funnel whose attempts and rejects it is not one of. """ if rec.get("known_lane"): return "known" + if rec.get("lane") == "dark_follow": + return "dark_follow" key = rec.get("solve_key") if key: return "dark" if key.startswith("mn-dark-") else "adsb" @@ -882,7 +890,7 @@ def _solver_window_stats(minutes: float) -> dict: effective_minutes = _window_effective_minutes(merged, minutes) all_records = [r for r in merged if r["ts_ms"] >= cutoff_ms] - lane_split = {"dark": 0, "adsb": 0, "known": 0} + lane_split = {"dark": 0, "adsb": 0, "known": 0, "dark_follow": 0} records: list[dict] = [] known_records: list[dict] = [] for r in all_records: @@ -1195,6 +1203,15 @@ def _solver_window_stats(minutes: float) -> dict: "queue_drops": state.solver_queue_drops, "worker_errors": state.solver_worker_errors, "vel_untrusted_published": state.solver_vel_untrusted_published, + # Dark track following (services/dark_follow.py), since boot except + # targets, which is a live gauge of the current pseudo-state list. + # The funnel is targets -> claims -> inputs -> published; dropped is + # the ghost guard's tally of keys it stopped following. + "dark_follow_targets": state.dark_follow_targets, + "dark_follow_claims": state.dark_follow_claims, + "dark_follow_inputs": state.dark_follow_inputs, + "dark_follow_published": state.dark_follow_published, + "dark_follow_dropped": state.dark_follow_dropped, }, } diff --git a/backend/services/dark_follow.py b/backend/services/dark_follow.py new file mode 100644 index 00000000..b2e66613 --- /dev/null +++ b/backend/services/dark_follow.py @@ -0,0 +1,348 @@ +"""Dark track following (DARK_FOLLOW_MODE): the pseudo-state store behind the +top-down claiming of aircraft that have no transponder. + +The dark lane is bottom-up all the way down — per-node tracker tracks, delay +grid pairing, clustering, solve, and only THEN a key, chosen by proximity to +whatever multinode entry happens to be nearest (solver.multinode_key_decision). +Two consequences the map shows directly. Continuity: consecutive solves of one +aircraft less than 5 s apart land on a different key 15% of the time, any gap +in solving re-mints the key from scratch, and two aircraft 3 km apart can share +one. Accuracy: nothing ever tells the solver where the aircraft is EXPECTED to +be, so a solve starts from a 3 km grid centroid. + +The known lane already solved both problems for ADS-B aircraft, by inverting +the order: identity first (services/known_claiming.py claims detections against +a dead-reckoned transponder fix), then a solve seeded with that fix. This +module supplies the missing half of the analogy — an identity for a dark +aircraft. There is one available and it is already good: an established +multinode track with a Kalman state. Its dead-reckoned position and velocity +predict (delay, Doppler) at any node exactly the way an ADS-B fix does, so the +same claiming machinery works against it unchanged. + +WHY NOT NODE-TRACK IDS. The obvious alternative — carry ``source_track_ids`` +forward and attach each new solve to the newest key sharing a node-track id — +was simulated against a dense metro cluster and linked the WRONG aircraft 12% +of the time. Single-node tracker tracks are genuinely shared between the +association candidates of different aircraft (the same measurement that made +solver._supersession_match stop trusting a bare shared id), so a node-track id +is evidence about a detection, not about an aircraft. A predicted observation +is evidence about an aircraft, which is what keying needs. + +THE GHOST RISK, AND THE GUARD. Following a track is a positive feedback loop: +a solve keeps a key alive, the key keeps predicting, the prediction keeps +claiming detections. Left alone that locks a ghost onto the map forever — the +bottom-up lane can no longer disagree with it, because binding mode takes the +detections away before the lane sees them. So a followed key is dropped (and +put in cooldown, letting the bottom-up lane re-find it or not) as soon as it +stops earning its place: two rejected follow-solves in a row, or a filter +velocity sigma past DARK_FOLLOW_MAX_VEL_SIGMA_MS. The guard is the reason this +lane is safe to bind; it is not optional tidiness. + +Modes (state.DARK_FOLLOW_MODE), same three-way shape as KNOWN_LANE_MODE: + off — nothing; no targets are built, so no claim can form. + shadow — claim, solve and record, but the claimed detections stay in the + dark pool and nothing is published. Default: the lane changes + which aircraft the map believes in, and that earns a soak. + binding — claimed detections leave the dark pool (the same + strip_claimed_detections the known lane uses) and the solve goes + onto the normal solver queue, keyed onto the followed track by + its anchor. +""" + +import logging +import os +import threading +import time + +from config.constants import C_KM_US +from core import state +from services import track_filter + +_logger = logging.getLogger(__name__) + +# ── Target eligibility ─────────────────────────────────────────────────────── +# How stale a dark track's last solve may be and still be followed. Shorter +# than the map's own 60 s expiry on purpose: past ~20 s the dead-reckoned +# position is the KF's extrapolation rather than a measurement, and a claim +# made against it would be the lane inventing its own evidence. +DARK_FOLLOW_MAX_AGE_S = float(os.getenv("DARK_FOLLOW_MAX_AGE_S", "20")) +# Minimum solves before a key may be followed. A key with one or two solves +# behind it is exactly what the bottom-up lane mints for a mis-associated +# fragment; requiring three means the aircraft has survived the whole gate +# stack repeatedly before anything is claimed on its behalf. Same intent as +# the library's CLAIM_ELIGIBLE_MIN_SOLVE_COUNT, one notch stricter because a +# claim here also REMOVES detections from the lane that would disagree. +DARK_FOLLOW_MIN_SOLVES = int(os.getenv("DARK_FOLLOW_MIN_SOLVES", "3")) +# ...and how many nodes the last solve used. n>=3 is where the multinode +# position is overdetermined; an n=2 track is a bistatic intersection that the +# displacement and beam gates are still arguing about. +DARK_FOLLOW_MIN_NODES = 3 +# Velocity sigma ceiling. The KF's own statement about how well it knows the +# track's velocity, and the term that dominates the prediction error the moment +# the track is coasted: at 60 m/s a 2 s coast is already 120 m of position +# uncertainty, and the Doppler allowance the gate below derives from it is +# ~78 Hz against a 25 Hz base — wider than the base gate, which is the point +# past which the "prediction" stops constraining anything. +DARK_FOLLOW_MAX_VEL_SIGMA_MS = float(os.getenv("DARK_FOLLOW_MAX_VEL_SIGMA_MS", "60")) +# How long a dropped key stays un-followable. Long enough that the bottom-up +# lane gets several association rounds (ASSOC_MIN_INTERVAL_S is 30 s at its +# widest, ~2 s at its narrowest) to re-find the aircraft on its own evidence +# before this lane is allowed to assert it again. +DARK_FOLLOW_COOLDOWN_S = float(os.getenv("DARK_FOLLOW_COOLDOWN_S", "30")) +# Minimum spacing between follow-solves for one key. Matched to the known +# lane's pass interval: the aircraft is already being solved bottom-up too, and +# a follow-solve every 2 s is four refreshes inside the map's 60 s expiry. +DARK_FOLLOW_INTERVAL_S = float(os.getenv("DARK_FOLLOW_INTERVAL_S", "2.0")) + +# Consecutive rejected follow-solves that drop a key. Two, not one: a single +# reject is routinely a bad epoch (one node's contaminated measurement trips +# the rms gate), while two in a row is the prediction itself being wrong. +_MAX_CONSECUTIVE_REJECTS = 2 + +# Position sigma to assume for a followed entry whose record carries none — +# the KF's own cold-start value (track_filter._KF_DEFAULT_POS_SIGMA_M). An +# entry without a filter-reported sigma is one the smoother has not updated, +# so the cold-start number is the honest statement about it; deliberately not +# a tighter guess, because understating sigma here narrows a gate. +_DEFAULT_POS_SIGMA_M = 1200.0 + +# Ceilings on the widened gates (see follow_gates). The uncertainty terms are +# bounded by the thresholds above in normal operation, but the filter's own +# sigma cap is 8 km, which alone would open a 53 µs delay gate — at that width +# the claim is no longer a prediction test. 4x the base gates: past there the +# aircraft should be re-found bottom-up, not followed. +_MAX_DELAY_GATE_US = 40.0 +_MAX_DOPPLER_GATE_HZ = 100.0 + +# Metres per microsecond and metres per second, from the one speed-of-light +# constant this repo has. +_C_M_PER_US = C_KM_US * 1000.0 +_C_M_PER_S = C_KM_US * 1.0e9 + +# ── Target cache ───────────────────────────────────────────────────────────── +# Rebuilt at most this often. The claiming stage runs once per frame per node +# — ~37 calls/s on the test fleet — and each rebuild walks multinode_tracks and +# takes track_filter's lock once per dark key. One second is short against +# DARK_FOLLOW_MAX_AGE_S (so a target never survives its own staleness by more +# than a rebuild) and long enough that the walk is amortised ~37x. The +# per-frame dead-reckoning happens at the CALL site against the frame's own +# timestamp, so caching the base list costs no prediction accuracy. +_TARGETS_TTL_S = 1.0 + +_TARGETS_LOCK = threading.Lock() +_targets: list[dict] = [] +_targets_built_mono = 0.0 + +# Guard state: consecutive rejects per key, and the cooldown each drop starts. +_GUARD_LOCK = threading.Lock() +_reject_streak: dict[str, int] = {} +_cooldown_until: dict[str, float] = {} + + +def _reset_for_tests() -> None: + """Drop the target cache and the guard state. Tests only.""" + global _targets, _targets_built_mono + with _TARGETS_LOCK: + _targets = [] + _targets_built_mono = 0.0 + with _GUARD_LOCK: + _reject_streak.clear() + _cooldown_until.clear() + state.dark_follow_targets = 0 + + +def mode() -> str: + """The follow mode, defensively — the known lane's ``_mode`` precedent. + + An absent or unrecognised value is "off": a flag this module cannot read + must disable the lane, never bind it. + """ + m = getattr(state, "DARK_FOLLOW_MODE", "off") + return m if m in ("off", "shadow", "binding") else "off" + + +def drop_target(key: str, reason: str) -> None: + """Stop following ``key`` for DARK_FOLLOW_COOLDOWN_S. + + Idempotent inside a cooldown window: a key already in cooldown is not + re-dropped, so the counter reads "keys dropped", not "times the drop + condition was re-observed". The target cache is not invalidated — it + expires within _TARGETS_TTL_S, and the cooldown is re-tested on rebuild. + """ + now = time.monotonic() + with _GUARD_LOCK: + if _cooldown_until.get(key, 0.0) > now: + return + _cooldown_until[key] = now + DARK_FOLLOW_COOLDOWN_S + _reject_streak.pop(key, None) + state.bump_counter("dark_follow_dropped") + _logger.debug("dark-follow: dropped %s (%s)", key, reason) + + +def record_outcome(key: str, ok: bool) -> None: + """Feed one follow-solve verdict to the ghost guard. + + ``ok`` is "this solve reached the feed" in binding mode and "this solve + landed within the dark displacement cap of its own prediction" in shadow — + the shadow pass has no gates to be rejected by, and a guard that stayed + inert there would leave the soak measuring a lane the binding one does not + have. Either way a good solve clears the streak and + _MAX_CONSECUTIVE_REJECTS bad ones in a row drop the key. + """ + if ok: + with _GUARD_LOCK: + _reject_streak.pop(key, None) + return + with _GUARD_LOCK: + streak = _reject_streak.get(key, 0) + 1 + _reject_streak[key] = streak + if streak >= _MAX_CONSECUTIVE_REJECTS: + drop_target(key, f"{streak} consecutive rejected follow-solves") + + +def _in_cooldown(key: str, now_mono: float) -> bool: + with _GUARD_LOCK: + return _cooldown_until.get(key, 0.0) > now_mono + + +def _sweep_guard(now_mono: float) -> None: + """Drop expired cooldowns. Both maps are keyed by mn-dark-* ids, which + churn for the process lifetime, so neither may grow unbounded.""" + with _GUARD_LOCK: + for k in [k for k, until in _cooldown_until.items() if until <= now_mono]: + del _cooldown_until[k] + _reject_streak.pop(k, None) + + +def _pos_sigma_m(rec: dict) -> float: + """The followed entry's own position uncertainty, in metres. + + ``kf_pos_sigma_m`` is the smoother's post-update marginal (the honest + number, and the one solve_uncertainty already calibrates against); + ``pos_sigma_km`` is the solver's formal pre-inflation sigma, used only when + the filter never ran on this key. Both absent means a cold entry, which + gets the filter's own cold-start sigma rather than a flattering guess. + """ + kf = rec.get("kf_pos_sigma_m") + if isinstance(kf, (int, float)) and kf > 0: + return float(kf) + formal = rec.get("pos_sigma_km") + if isinstance(formal, (int, float)) and formal > 0: + return float(formal) * 1000.0 + return _DEFAULT_POS_SIGMA_M + + +def _build_targets(now_s: float, now_mono: float) -> list[dict]: + """One pseudo-state per followable dark track. Cheap, and cached.""" + out: list[dict] = [] + for key, rec in list(state.multinode_tracks.items()): + if not key.startswith("mn-dark-"): + continue + if _in_cooldown(key, now_mono): + continue + lat, lon = rec.get("lat"), rec.get("lon") + if lat is None or lon is None: + continue + ts_ms = rec.get("timestamp_ms") or 0 + age_s = now_s - ts_ms / 1000.0 + if not (0.0 <= age_s <= DARK_FOLLOW_MAX_AGE_S): + continue + if int(rec.get("solve_count") or 0) < DARK_FOLLOW_MIN_SOLVES: + continue + if int(rec.get("n_nodes") or 0) < DARK_FOLLOW_MIN_NODES: + continue + # No filter state, no follow. The velocity and its sigma are the whole + # prediction: without them there is nothing to dead-reckon with and no + # way to widen a gate honestly, and the raw solved velocity is exactly + # the under-determined quantity (n<=3 Doppler) the KF exists to fix. + lv = track_filter.learned_velocity(key) + if lv is None: + continue + vel_east, vel_north, vel_sigma_ms, _last_ts_s = lv + if vel_sigma_ms > DARK_FOLLOW_MAX_VEL_SIGMA_MS: + drop_target(key, f"velocity sigma {vel_sigma_ms:.0f} m/s") + continue + # World tag. The overlap-zone world gate means every node that + # contributed to one multinode entry is from a single world, so the + # first contributor answers for all of them; an entry with no + # contributor list left (an old record shape) is untagged, and untagged + # passes every world — the same rule known_claiming's ADS-B world gate + # applies to a cache entry written before worlds existed. + node_ids = list(rec.get("contributing_node_ids") or ()) + world = state.node_world(node_ids[0]) if node_ids else None + out.append( + { + "key": key, + "lat": float(lat), + "lon": float(lon), + "alt_m": float(rec.get("alt_m") or 0.0), + "vel_east": float(vel_east), + "vel_north": float(vel_north), + "vel_sigma_ms": float(vel_sigma_ms), + "pos_sigma_m": _pos_sigma_m(rec), + "timestamp_ms": int(ts_ms), + "world": world, + } + ) + return out + + +def follow_targets() -> list[dict]: + """The current dark follow targets, rebuilt at most once per _TARGETS_TTL_S. + + Returns the shared cached list; callers must treat it (and the dicts in it) + as read-only. Empty in ``off`` mode, which is what makes the whole lane + cost one attribute read per frame when it is switched off. + """ + global _targets, _targets_built_mono + if mode() == "off": + return [] + now_mono = time.monotonic() + with _TARGETS_LOCK: + if _targets_built_mono and now_mono - _targets_built_mono < _TARGETS_TTL_S: + return _targets + _sweep_guard(now_mono) + _targets = _build_targets(time.time(), now_mono) + _targets_built_mono = now_mono + # A gauge, not a counter: it is the size of the list above, so it is + # assigned rather than bumped (state.bump_counter only adds). + state.dark_follow_targets = len(_targets) + return _targets + + +def follow_gates(target: dict, dt_s: float, base_delay_us: float, base_doppler_hz: float, fc_hz: float) -> tuple[float, float]: + """Claim gates for one follow target coasted ``dt_s`` seconds, in + (delay µs, Doppler Hz). + + Two terms, because there are two independent error sources and the ADS-B + path only has the first: + + MEASUREMENT. ``base * _gate_scale(dt)`` — the known lane's own gate, + age-scaled exactly as it is there (services/known_claiming._gate_scale). + This covers detection noise and the node's own bias. + + STATE. The followed track's own uncertainty, converted into observation + space. A position error of ``s`` metres moves the bistatic range by at + most ``2s`` (the target can be displaced toward both the transmitter and + the receiver), so it is worth ``2s / c`` microseconds of delay; the + position error at claim time is the filter's position sigma plus its + velocity sigma coasted over ``dt``. A velocity error of ``u`` m/s + likewise moves the bistatic Doppler by at most ``2u / λ`` = ``2u·fc / c`` + hertz. Both are worst-case projections (the true geometry factor is a + cosine ≤ 1), which is the right direction for a gate: it may admit a + detection the geometry would have excluded, never exclude one it should + have admitted. + + Capped at _MAX_DELAY_GATE_US / _MAX_DOPPLER_GATE_HZ — see those constants. + """ + # Imported here rather than at module scope: known_claiming imports this + # module for the claiming path, and a top-level import back would be a + # cycle. _gate_scale is the age-scaling rule itself, and duplicating it + # would let the two lanes drift apart on the same physics. + from services.known_claiming import _gate_scale + + scale = _gate_scale(dt_s) + pos_err_m = target["pos_sigma_m"] + target["vel_sigma_ms"] * max(dt_s, 0.0) + d_gate = base_delay_us * scale + 2.0 * pos_err_m / _C_M_PER_US + f_gate = base_doppler_hz * scale + 2.0 * target["vel_sigma_ms"] * fc_hz / _C_M_PER_S + return min(d_gate, _MAX_DELAY_GATE_US), min(f_gate, _MAX_DOPPLER_GATE_HZ) diff --git a/backend/services/frame_processor.py b/backend/services/frame_processor.py index 0fb2bbd5..083afed2 100644 --- a/backend/services/frame_processor.py +++ b/backend/services/frame_processor.py @@ -339,7 +339,9 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP # cannot cross-pair into a phantom solve). _pframe is what the dark # lane processes from here down; the original frame is untouched, so # the archive and the ADS-B cache extraction below still see everything - # the node sent. + # the node sent. Dark track following (DARK_FOLLOW_MODE) rides the same + # stage — it claims what the ADS-B paths leave, so it cannot run without + # them, which is why it is gated on KNOWN_LANE_MODE too. _pframe = frame if state.KNOWN_LANE_MODE != "off" and frame.get("delay"): # Fail open: the known lane is an overlay on the dark lane, and in @@ -347,10 +349,20 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP # this guard, one ADS-B record with alt_baro="ground" threw here and # took every frame down with it until the record aged out. try: - _claimed = claim_known_targets(node_id, frame) + # Two lanes claim here, with independent binding modes, so the + # strip is computed as one union and applied once: strip_claimed_ + # detections re-bases the indices it keeps, and a second strip + # against the first's output would delete the wrong detections. + _followed: set[int] = set() + _claimed = claim_known_targets(node_id, frame, follow_claimed=_followed) + _strip: set[int] = set() if _claimed and state.KNOWN_LANE_MODE == "binding": - _pframe = strip_claimed_detections(frame, _claimed) + _strip |= _claimed state.bump_counter("known_claims_bound", len(_claimed)) + if _followed and state.DARK_FOLLOW_MODE == "binding": + _strip |= _followed + if _strip: + _pframe = strip_claimed_detections(frame, _strip) except Exception: state.bump_counter("known_claims_errors") now = time.time() diff --git a/backend/services/known_claiming.py b/backend/services/known_claiming.py index 684d9a16..cf44df71 100644 --- a/backend/services/known_claiming.py +++ b/backend/services/known_claiming.py @@ -65,6 +65,7 @@ from config.constants import FT_TO_M, as_num from core import state +from services import dark_follow from services.id_utils import normalize_hex_key # Same base constants as the seeding path: the comparison is the identical @@ -195,7 +196,143 @@ def _is_contested(delay_us: float, doppler_hz: float, projections: list[tuple[fl ) -def claim_known_targets(node_id: str, frame: dict) -> set[int]: +def _claim_dark_follow( + node_id: str, + geo, + frame_ts_s: float, + ts_ms: int, + delays: list, + dopplers: list, + free: list[int], +) -> set[int]: + """Path 3: claim the leftover detections against followed dark tracks. + + The ADS-B paths' assignment, run a second time with the dark pseudo-states + (services/dark_follow.py) standing in for cached transponder fixes — same + dead-reckoning, same visibility gate, same Hungarian one-to-one, same + per-axis normalised score. Only the gate widths differ, because a dark + pseudo-state carries its own uncertainty and an ADS-B fix is treated as + truth (see dark_follow.follow_gates). + + Runs on ``free`` — what the ADS-B paths did not take — which IS the + precedence rule: an aircraft with a transponder can never lose a detection + to a dark track's prediction, whatever the residuals say. The reverse is + tolerable; a dark track that loses a detection is solved from its other + nodes, and a wrong ADS-B claim would charge a fix the node never saw to + that node's trust. + + Claims land in state.known_claims under the mn-dark-* KEY rather than a + hex, marked ``dark_follow`` so known_lane's two passes can tell them apart. + They carry ``follow_fix`` rather than ``adsb_fix`` deliberately: every + other reader of the registry (the feed's single-node ADS-B section, the + per-node trust residuals) keys on ``adsb_fix``, and a follow claim has no + transponder fix to offer them — its absence is what keeps those readers + unchanged. + """ + if not free or dark_follow.mode() == "off": + return set() + targets = dark_follow.follow_targets() + if not targets: + return set() + + node_world = state.node_world(node_id) + cands: list[tuple[dict, float, float, float, float, float, float]] = [] + for t in targets: + # Same world gate as path 2, same reason: a synthetic node's echoes are + # only ever of simulated aircraft. Untagged targets pass. + if t["world"] is not None and t["world"] != node_world: + continue + dt = frame_ts_s - t["timestamp_ms"] / 1000.0 + if not (0.0 <= dt <= dark_follow.DARK_FOLLOW_MAX_AGE_S): + continue + dr_lat, dr_lon = offset_latlon_m( + t["lat"], + t["lon"], + east_m=t["vel_east"] * dt, + north_m=t["vel_north"] * dt, + ) + # The associator's own visibility predicate, applied whole — the same + # call path 2 makes, for the same asymmetry: a false accept binds a + # detection to an aircraft this node cannot see and takes it out of the + # lane that would have disagreed. + if not _point_in_beam(dr_lat, dr_lon, geo): + continue + alt_km = t["alt_m"] / 1000.0 + pred_d, pred_f = predict_observation( + geo, + dr_lat, + dr_lon, + alt_km, + t["vel_east"], + t["vel_north"], + ) + d_gate, f_gate = dark_follow.follow_gates( + t, + dt, + KNOWN_CLAIM_DELAY_GATE_US, + KNOWN_CLAIM_DOPPLER_GATE_HZ, + geo.fc_hz, + ) + cands.append((t, pred_d, pred_f, d_gate, f_gate, dr_lat, dr_lon)) + if not cands: + return set() + + cost = np.full((len(free), len(cands)), _GATE_INFEASIBLE) + for c, (_t, pred_d, pred_f, d_gate, f_gate, _dr_lat, _dr_lon) in enumerate(cands): + for r, i in enumerate(free): + d_res = abs(pred_d - float(delays[i])) + f_res = abs(pred_f - float(dopplers[i])) + if d_res > d_gate or f_res > f_gate: + continue + cost[r, c] = d_res / d_gate + f_res / f_gate + rows, cols = linear_sum_assignment(cost) + + claimed: set[int] = set() + for r, c in zip(rows, cols): + if cost[r, c] >= _GATE_INFEASIBLE: + continue + i = free[r] + t, pred_d, pred_f, _d_gate, _f_gate, dr_lat, dr_lon = cands[c] + dq = state.known_claims.get(t["key"]) + if dq is None: + dq = state.known_claims.setdefault(t["key"], deque(maxlen=state.KNOWN_CLAIMS_PER_HEX_MAX)) + dq.append( + { + "node_id": node_id, + "delay_us": float(delays[i]), + "doppler_hz": float(dopplers[i]), + "pred_delay_us": float(pred_d), + "pred_doppler_hz": float(pred_f), + "ts_ms": ts_ms, + "dark_follow": True, + # The prediction itself, at the frame epoch — this is what the + # follow solve uses as its initial guess, which is the second + # thing this lane exists for (the first being the key). Unlike + # path 2's REPORTED-position rule there is no reported position + # to prefer: the dead-reckoned state is the only estimate there + # has ever been. + "follow_fix": { + "lat": dr_lat, + "lon": dr_lon, + "alt_km": t["alt_m"] / 1000.0, + "vel_east": t["vel_east"], + "vel_north": t["vel_north"], + "fix_ts_ms": ts_ms, + }, + # Contention is an ADS-B-vs-dark question (identity evidence + # beating a dark projection). A follow claim IS the dark + # projection, so there is nothing for it to contend with, and + # leaving the flag false is what lets known_lane's selection + # reuse _select_claims unchanged. + "contested": False, + } + ) + claimed.add(i) + state.bump_counter("dark_follow_claims") + return claimed + + +def claim_known_targets(node_id: str, frame: dict, follow_claimed: set[int] | None = None) -> set[int]: """Run the claiming stage for one frame; return the claimed detection indices. @@ -212,6 +349,15 @@ def claim_known_targets(node_id: str, frame: dict) -> set[int]: 2. Remaining detections × fresh cached ADS-B states whose dead-reckoned position this node can see, global one-to-one via linear_sum_assignment under age-scaled gates. + 3. Dark track following (DARK_FOLLOW_MODE) — the same assignment again, + against established dark tracks' predicted observations instead of + ADS-B fixes. See _claim_dark_follow. + + ``follow_claimed``, when given, is the set path 3's indices are written + into. They are deliberately NOT part of the return value: the two lanes + have independent binding modes, so the caller must be able to strip one + lane's claims from the frame without the other's. Omit it and path 3 does + not run at all — a caller that cannot receive the split cannot honour it. Claims nothing without a registered geometry: the registry contract requires the predicted observation, and there is nothing to predict @@ -412,12 +558,9 @@ def claim_known_targets(node_id: str, frame: dict) -> set[int]: ) claimed_idx.add(i) - if not claims: - return set() - # ── Contention, registry, counters, residual hook ───────────────────────── - projections = _dark_global_projections(geo, frame_ts_s) - nb = _node_bias() + projections = _dark_global_projections(geo, frame_ts_s) if claims else [] + nb = _node_bias() if claims else None for i, hexn, fix, pred_d, pred_f in claims: d_meas = float(delays[i]) f_meas = float(dopplers[i]) @@ -452,6 +595,21 @@ def claim_known_targets(node_id: str, frame: dict) -> set[int]: # makes a bias a bias. nb.record_claim_residual(node_id, hexn, d_meas - pred_d, f_meas - pred_f, ts_ms) + # ── Path 3: dark track following ───────────────────────────────────────── + # Last, on what the ADS-B paths left behind — see _claim_dark_follow for + # why that ordering is the precedence rule rather than an implementation + # detail. + if follow_claimed is not None: + follow_claimed |= _claim_dark_follow( + node_id, + geo, + frame_ts_s, + ts_ms, + delays, + dopplers, + [i for i in range(len(delays)) if i not in claimed_idx], + ) + return claimed_idx diff --git a/backend/services/tasks/known_lane.py b/backend/services/tasks/known_lane.py index 6ffe29ad..c8630e40 100644 --- a/backend/services/tasks/known_lane.py +++ b/backend/services/tasks/known_lane.py @@ -41,6 +41,17 @@ published: a displaced solve under a real hex is a wrong map marker, the same reason the regular displacement gate exists. +A SECOND PASS lives here too: the dark-follow lane (DARK_FOLLOW_MODE, see +services/dark_follow.py) applies the same inversion to aircraft that have no +transponder, using an established mn-dark-* track's Kalman state where this +lane uses an ADS-B fix. It shares this module's registry reader, pass lock and +interval — the two passes read the same ``state.known_claims`` and differ only +in which kind of claim they select — but not its solve path: a follow solve +goes onto the normal solver queue so the dark gate stack judges it, whereas a +known-lane solve is deliberately free of that stack (see the free-solve +invariant above). ``run_dark_follow_pass`` and ``_build_follow_solver_input`` +carry the detail. + Neither ``state.known_claims`` nor ``state.KNOWN_LANE_MODE`` exists on this branch — slice A owns core/state.py — so every access goes through getattr with an inert default, and the counters below are registered onto the state @@ -55,7 +66,7 @@ from config.constants import FT_TO_M from core import state -from services import track_filter +from services import dark_follow, track_filter from services.geo import haversine_km, offset_latlon_m from services.id_utils import normalize_hex_key @@ -128,6 +139,13 @@ _last_pass_ts = 0.0 _last_attempt_ts_ms: dict[str, int] = {} _last_sample_mono: dict[str, float] = {} +# Dark-follow pass bookkeeping, per followed key: when it last produced a +# follow-solve (monotonic, the DARK_FOLLOW_INTERVAL_S rate limit) and the claim +# epoch that solve was built from (the same "no newer claim, no new solve" +# dedup _last_attempt_ts_ms gives the ADS-B pass). Both single-writer under +# _PASS_LOCK, like the maps above. +_last_follow_mono: dict[str, float] = {} +_last_follow_ts_ms: dict[str, int] = {} def _reset_for_tests() -> None: @@ -137,6 +155,8 @@ def _reset_for_tests() -> None: _last_pass_ts = 0.0 _last_attempt_ts_ms.clear() _last_sample_mono.clear() + _last_follow_mono.clear() + _last_follow_ts_ms.clear() with state.counters_lock: for name in _COUNTERS: setattr(state, name, 0) @@ -157,7 +177,7 @@ def _num(v, fallback=0.0) -> float: return float(fallback) -def _select_claims(dq, now_ms: int) -> dict[str, dict]: +def _select_claims(dq, now_ms: int, follow: bool = False) -> dict[str, dict]: """Newest usable claim per node from one hex's deque, or {} if fewer than two nodes survive. @@ -168,11 +188,20 @@ def _select_claims(dq, now_ms: int) -> dict[str, dict]: is slice A's, written concurrently, and this reader must survive any single bad entry. Nodes whose newest claim trails the newest overall by more than _CLAIM_SPREAD_S are dropped rather than failing the whole hex. + + ``follow`` selects which KIND of claim: the dark-follow claims a key's + deque holds (DARK_FOLLOW_MODE, see services/dark_follow.py) or the ADS-B + ones a hex's does. The two never share a deque in practice — a registry + entry is keyed either by transponder hex or by mn-dark-* key — but the + filter is explicit rather than implied by the key shape, so a stray entry + can only be ignored, never solved by the wrong lane. """ best: dict[str, dict] = {} for c in list(dq): if not isinstance(c, dict): continue + if bool(c.get("dark_follow")) is not follow: + continue try: ts_ms = int(c["ts_ms"]) node_id = c["node_id"] @@ -484,6 +513,207 @@ def run_known_lane_pass(solve_fn, node_cfgs: dict | None = None, mode: str | Non return attempts +def _build_follow_solver_input(key: str, claims: dict[str, dict]) -> dict | None: + """Shape one followed dark track's claims into a solver input. + + _build_solver_input's shape, with the dark pseudo-state (see + services/dark_follow.py) supplying what the ADS-B fix supplies there: the + initial guess is the KF prediction dead-reckoned to the newest claim's + epoch, and the velocity seed is the KF's learned velocity. Two fields the + ADS-B lane has no use for carry the rest of the point of this lane: + + anchor_key — the followed key itself. multinode_key_decision's anchor + branch honours it (it is mn-dark-*, it is live, and the solve started + from this track's own prediction so it lands inside the branch's flat + _MN_ASSOC_MAX_DIST_KM check by construction), so the solve keys onto the + SAME track it was predicted from. That is continuity by construction + rather than by the proximity scan happening to pick the right neighbour + — the failure the measurements behind this lane are of. + + track_ids — deliberately empty. A follow input is detection-level: its + correspondence came from the prediction, not from a tracker pairing, so + there is no provenance to record. The empty set is also what admits it + past _claim_resolve_slot (an input with no track provenance has nothing + to be a duplicate of) and what keeps it out of supersession, which must + not pop a neighbour on a track id this solve never used. + + ``lane``/``guess_source``/``follow_key`` ride through to the history record + (solver._record_solve_history stamps all three) so the lane is separable in + /api/test/solver-stats without inferring it from the key. + """ + newest = max(claims.values(), key=lambda c: int(c["ts_ms"])) + newest_ts_ms = int(newest["ts_ms"]) + fix = newest.get("follow_fix") + if not isinstance(fix, dict): + return None + lat, lon = fix.get("lat"), fix.get("lon") + if lat is None or lon is None: + return None + + vel_east = _num(fix.get("vel_east")) + vel_north = _num(fix.get("vel_north")) + # Normally zero: the newest claim's prediction was made at its own frame + # epoch, which is this epoch. Kept for the same reason the ADS-B path + # keeps it — the claim record is written by another thread and the epoch it + # names is the only thing that says when the prediction was true. + dt_s = (newest_ts_ms - int(_num(fix.get("fix_ts_ms"), newest_ts_ms))) / 1000.0 + guess_lat, guess_lon = offset_latlon_m( + float(lat), + float(lon), + east_m=vel_east * dt_s, + north_m=vel_north * dt_s, + ) + + return { + "initial_guess": { + "lat": guess_lat, + "lon": guess_lon, + "alt_km": _num(fix.get("alt_km")), + }, + "initial_velocity": { + "vel_east_ms": vel_east, + "vel_north_ms": vel_north, + }, + "measurements": [ + { + "node_id": nid, + "delay_us": float(c["delay_us"]), + "doppler_hz": float(c["doppler_hz"]), + "snr": _num(c.get("snr")), + } + for nid, c in sorted(claims.items()) + ], + "n_nodes": len(claims), + "timestamp_ms": newest_ts_ms, + "anchor_key": key, + "track_ids": [], + "lane": "dark_follow", + "guess_source": "prediction", + "follow_key": key, + } + + +def _follow_shadow_attempt(key: str, s_in: dict, node_cfgs: dict, solve_fn) -> None: + """Solve one follow input without touching the feed, and record it. + + Shadow's whole job is to answer "would this lane have helped?", which needs + the solve to actually run — the claim alone says nothing about whether the + prediction was right. Classified against the dark displacement cap, the + same number the binding path's gate would judge it by, so the shadow record + and a binding reject mean the same thing; the verdict also feeds the ghost + guard, which would otherwise be inert for the whole soak. + """ + try: + result = solve_fn(s_in, node_cfgs) + except Exception: + logging.exception("Dark-follow shadow solve failed for %s", key) + result = None + + ok = False + disp_km = None + if result and result.get("success"): + ig = s_in["initial_guess"] + disp_km = haversine_km(float(ig["lat"]), float(ig["lon"]), float(result["lat"]), float(result["lon"])) + ok = disp_km <= solver_mod._MAX_DISPLACEMENT_KM_DARK + solver_mod._record_solve_history( + "dark_follow_shadow", + s_in, + result if isinstance(result, dict) else None, + displacement_km=disp_km, + extra={"shadow": True, "published": False, "follow_ok": ok}, + ) + + +def run_dark_follow_pass(solve_fn, node_cfgs: dict | None = None, mode: str | None = None) -> int: + """One pass over the dark-follow claims; returns the inputs produced. + + A followed key is solved when it has fresh follow-claims from >= 2 nodes at + compatible timestamps (_select_claims, follow=True), a claim newer than its + last follow-solve, and its DARK_FOLLOW_INTERVAL_S rate limit has elapsed. + In binding mode the input goes onto state.solver_queue and is processed by + the normal dark path — the same gate stack, keying, KF smoothing and + history records every bottom-up solve gets, which is the point: this lane + changes where a solve STARTS and which key it lands on, not what a solve + has to survive. In shadow it is solved and recorded here instead, because + anything reaching the queue would publish. + + ``mode`` overrides state.DARK_FOLLOW_MODE for this pass, for the same + reason run_known_lane_pass takes the override: a solver worker daemon + leaked into the test process would otherwise race the test for the per-key + rate limit. + """ + if mode is None: + mode = dark_follow.mode() + elif mode not in ("off", "shadow", "binding"): + mode = "off" + if mode == "off": + return 0 + claims_by_key = getattr(state, "known_claims", None) + if not claims_by_key: + return 0 + + now_ms = int(time.time() * 1000) + now_mono = time.monotonic() + inputs = 0 + for key, dq in list(claims_by_key.items()): + if not isinstance(key, str) or not key.startswith("mn-dark-"): + continue + last_mono = _last_follow_mono.get(key) + if last_mono is not None and now_mono - last_mono < dark_follow.DARK_FOLLOW_INTERVAL_S: + continue + claims = _select_claims(dq, now_ms, follow=True) + if not claims: + continue + if node_cfgs is None: + from services.frame_processor import get_node_configs + + node_cfgs = get_node_configs() + # A node whose config has gone (disconnected since the claim) cannot be + # solved with: the LM needs its geometry. Drop the node rather than + # the key — the remaining nodes are still a solve if there are two. + claims = {nid: c for nid, c in claims.items() if nid in node_cfgs} + if len(claims) < 2: + continue + newest_ts = max(int(c["ts_ms"]) for c in claims.values()) + if _last_follow_ts_ms.get(key, -1) >= newest_ts: + continue + s_in = _build_follow_solver_input(key, claims) + if s_in is None: + continue + # Stamped before the solve, as the ADS-B pass does: a solve that raises + # must not be retried against the same claims on every pass forever. + _last_follow_mono[key] = now_mono + _last_follow_ts_ms[key] = newest_ts + cfgs = {nid: node_cfgs[nid] for nid in claims} + state.bump_counter("dark_follow_inputs") + inputs += 1 + if mode == "binding": + try: + state.solver_queue.put_nowait((s_in, cfgs, time.time())) + except Exception: + state.bump_counter("solver_queue_drops") + else: + _follow_shadow_attempt(key, s_in, cfgs, solve_fn) + + # Opportunistic TTL sweep of both dedup maps — keys churn for the process + # lifetime, the same reason _ATTEMPT_TTL_S exists. + cutoff_ms = now_ms - _ATTEMPT_TTL_S * 1000.0 + for k in [k for k, ts in _last_follow_ts_ms.items() if ts < cutoff_ms]: + del _last_follow_ts_ms[k] + _last_follow_mono.pop(k, None) + return inputs + + +def lanes_armed() -> bool: + """True when either lane in this module has something to do. + + The solver worker arms its per-iteration pass call on this ONCE, at thread + start (see _run_solver_worker for why an off lane must not cost the idle + loop even a mode read). + """ + return _mode() != "off" or dark_follow.mode() != "off" + + def maybe_run_pass(solve_fn, mode: str | None = None) -> None: """Interval- and mode-gated pass entry point for the solver worker loop. @@ -493,11 +723,13 @@ def maybe_run_pass(solve_fn, mode: str | None = None) -> None: rather than queue, and the interval check lives under the same lock so two workers cannot both pass it in the same window. ``mode`` is the same test-only override run_known_lane_pass documents; the worker loop - always passes nothing and reads the live flag. + always passes nothing and reads the live flag. It applies to the ADS-B + pass only — the dark-follow pass reads its own flag, so a test that arms + one lane explicitly does not silently arm the other. """ global _last_pass_ts try: - if (mode if mode is not None else _mode()) == "off": + if (mode if mode is not None else _mode()) == "off" and dark_follow.mode() == "off": return if not _PASS_LOCK.acquire(blocking=False): return @@ -507,6 +739,11 @@ def maybe_run_pass(solve_fn, mode: str | None = None) -> None: return _last_pass_ts = now run_known_lane_pass(solve_fn, mode=mode) + # Second, and under the same lock and interval: the two passes read + # the same registry, and the follow pass's per-key rate limit is + # single-writer for exactly the reason the ADS-B pass's dedup map + # is. Its own mode flag decides whether it does anything. + run_dark_follow_pass(solve_fn) finally: _PASS_LOCK.release() except Exception: diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 7c6b947c..8261ea08 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -19,7 +19,7 @@ N2_TRACK_ASSOCIATION, ) from core import state -from services import track_filter +from services import dark_follow, track_filter # Beam-coverage geometry, used to reject solver results whose range or (at # n=2) bearing fall outside a contributing node's detection area. This @@ -1487,6 +1487,16 @@ def _record_solve_history( # windowed fragmentation breakdown can see what fraction of ALL # attempts (not just successful ones) were anchor-carrying. "anchor_key": s.get("anchor_key"), + # Lane provenance, carried by the solver input rather than inferred + # from the key: a dark-follow solve (services/dark_follow.py) lands on + # an mn-dark-* key and would otherwise be indistinguishable from the + # bottom-up solves whose funnel it is not part of. guess_source says + # what the initial guess WAS — "prediction" for a followed track, + # absent for the association grid centroid every other dark input + # carries — and follow_key names the track that predicted it. + "lane": s.get("lane"), + "guess_source": s.get("guess_source"), + "follow_key": s.get("follow_key"), "raw_lat": round(float(raw_lat), 6) if raw_lat is not None else None, "raw_lon": round(float(raw_lon), 6) if raw_lon is not None else None, "lat": round(float(r["lat"]), 6) if outcome == "published" else None, @@ -1551,6 +1561,19 @@ def _record_solve_history( } if extra: rec.update(extra) + _follow_key = rec.get("follow_key") + if _follow_key: + # The dark-follow ghost guard (services/dark_follow.py) needs a verdict + # for every follow-solve, and this is the one place all of them pass + # through — published, every rejected_* gate, unconverged, and the + # shadow pass's own record. Following a track is a feedback loop (the + # solve keeps the key alive, the key keeps claiming detections), so a + # key that stops earning its solves has to be droppable from OUTSIDE + # that loop. A shadow record carries its own verdict in follow_ok: + # it never reached the gates, so "did it publish" says nothing. + dark_follow.record_outcome(_follow_key, bool(rec.get("follow_ok", outcome == "published"))) + if outcome == "published": + state.bump_counter("dark_follow_published") if raw_lat is not None and raw_lon is not None: meas_ts_s = (rec["measurement_ts_ms"] or now_ms) / 1000.0 rec.update(_gt_for_record(rec["adsb_hex"], float(raw_lat), float(raw_lon), meas_ts_s)) @@ -2370,11 +2393,12 @@ def _run_solver_worker(): # coverage collector lock hard enough to stall test_mlat_history's # trail-race stress test (~50 daemons caught inside the mode check in a # single py-spy snapshot). - known_lane_armed = known_lane._mode() != "off" + known_lane_armed = known_lane.lanes_armed() while True: _solver_worker_iteration() if known_lane_armed: - # Known-lane pass (identity-first claims → per-hex solves). + # Known-lane pass (identity-first claims → per-hex solves), plus + # the dark-follow pass behind the same lock and interval. # Ridden on the worker loop rather than its own thread so the # solve compute stays on the threads that already own the solver # locks and pool; interval- and concurrency-gated inside, and it diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 9aa250c7..45d8f82a 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -155,6 +155,7 @@ def _reset_module_state(): from services import ( aircraft_feed, alerting, + dark_follow, feed_helpers, frame_processor, known_claiming, @@ -177,6 +178,7 @@ def _reset_module_state(): alerting, tcp_handler, known_claiming, + dark_follow, node_bias, publication, periodic, From 9e3fd15f76e8b679b1d58b272dc305bfd3e293a6 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 17:25:45 +0000 Subject: [PATCH 18/25] Mirror the tightened disc-growth horizon on the backend helper solve_uncertainty.grown_sigma_m is the backend twin of the frontend's uncertainty.ts and exists so the two can be pinned to the same shape. Its horizon stays with UNCERTAINTY_DR_CAP_S: 60 -> 30 s. Its docstring claimed the frontend stops dead-reckoning at 60 s, which stopped being true in the previous commit, and a dark entry no longer reaches 60 s of age at all. No production caller -- the feed ships pos_sigma_m at the solve epoch and the frontend does the growing. Co-Authored-By: Claude Fable 5.1 --- backend/services/solve_uncertainty.py | 23 +++++++++++++---------- backend/tests/test_solve_uncertainty.py | 13 ++++++++----- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/backend/services/solve_uncertainty.py b/backend/services/solve_uncertainty.py index 04e58e06..72f6de50 100644 --- a/backend/services/solve_uncertainty.py +++ b/backend/services/solve_uncertainty.py @@ -81,12 +81,15 @@ _SIGMA_MIN_M = 50.0 _SIGMA_MAX_M = 5000.0 -# Velocity-sigma clamp band and the growth horizon. Past 60 s the frontend -# stops dead-reckoning entirely, so growing the disc further would describe -# a position nothing is drawing. +# Velocity-sigma clamp band and the growth horizon. Past the horizon the +# frontend stops dead-reckoning entirely, so growing the disc further would +# describe a position nothing is drawing. It tracks the frontend's +# UNCERTAINTY_DR_CAP_S, which in turn tracks MN_DARK_EXPIRY_S: a dark entry no +# longer reaches 60 s at all, so the old 60 s horizon described entries that +# cannot exist. _VEL_MIN_MS = 5.0 _VEL_MAX_MS = 150.0 -_GROWTH_MAX_AGE_S = 60.0 +_GROWTH_MAX_AGE_S = 30.0 def _floor_m(n_nodes: int) -> float: @@ -152,13 +155,13 @@ def grown_sigma_m(sigma_m: float, vel_sigma_ms: float, age_s: float) -> float: Position error at the solve epoch and velocity error over the coast are independent, so their variances add: - sigma(t) = sqrt( sigma_solve^2 + (sigma_v * min(t, 60))^2 ) + sigma(t) = sqrt( sigma_solve^2 + (sigma_v * min(t, _GROWTH_MAX_AGE_S))^2 ) - Age is clamped to [0, 60]: negative is a clock artefact, and past 60 s the - frontend has stopped dead-reckoning, so a bigger disc would not correspond - to anything on screen. The frontend's own uncertainty.ts is the mirror of - this function; this one exists for backend callers and for the tests that - pin the two to the same shape. + Age is clamped to [0, _GROWTH_MAX_AGE_S]: negative is a clock artefact, and + past the horizon the frontend has stopped dead-reckoning, so a bigger disc + would not correspond to anything on screen. The frontend's own + uncertainty.ts is the mirror of this function; this one exists for backend + callers and for the tests that pin the two to the same shape. """ t = min(max(age_s, 0.0), _GROWTH_MAX_AGE_S) return float(math.sqrt(sigma_m**2 + (vel_sigma_ms * t) ** 2)) diff --git a/backend/tests/test_solve_uncertainty.py b/backend/tests/test_solve_uncertainty.py index 1d7628a2..c4b335c0 100644 --- a/backend/tests/test_solve_uncertainty.py +++ b/backend/tests/test_solve_uncertainty.py @@ -121,16 +121,19 @@ def test_growth_adds_in_quadrature(self): def test_zero_age_is_the_solve_sigma(self): assert su.grown_sigma_m(650.0, 25.0, 0.0) == pytest.approx(650.0) - def test_growth_is_capped_at_60s(self): - at_60 = su.grown_sigma_m(180.0, 25.0, 60.0) - assert su.grown_sigma_m(180.0, 25.0, 600.0) == pytest.approx(at_60) - assert at_60 == pytest.approx(math.sqrt(180.0**2 + 1500.0**2)) + def test_growth_is_capped_at_the_growth_horizon(self): + # The horizon mirrors the frontend's UNCERTAINTY_DR_CAP_S (30 s), + # which in turn mirrors MN_DARK_EXPIRY_S. + assert su._GROWTH_MAX_AGE_S == 30.0 + at_cap = su.grown_sigma_m(180.0, 25.0, su._GROWTH_MAX_AGE_S) + assert su.grown_sigma_m(180.0, 25.0, 600.0) == pytest.approx(at_cap) + assert at_cap == pytest.approx(math.sqrt(180.0**2 + 750.0**2)) def test_negative_age_does_not_shrink_or_grow(self): assert su.grown_sigma_m(650.0, 25.0, -5.0) == pytest.approx(650.0) def test_growth_is_monotonic_up_to_the_cap(self): - seq = [su.grown_sigma_m(210.0, 30.0, t) for t in (0.0, 5.0, 20.0, 59.0)] + seq = [su.grown_sigma_m(210.0, 30.0, t) for t in (0.0, 5.0, 20.0, 29.0)] assert seq == sorted(seq) assert seq[0] < seq[-1] From 9ec77e676a14163bcaa68af426e0d0c0322a0597 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 17:30:50 +0000 Subject: [PATCH 19/25] Add regression tests for the non-PSD covariance failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every test here fails on origin/main and passes on this branch. test_track_filter.TestIndefiniteCovariance: - the indefinite cov is rejected and R falls back to the base floor, while a strongly-correlated-but-valid cov (corr 0.9, det > 0) still gets through — the check must reject only the impossible matrices, or it silently discards the relative weighting _KF_R_INFLATE exists for - 200 solves carrying an indefinite cov at both a 1.4 s and a 20 s cadence keep P PSD and symmetric and learned_velocity answering. On origin/main the 20 s case raises at solve 16 with the droplet's exact traceback, track_filter.py:278 ValueError: math domain error - a 1 m and a 1e-6 m formal sigma both compose to the 1200 m floor, so a metre-scale R never reaches the update - 200 updates at an effectively-zero cov with alternating positions stay PSD - a hand-poisoned negative P[1,1]/P[3,3] returns sigma 0.0 rather than raising test_feed_multinode.TestMultinodeEntryFailureIsolation: - three multinode keys, the middle one raising ValueError("math domain error") from learned_velocity: the other two are still served, only the sick one is missing, and one log line names it. On origin/main the exception escapes build_combined_aircraft_json and the whole tick is lost, which is what "Aircraft flush failed" was - 20 consecutive failures produce one log line, not 20 Co-Authored-By: Claude Fable 5.1 --- backend/tests/test_feed_multinode.py | 95 +++++++++++++++++ backend/tests/test_track_filter.py | 153 +++++++++++++++++++++++++++ 2 files changed, 248 insertions(+) diff --git a/backend/tests/test_feed_multinode.py b/backend/tests/test_feed_multinode.py index 9fb4ab28..7e555f73 100644 --- a/backend/tests/test_feed_multinode.py +++ b/backend/tests/test_feed_multinode.py @@ -485,3 +485,98 @@ def test_kf_state_supplies_the_velocity_sigma(self): ac = self._build_mn() assert ac["pos_sigma_vel_ms"] == pytest.approx(round(lv[2], 1), abs=0.05) assert ac["pos_sigma_vel_ms"] != pytest.approx(25.0) + + +class TestMultinodeEntryFailureIsolation: + """One bad multinode key must cost one aircraft, not the whole feed. + + The 2026-09-05 droplet failure went through here: a track whose filter + covariance had gone non-PSD made learned_velocity raise, and because the + per-entry work sat inline in build_combined_aircraft_json's loop the + exception propagated out of the flush task ("Aircraft flush failed") and + dropped the ENTIRE tick's payload -- every other aircraft with it, 91 + times in 40 minutes. track_filter now stops that covariance ever + forming; this is the second line of defence, which has to hold for any + future per-entry bug, not just that one. + """ + + @pytest.fixture(autouse=True) + def _clean_state(self): + from services import aircraft_feed + + state.multinode_tracks.clear() + state.track_histories.clear() + track_filter.reset() + aircraft_feed._reset_for_tests() + yield + state.multinode_tracks.clear() + state.track_histories.clear() + track_filter.reset() + aircraft_feed._reset_for_tests() + + def _build(self): + from services.frame_processor import build_combined_aircraft_json + + pipeline = types.SimpleNamespace(geolocated_tracks={}, config={}) + return build_combined_aircraft_json(pipeline) + + def test_one_raising_entry_does_not_abort_the_build(self, monkeypatch, caplog): + from services import aircraft_feed + + good_a, bad, good_b = "mn-dark-good-a", "mn-dark-bad", "mn-dark-good-b" + for i, key in enumerate((good_a, bad, good_b)): + # Spread them out: co-located entries are collapsed by + # dedup_aircraft, which would hide the very thing under test. + entry = _mn_entry(age_s=5.0, vel_north=100.0) + entry["lat"] = LAT + 0.1 * i + state.multinode_tracks[key] = entry + + real_learned_velocity = track_filter.learned_velocity + + def _boom(track_key): + # Exactly the failure the droplet saw, raised from exactly the + # function it was raised from. + if track_key == bad: + raise ValueError("math domain error") + return real_learned_velocity(track_key) + + monkeypatch.setattr(aircraft_feed.track_filter, "learned_velocity", _boom) + + with caplog.at_level("ERROR"): + result = self._build() + + mn_hexes = {a["hex"] for a in result["aircraft"] if a.get("multinode")} + from services.id_utils import multinode_hex_from_key + + # The two healthy aircraft are still served ... + assert multinode_hex_from_key(good_a) in mn_hexes + assert multinode_hex_from_key(good_b) in mn_hexes + # ... and only the sick one is missing. + assert multinode_hex_from_key(bad) not in mn_hexes + # Logged once, naming the key, so this is diagnosable rather than silent. + assert sum("Multinode feed entry failed" in r.message for r in caplog.records) == 1 + assert bad in caplog.text + + def test_repeated_failures_are_rate_limited_to_one_log_line(self, monkeypatch, caplog): + from services import aircraft_feed + + bad = "mn-dark-bad" + state.multinode_tracks[bad] = _mn_entry(age_s=5.0, vel_north=100.0) + + def _boom(track_key): + raise ValueError("math domain error") + + monkeypatch.setattr(aircraft_feed.track_filter, "learned_velocity", _boom) + + with caplog.at_level("ERROR"): + for _ in range(20): + # Re-stamp: the entry would otherwise age past the 60 s expiry + # only after many more ticks, but keeping it fresh makes the + # 20 failures unambiguous. + state.multinode_tracks[bad] = _mn_entry(age_s=5.0, vel_north=100.0) + self._build() + + # 20 failures inside one _MN_ENTRY_FAIL_LOG_INTERVAL_S window -> + # exactly one line, not 20. + assert sum("Multinode feed entry failed" in r.message for r in caplog.records) == 1 + assert aircraft_feed._mn_entry_fail_count == 20 diff --git a/backend/tests/test_track_filter.py b/backend/tests/test_track_filter.py index 44beee81..2e15e94d 100644 --- a/backend/tests/test_track_filter.py +++ b/backend/tests/test_track_filter.py @@ -829,3 +829,156 @@ def _feed(): track_filter.reset() assert track_filter.learned_velocity(key) is None + + +class TestIndefiniteCovariance: + """An indefinite cov_en_km2 must never reach the filter. + + This is the 2026-09-05 droplet failure (91 ValueError: math domain error + in 40 minutes out of learned_velocity), and the mechanism is not roundoff + — the Joseph form in _kf_correct was already deployed when it happened. + cov_en_km2 is the top-left 2x2 of s2 * inv(JtJ) for the solver's 5-state + fit, and the solver falls back to pinv only on an outright LinAlgError, + so an ill-conditioned-but-not-singular JtJ (near-parallel baselines) + inverts to garbage that is INDEFINITE while both diagonals stay positive + — passing the solver's own check and, before the fix, _measurement_R's. + + Joseph preserves PSD for any gain but only GIVEN PSD P and R: its + K R K^T term inherits R's negative eigenvalue, and _init_entry seeds P's + position block straight from R. Hence the determinant check. + + The covariances below are exactly that shape: equal diagonals with an + off-diagonal larger than their geometric mean, i.e. a "correlation" + above 1, which no real covariance has. + """ + + def setup_method(self): + track_filter.reset() + state.adsb_aircraft.clear() + + def teardown_method(self): + track_filter.reset() + state.adsb_aircraft.clear() + + @staticmethod + def _indefinite_cov(var_km2=1.0, corr=1.2): + return [[var_km2, corr * var_km2], [corr * var_km2, var_km2]] + + def test_indefinite_cov_falls_back_to_the_base_floor(self): + r = track_filter._measurement_R(make_result(35.0, -82.0, 1_000, cov_en_km2=self._indefinite_cov())) + + # Rejected outright, so R is the cov=0 limit — exactly `base`, the + # same answer a missing cov gets. + assert r[0, 1] == 0.0 + assert math.sqrt(0.5 * (r[0, 0] + r[1, 1])) == pytest.approx(track_filter._KF_DEFAULT_POS_SIGMA_M) + # The property that actually matters downstream. + assert np.all(np.linalg.eigvalsh(r) >= 0.0) + + def test_a_correlated_but_psd_cov_is_still_accepted(self): + """The determinant check must reject only the impossible matrices. + + A genuine off-diagonal is real information about solve geometry (a + two-node baseline has a long axis), and rejecting it would quietly + throw away the relative weighting _KF_R_INFLATE exists to provide. + corr=0.9 is strongly correlated but perfectly valid: det > 0. + """ + cov = self._indefinite_cov(corr=0.9) + r = track_filter._measurement_R(make_result(35.0, -82.0, 1_000, cov_en_km2=cov)) + + assert r[0, 1] != 0.0 # the correlation survived + assert np.all(np.linalg.eigvalsh(r) >= 0.0) + + @pytest.mark.parametrize("cadence_s", [1.4, 20.0]) + def test_indefinite_cov_keeps_the_filter_psd_and_learned_velocity_alive(self, monkeypatch, cadence_s): + """The end-to-end regression: 200 solves carrying an indefinite cov. + + On origin/main this raised the droplet's ValueError within 16 solves + at the 20 s cadence real solves arrive on, and within 7 at 1.4 s. + """ + monkeypatch.setenv("TRACK_SMOOTHER", "kf") + cov = self._indefinite_cov() + key = f"indef-{cadence_s}" + lat0, lon0 = 35.0, -82.0 + + for i in range(200): + lat_i, lon_i = offset_latlon_m(lat0, lon0, east_m=250.0 * cadence_s * i, north_m=0.0) + result = make_result(lat_i, lon_i, 1_000 + int(i * cadence_s * 1000), cov_en_km2=cov) + track_filter.smooth_solve(result, key, None) + + entry = track_filter._KF_TRACKS.get(key) + assert entry is not None + assert np.all(np.diag(entry.P) >= 0.0), f"solve {i}: negative variance in {np.diag(entry.P)}" + assert np.allclose(entry.P, entry.P.T, rtol=0, atol=0), f"solve {i}: asymmetric covariance" + # The call that was throwing. + assert track_filter.learned_velocity(key) is not None + + def test_tiny_position_sigma_cannot_produce_a_tight_r(self): + """A 1 m formal sigma is floored, not believed. + + R is what makes the Joseph update ill-conditioned when it is tiny + relative to P, so the unmodeled-error floor is the other half of this + fix holding: frame-time skew, association contamination and altitude + pinning do not shrink because one solve's Jacobian was tight, and no + fix on this network is good to a metre. The floor is additive and + unconditional, so a metre-scale cov contributes only + (_KF_R_INFLATE * 1)^2 = 16 m^2 against a base of 1200^2, leaving the + composed sigma at the base to within a hundredth of a percent. + """ + for sigma_m in (1e-6, 1.0): + cov_km2 = (sigma_m / 1000.0) ** 2 + r = track_filter._measurement_R( + make_result(35.0, -82.0, 1_000, cov_en_km2=[[cov_km2, 0.0], [0.0, cov_km2]]) + ) + actual_sigma = math.sqrt(0.5 * (r[0, 0] + r[1, 1])) + assert actual_sigma == pytest.approx(track_filter._KF_DEFAULT_POS_SIGMA_M, rel=1e-4) + assert actual_sigma >= track_filter._KF_MIN_POS_SIGMA_M + # Nowhere near the few-metre R that would make the update + # ill-conditioned, and an order of magnitude above even a + # bare-sensor floor: the unmodeled-error terms dominate. + assert actual_sigma > 1000.0 + + def test_repeated_updates_at_a_tiny_r_stay_psd(self, monkeypatch): + """200 solves with an effectively-zero cov and alternating positions. + + The floor above means the filter never actually sees a metre-scale R, + so this asserts the composite invariant end to end rather than the + ill-conditioning in isolation (TestJosephCovariance drives + _kf_correct at R = 1e-4 directly). + """ + monkeypatch.setenv("TRACK_SMOOTHER", "kf") + key = "tiny-r" + tiny = (1e-6 / 1000.0) ** 2 + cov = [[tiny, 0.0], [0.0, tiny]] + + for i in range(200): + # Alternating, so the innovation never settles to zero. + lat_i, lon_i = offset_latlon_m(35.0, -82.0, east_m=100.0 * (i % 2), north_m=0.0) + track_filter.smooth_solve(make_result(lat_i, lon_i, 1_000 + i * 1_400, cov_en_km2=cov), key, None) + + entry = track_filter._KF_TRACKS.get(key) + assert np.all(np.diag(entry.P) >= 0.0), f"solve {i}: negative variance in {np.diag(entry.P)}" + assert np.allclose(entry.P, entry.P.T, rtol=0, atol=0), f"solve {i}: asymmetric covariance" + assert track_filter.learned_velocity(key) is not None + + def test_negative_velocity_variance_degrades_to_zero_sigma(self, monkeypatch): + """The last line of defence, on a hand-poisoned entry. + + learned_velocity has two callers that each lose real work when it + raises — solver.py's multinode_key_decision drops the solve, + aircraft_feed's multinode_to_aircraft drops the whole broadcast — so + even a state no code path should now be able to reach must return a + number rather than throw. + """ + monkeypatch.setenv("TRACK_SMOOTHER", "kf") + key = "neg-vel-var" + track_filter.smooth_solve(make_result(35.0, -82.0, 1_000), key, None) + track_filter.smooth_solve(make_result(35.001, -82.0, 21_000), key, None) + + with track_filter._KF_LOCK: + entry = track_filter._KF_TRACKS[key] + entry.P[1, 1] = -1e6 + entry.P[3, 3] = -1e6 + + lv = track_filter.learned_velocity(key) + assert lv is not None + assert lv[2] == 0.0 # clamped, not raised From 60486a0acde42c3e3fbcb80d74a6461b536a218f Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 17:31:10 +0000 Subject: [PATCH 20/25] Test the dark-follow lane, and dead-reckon a follow input's anchor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The anchor branch's distance check compares a solve against where the entry was last STORED. For a follow input that premise is false by construction — its guess IS the prediction of where the anchor drifted to — and the numbers bite: the dark displacement cap and the flat anchor gate are both 6.0 km, so a solve at the edge of the gate that admitted it is at the edge of the gate that must key it before any drift is added, and at the lane's 20 s staleness limit a 270 m/s target adds 5.4 km more. anchor_dr, set only by a follow input, dead-reckons the anchor and applies the proximity scan's own age-scaled gate. Every other anchored input is unchanged. Also documents the lane in docs/solverflow.md 3d and backend/.env.example. Co-Authored-By: Claude Fable 5.1 --- backend/.env.example | 22 ++ backend/services/dark_follow.py | 16 +- backend/services/tasks/solver.py | 40 +- backend/tests/test_dark_follow.py | 628 ++++++++++++++++++++++++++++++ docs/solverflow.md | 125 +++++- 5 files changed, 826 insertions(+), 5 deletions(-) create mode 100644 backend/tests/test_dark_follow.py diff --git a/backend/.env.example b/backend/.env.example index 20a107b4..481456d2 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -133,6 +133,28 @@ MENDER_PAT= # SOLVER_RESOLVE_INTERVAL_S=12 # SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0 +# Dark track following (services/dark_follow.py, docs/solverflow.md 3d). The +# known lane's top-down claim applied to aircraft with no transponder: an +# established mn-dark-* track's Kalman state predicts delay/Doppler at each +# node, matching detections are claimed against it, and the resulting solve +# carries the followed key as its anchor so it lands back on the SAME track. +# +# DARK_FOLLOW_MODE: off | shadow | binding. shadow (the default) claims, +# solves and records but leaves the detections in the dark pool and publishes +# nothing; binding strips them and publishes through the normal dark gates. +# +# The rest are the eligibility and guard thresholds. A key is followable while +# its last solve is under MAX_AGE_S old with at least MIN_SOLVES solves and a +# filter velocity sigma under MAX_VEL_SIGMA_MS; it is dropped for COOLDOWN_S +# after two consecutive rejected follow-solves, which is what stops a ghost +# locking itself onto the map. INTERVAL_S rate-limits follow-solves per key. +# DARK_FOLLOW_MODE=shadow +# DARK_FOLLOW_MAX_AGE_S=20 +# DARK_FOLLOW_MIN_SOLVES=3 +# DARK_FOLLOW_MAX_VEL_SIGMA_MS=60 +# DARK_FOLLOW_INTERVAL_S=2.0 +# DARK_FOLLOW_COOLDOWN_S=30 + # Detection mirror. Production only. Every accepted v1 detection frame is # forwarded to another environment's /api/radar/detections/bulk, batched once a # second, dropped rather than retried when that environment is unreachable, and diff --git a/backend/services/dark_follow.py b/backend/services/dark_follow.py index b2e66613..b21459b8 100644 --- a/backend/services/dark_follow.py +++ b/backend/services/dark_follow.py @@ -151,6 +151,18 @@ def _reset_for_tests() -> None: state.dark_follow_targets = 0 +def _expire_targets_for_tests() -> None: + """Force the next follow_targets() call to rebuild, KEEPING the guard state. + + Tests only, and distinct from _reset_for_tests for exactly that reason: the + guard's whole behaviour is "a key that was a target stops being one", which + is unobservable if the only way to re-read the list also forgets the drop. + """ + global _targets_built_mono + with _TARGETS_LOCK: + _targets_built_mono = 0.0 + + def mode() -> str: """The follow mode, defensively — the known lane's ``_mode`` precedent. @@ -310,7 +322,9 @@ def follow_targets() -> list[dict]: return _targets -def follow_gates(target: dict, dt_s: float, base_delay_us: float, base_doppler_hz: float, fc_hz: float) -> tuple[float, float]: +def follow_gates( + target: dict, dt_s: float, base_delay_us: float, base_doppler_hz: float, fc_hz: float +) -> tuple[float, float]: """Claim gates for one follow target coasted ``dt_s`` seconds, in (delay µs, Doppler Hz). diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 8261ea08..4ff017aa 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -736,6 +736,7 @@ def multinode_key_decision( max_dist_km: float = _MN_ASSOC_MAX_DIST_KM, max_age_s: float = _MN_ASSOC_MAX_AGE_S, learned_vel_fn=track_filter.learned_velocity, + anchor_dr: bool = False, ) -> tuple[str, str, float | None]: """The keying rule itself, clock-free — the multinode-track analogue of claim_decision. Extracted so the offline bench measures the SHIPPED rule @@ -799,21 +800,45 @@ def multinode_key_decision( return f"mn-adsb-{adsb_hex}", "adsb", None lat, lon = result["lat"], result["lon"] + ts_s = result.get("timestamp_ms", 0) / 1000.0 # The anchor branch keeps the FLAT gate. It is not a dead-reckoning # question: the claim named this entry as the aircraft this solve is of, # and the distance check exists only to refuse an anchor whose solve # converged somewhere else entirely. Nothing here is predicting where the # anchor drifted to, so there is no drift term to allow for. + # + # ...unless the caller says otherwise (anchor_dr). A dark-follow input + # (services/dark_follow.py) breaks that premise by construction: its guess + # IS a prediction of where the anchor drifted to, so its solve is compared + # against an entry the follow lane already knows to be stale. The numbers + # make it more than a nicety — the dark displacement cap is 6.0 km and the + # flat anchor gate is 6.0 km, so a solve at the edge of the gate that let + # it through is at the edge of the gate that must key it, before any drift + # is added; at the follow lane's 20 s staleness limit a 270 m/s target adds + # another 5.4 km of it. Without this the anchor would be refused exactly + # when the aircraft is moving fastest, and the solve would fall through to + # the proximity scan the whole lane exists to stop relying on. Same DR and + # same age-scaled gate as that scan, so "near the anchor" means one thing. if anchor_key and anchor_key.startswith("mn-dark-") and anchor_key in tracks: anchor = tracks[anchor_key] a_lat, a_lon = anchor.get("lat"), anchor.get("lon") if a_lat is not None and a_lon is not None: + a_gate_km = max_dist_km + a_dt = ts_s - anchor.get("timestamp_ms", 0) / 1000.0 + if anchor_dr and 0.0 < a_dt <= max_age_s: + a_vel_east, a_vel_north = _entry_dr_velocity(anchor_key, anchor, learned_vel_fn) + a_lat, a_lon = offset_latlon_m( + a_lat, + a_lon, + east_m=a_vel_east * a_dt, + north_m=a_vel_north * a_dt, + ) + a_gate_km = _mn_assoc_gate_km(a_dt, max_dist_km) a_dist = _haversine_km(lat, lon, a_lat, a_lon) - if a_dist <= max_dist_km: + if a_dist <= a_gate_km: return anchor_key, "anchor", a_dist - ts_s = result.get("timestamp_ms", 0) / 1000.0 # Candidates compete on d / gate_km, not on d: an entry solved 2 s ago at # 5 km is a worse match than one solved 40 s ago at 8 km only if you # ignore that the second one's position is a 40 s extrapolation. A score @@ -2202,7 +2227,16 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus # _MN_POS_HISTORY_LOCK (inside the smoother) is never taken in # reverse anywhere. _anchor_key = s_in.get("anchor_key") if isinstance(s_in, dict) else None - key, _key_how, _key_dist_km = multinode_key_decision(state.multinode_tracks, result, _adsb_hex, _anchor_key) + key, _key_how, _key_dist_km = multinode_key_decision( + state.multinode_tracks, + result, + _adsb_hex, + _anchor_key, + # Only the follow lane's anchor is a stale position by + # construction — see the anchor branch for why that changes + # the distance check it must be judged by. + anchor_dr=bool(isinstance(s_in, dict) and s_in.get("follow_key")), + ) # Dark-lane key births vs re-keys. The fragmentation question is # "how often does one aircraft get a second key", and the only # place that is decided is right here — solver_successes counts diff --git a/backend/tests/test_dark_follow.py b/backend/tests/test_dark_follow.py new file mode 100644 index 00000000..07a547f7 --- /dev/null +++ b/backend/tests/test_dark_follow.py @@ -0,0 +1,628 @@ +"""Dark track following (DARK_FOLLOW_MODE) — services/dark_follow.py, +known_claiming's path 3, known_lane's follow pass, and the solver's anchor +dead-reckoning. + +The lane is the known lane's shape applied to aircraft with no transponder: +an established mn-dark-* track's Kalman state stands in for a dead-reckoned +ADS-B fix, and the solve it produces carries the followed key as its anchor so +it lands back on the same track. Pinned here: + +- pseudo-state eligibility (age, solve count, node count, filter state, and + the velocity-sigma ceiling that also drops the key); +- claiming: a matching detection is claimed and a non-matching one is not, an + ADS-B state always beats a dark pseudo-state for the same detection, and + ``off`` claims nothing; +- the follow pass: one queue item per key with the anchor, the prediction as + initial guess and no track provenance; the per-key rate limit; shadow + records without publishing and binding reaches the queue; +- binding mode removing the claimed detections from the frame the dark lane + sees (frame_processor); +- the ghost guard: two rejected follow-solves drop the key for the cooldown, + including when the verdicts arrive through _record_solve_history. + +Style follows test_known_claiming.py (registered associator geometry, frames +built around a real predicted observation) and test_solver_anchor.py. +""" + +import time + +import pytest +from retina_analytics.association import predict_observation + +from config.constants import FT_TO_M +from core import state +from pipeline.passive_radar import DEFAULT_NODE_CONFIG, PassiveRadarPipeline +from services import dark_follow, track_filter +from services import known_claiming as kc +from services.frame_processor import process_one_frame +from services.geo import offset_latlon_m +from services.tasks import known_lane +from services.tasks import solver as solver_mod + +_NODE_CFG = { + "rx_lat": 34.85, + "rx_lon": -82.40, + "rx_alt_ft": 1000, + "tx_lat": 34.9412, + "tx_lon": -82.4103, + "tx_alt_ft": 2000, + "fc_hz": 183e6, + "beam_width_deg": 90, + "max_range_km": 60, + "beam_azimuth_deg": 45.0, +} + +_NODE_ID = "test-dark-follow" +_KEY = "mn-dark-followed" + +# Well inside _NODE_CFG's beam — the same corner of Greenville the claiming +# tests use, so the visibility gate is never the thing under test. +_LAT, _LON, _ALT_M = 34.88, -82.35, 7000.0 + + +@pytest.fixture(autouse=True) +def _lane_off_unless_armed(monkeypatch): + """Leave the live flag off between tests. + + state.DARK_FOLLOW_MODE defaults to "shadow", and every solver worker daemon + a TestClient lifespan has leaked into this process polls it — arming it + globally would let one race these tests for the per-key rate limit, the + same hazard test_known_lane.py documents for KNOWN_LANE_MODE. Tests that + need the lane arm it themselves (``_install``) or pass ``mode`` explicitly. + """ + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "off") + + +def _register(node_id=_NODE_ID): + state.node_associator.register_node(node_id, _NODE_CFG) + return state.node_associator.node_geometries[node_id] + + +def _frame(ts_ms, delays, dopplers, adsb=None): + f = { + "timestamp": ts_ms, + "delay": list(delays), + "doppler": list(dopplers), + "snr": [20.0] * len(delays), + } + if adsb is not None: + f["adsb"] = adsb + return f + + +def _track(ts_ms, **overrides) -> dict: + """A live, followable mn-dark-* entry in state.multinode_tracks.""" + rec = { + "lat": _LAT, + "lon": _LON, + "alt_m": _ALT_M, + "vel_east": 0.0, + "vel_north": 0.0, + "timestamp_ms": ts_ms, + "n_nodes": 3, + "solve_count": 5, + "contributing_node_ids": [_NODE_ID], + } + rec.update(overrides) + return rec + + +def _kf(monkeypatch, vel_east=0.0, vel_north=0.0, vel_sigma=5.0, keys=(_KEY,)): + """Fake filter state for ``keys`` — the real KF is fed by the publish path, + which none of these tests go through.""" + lookup = {k: (vel_east, vel_north, vel_sigma, 0.0) for k in keys} + monkeypatch.setattr(track_filter, "learned_velocity", lookup.get) + + +def _install(monkeypatch, ts_ms, mode="shadow", key=_KEY, kf=True, **overrides): + """Register the node, install one followable track, arm the lane.""" + geo = _register() + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", mode) + state.multinode_tracks[key] = _track(ts_ms, **overrides) + if kf: + _kf(monkeypatch, keys=(key,)) + else: + monkeypatch.setattr(track_filter, "learned_velocity", lambda _k: None) + # The target list is TTL-cached; a test that rewrites multinode_tracks has + # to invalidate it or it reads the previous assertion's world. + dark_follow._reset_for_tests() + return geo + + +def _pred(geo, lat=_LAT, lon=_LON, alt_m=_ALT_M, ve=0.0, vn=0.0): + return predict_observation(geo, lat, lon, alt_m / 1000.0, ve, vn) + + +def _follow_claim(node_id, ts_ms, delay_us, doppler_hz, lat=_LAT, lon=_LON): + return { + "node_id": node_id, + "delay_us": delay_us, + "doppler_hz": doppler_hz, + "pred_delay_us": delay_us, + "pred_doppler_hz": doppler_hz, + "ts_ms": ts_ms, + "dark_follow": True, + "follow_fix": { + "lat": lat, + "lon": lon, + "alt_km": _ALT_M / 1000.0, + "vel_east": 0.0, + "vel_north": 0.0, + "fix_ts_ms": ts_ms, + }, + "contested": False, + } + + +def _install_follow_claims(node_ids, ts_ms, key=_KEY): + from collections import deque + + dq = state.known_claims.setdefault(key, deque(maxlen=state.KNOWN_CLAIMS_PER_HEX_MAX)) + for i, nid in enumerate(node_ids): + dq.append(_follow_claim(nid, ts_ms, 100.0 + i, 10.0 + i)) + return dq + + +def _reject_result(): + """A minimal converged solver result, enough for a history record.""" + return { + "success": True, + "lat": _LAT, + "lon": _LON, + "alt_m": _ALT_M, + "n_nodes": 3, + "timestamp_ms": int(time.time() * 1000), + "contributing_node_ids": ["n1", "n2", "n3"], + } + + +def _drain_queue(): + items = [] + while True: + try: + items.append(state.solver_queue.get_nowait()) + except Exception: + return items + + +class TestPseudoStates: + """Which dark tracks may be followed at all.""" + + def test_an_established_track_is_a_target(self, monkeypatch): + ts = int(time.time() * 1000) - 2000 + _install(monkeypatch, ts) + + (t,) = dark_follow.follow_targets() + + assert t["key"] == _KEY + assert t["lat"] == _LAT + assert t["pos_sigma_m"] == dark_follow._DEFAULT_POS_SIGMA_M + assert state.dark_follow_targets == 1 + + def test_a_stale_track_is_not(self, monkeypatch): + ts = int((time.time() - dark_follow.DARK_FOLLOW_MAX_AGE_S - 5) * 1000) + _install(monkeypatch, ts) + + assert dark_follow.follow_targets() == [] + + def test_too_few_solves_is_not(self, monkeypatch): + ts = int(time.time() * 1000) - 2000 + _install(monkeypatch, ts, solve_count=dark_follow.DARK_FOLLOW_MIN_SOLVES - 1) + + assert dark_follow.follow_targets() == [] + + def test_too_few_nodes_is_not(self, monkeypatch): + ts = int(time.time() * 1000) - 2000 + _install(monkeypatch, ts, n_nodes=2) + + assert dark_follow.follow_targets() == [] + + def test_no_filter_state_is_not(self, monkeypatch): + """The velocity and its sigma ARE the prediction; without them there is + nothing to dead-reckon with and no honest way to widen a gate.""" + ts = int(time.time() * 1000) - 2000 + _install(monkeypatch, ts, kf=False) + + assert dark_follow.follow_targets() == [] + + def test_an_adsb_key_is_never_followed(self, monkeypatch): + ts = int(time.time() * 1000) - 2000 + _install(monkeypatch, ts, key="mn-adsb-abc123") + + assert dark_follow.follow_targets() == [] + + def test_a_noisy_velocity_drops_the_key(self, monkeypatch): + ts = int(time.time() * 1000) - 2000 + _install(monkeypatch, ts) + _kf(monkeypatch, vel_sigma=dark_follow.DARK_FOLLOW_MAX_VEL_SIGMA_MS + 1.0) + dark_follow._reset_for_tests() + + assert dark_follow.follow_targets() == [] + assert state.dark_follow_dropped == 1 + + def test_off_mode_has_no_targets(self, monkeypatch): + ts = int(time.time() * 1000) - 2000 + _install(monkeypatch, ts, mode="off") + + assert dark_follow.follow_targets() == [] + + +class TestClaiming: + """Path 3 of the claiming stage: leftover detections vs pseudo-states.""" + + def test_a_matching_detection_is_claimed_and_a_stray_is_not(self, monkeypatch): + ts = int(time.time() * 1000) + geo = _install(monkeypatch, ts - 2000) + pd, pf = _pred(geo) + followed: set[int] = set() + + claimed = kc.claim_known_targets( + _NODE_ID, + _frame(ts, [pd, pd + 500.0], [pf, pf + 500.0]), + follow_claimed=followed, + ) + + assert claimed == set() + assert followed == {0} + assert state.dark_follow_claims == 1 + (c,) = list(state.known_claims[_KEY]) + assert c["dark_follow"] is True + assert c["node_id"] == _NODE_ID + assert c["delay_us"] == pytest.approx(pd) + # follow_fix, never adsb_fix: the feed's single-node ADS-B section and + # the node-trust residuals both key on adsb_fix, and a follow claim has + # no transponder fix to offer them. + assert "adsb_fix" not in c + assert c["follow_fix"]["lat"] == pytest.approx(_LAT) + + def test_adsb_wins_a_contested_detection(self, monkeypatch): + """One detection both a cached transponder fix and a followed track + explain. The ADS-B paths run first and path 3 only ever sees what they + left, so the aircraft with an identity keeps it.""" + ts = int(time.time() * 1000) + geo = _install(monkeypatch, ts - 2000) + state.adsb_aircraft["abc123"] = { + "hex": "abc123", + "lat": _LAT, + "lon": _LON, + "alt_baro": _ALT_M / FT_TO_M, + "gs": 0, + "track": 0, + "last_seen_ms": ts, + } + pd, pf = _pred(geo) + followed: set[int] = set() + + claimed = kc.claim_known_targets(_NODE_ID, _frame(ts, [pd], [pf]), follow_claimed=followed) + + assert claimed == {0} + assert followed == set() + assert _KEY not in state.known_claims + assert state.dark_follow_claims == 0 + + def test_off_mode_claims_nothing(self, monkeypatch): + ts = int(time.time() * 1000) + geo = _install(monkeypatch, ts - 2000, mode="off") + pd, pf = _pred(geo) + followed: set[int] = set() + + kc.claim_known_targets(_NODE_ID, _frame(ts, [pd], [pf]), follow_claimed=followed) + + assert followed == set() + assert state.known_claims == {} + + def test_a_caller_that_cannot_take_the_split_gets_no_path_3(self, monkeypatch): + """Omitting follow_claimed disables path 3: the two lanes have + independent binding modes, so a caller that cannot separate them must + not be handed a set it would strip wholesale.""" + ts = int(time.time() * 1000) + geo = _install(monkeypatch, ts - 2000) + pd, pf = _pred(geo) + + assert kc.claim_known_targets(_NODE_ID, _frame(ts, [pd], [pf])) == set() + assert state.known_claims == {} + + def test_a_stale_track_claims_nothing(self, monkeypatch): + ts = int(time.time() * 1000) + geo = _register() + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "shadow") + state.multinode_tracks[_KEY] = _track(int((time.time() - 120) * 1000)) + _kf(monkeypatch) + dark_follow._reset_for_tests() + pd, pf = _pred(geo) + followed: set[int] = set() + + kc.claim_known_targets(_NODE_ID, _frame(ts, [pd], [pf]), follow_claimed=followed) + + assert followed == set() + + +class TestFollowPass: + """known_lane's second pass: claims → solver input.""" + + _CFGS = {"n1": _NODE_CFG, "n2": _NODE_CFG, "n3": _NODE_CFG} + + @pytest.fixture(autouse=True) + def _private_queue(self, monkeypatch): + """A queue of this test's own: a leaked solver worker daemon drains + state.solver_queue, and would take the item under assertion.""" + import queue + + monkeypatch.setattr(state, "solver_queue", queue.Queue(maxsize=200)) + + def test_three_nodes_produce_one_anchored_queue_item(self): + ts = int(time.time() * 1000) + _install_follow_claims(["n1", "n2", "n3"], ts) + + assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 1 + + (item,) = _drain_queue() + s_in, cfgs, _enqueued = item + assert s_in["anchor_key"] == _KEY + assert s_in["follow_key"] == _KEY + assert s_in["lane"] == "dark_follow" + assert s_in["guess_source"] == "prediction" + assert s_in["track_ids"] == [] + assert s_in["n_nodes"] == 3 + assert s_in["initial_guess"]["lat"] == pytest.approx(_LAT) + assert s_in["initial_guess"]["lon"] == pytest.approx(_LON) + assert s_in["initial_guess"]["alt_km"] == pytest.approx(_ALT_M / 1000.0) + assert set(cfgs) == {"n1", "n2", "n3"} + assert state.dark_follow_inputs == 1 + + def test_a_single_node_is_not_enough(self): + _install_follow_claims(["n1"], int(time.time() * 1000)) + + assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 0 + assert _drain_queue() == [] + + def test_the_rate_limit_holds_between_passes(self): + ts = int(time.time() * 1000) + _install_follow_claims(["n1", "n2"], ts) + assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 1 + + # A newer claim set, but inside DARK_FOLLOW_INTERVAL_S. + _install_follow_claims(["n1", "n2"], ts + 500) + assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 0 + assert len(_drain_queue()) == 1 + + def test_no_newer_claim_is_no_new_solve(self, monkeypatch): + monkeypatch.setattr(dark_follow, "DARK_FOLLOW_INTERVAL_S", 0.0) + _install_follow_claims(["n1", "n2"], int(time.time() * 1000)) + assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 1 + + assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 0 + + def test_off_mode_does_nothing(self): + _install_follow_claims(["n1", "n2"], int(time.time() * 1000)) + + assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="off") == 0 + assert _drain_queue() == [] + + def test_shadow_records_and_never_publishes(self): + ts = int(time.time() * 1000) + _install_follow_claims(["n1", "n2"], ts) + + def solve(s_in, cfgs): + ig = s_in["initial_guess"] + return { + "success": True, + "lat": ig["lat"], + "lon": ig["lon"], + "alt_m": ig["alt_km"] * 1000.0, + "timestamp_ms": s_in["timestamp_ms"], + "n_nodes": s_in["n_nodes"], + "contributing_node_ids": ["n1", "n2"], + } + + assert known_lane.run_dark_follow_pass(solve, self._CFGS, mode="shadow") == 1 + + assert _drain_queue() == [] + assert state.multinode_tracks == {} + assert state.dark_follow_published == 0 + (rec,) = [r for r in state.mlat_solve_history if r.get("follow_key")] + assert rec["outcome"] == "dark_follow_shadow" + assert rec["lane"] == "dark_follow" + assert rec["guess_source"] == "prediction" + assert rec["published"] is False + assert rec["displacement_km"] == pytest.approx(0.0, abs=1e-3) + + def test_shadow_classifies_a_displaced_solve_as_not_ok(self): + """The shadow verdict is the dark displacement cap — the same number + the binding path's gate would judge the solve by — so the guard is not + inert for the whole soak.""" + _install_follow_claims(["n1", "n2"], int(time.time() * 1000)) + + def solve(s_in, cfgs): + ig = s_in["initial_guess"] + return { + "success": True, + "lat": ig["lat"] + 0.5, # ~55 km + "lon": ig["lon"], + "timestamp_ms": s_in["timestamp_ms"], + "n_nodes": s_in["n_nodes"], + "contributing_node_ids": ["n1", "n2"], + } + + known_lane.run_dark_follow_pass(solve, self._CFGS, mode="shadow") + + (rec,) = [r for r in state.mlat_solve_history if r.get("follow_key")] + assert rec["follow_ok"] is False + + def test_a_node_with_no_config_is_dropped_not_the_key(self): + ts = int(time.time() * 1000) + _install_follow_claims(["n1", "n2", "gone"], ts) + + assert known_lane.run_dark_follow_pass(None, self._CFGS, mode="binding") == 1 + + (item,) = _drain_queue() + assert set(item[1]) == {"n1", "n2"} + assert item[0]["n_nodes"] == 2 + + def test_the_adsb_pass_ignores_follow_claims(self): + """Both passes read state.known_claims; neither may solve the other's + entries.""" + _install_follow_claims(["n1", "n2"], int(time.time() * 1000)) + + assert known_lane.run_known_lane_pass(lambda s, c: None, self._CFGS, mode="shadow") == 0 + + +class TestModesInProcessOneFrame: + """Binding removes the followed detections from the frame the dark lane + processes; shadow leaves the frame whole.""" + + def _run(self, monkeypatch, mode): + ts = int(time.time() * 1000) + geo = _install(monkeypatch, ts - 2000, mode=mode) + monkeypatch.setattr(state, "KNOWN_LANE_MODE", "binding") + pd, pf = _pred(geo) + frame = _frame(ts, [pd, pd + 500.0], [pf, pf + 500.0]) + + default = PassiveRadarPipeline(DEFAULT_NODE_CONFIG) + seen = [] + monkeypatch.setattr(default, "process_frame", lambda f: seen.append(f)) + process_one_frame(_NODE_ID, frame, default) + assert len(seen) == 1 + return frame, seen[0] + + def test_binding_strips_the_followed_detection(self, monkeypatch): + frame, processed = self._run(monkeypatch, "binding") + + assert len(processed["delay"]) == 1 + assert processed["delay"][0] == frame["delay"][1] + # The original frame is untouched — the archive and the ADS-B cache + # extraction still see everything the node sent. + assert len(frame["delay"]) == 2 + + def test_shadow_leaves_the_frame_whole(self, monkeypatch): + _frame_in, processed = self._run(monkeypatch, "shadow") + + assert len(processed["delay"]) == 2 + assert state.dark_follow_claims == 1 + + +class TestGhostGuard: + """Following is a feedback loop; the guard is what makes it droppable.""" + + def _armed(self, monkeypatch): + ts = int(time.time() * 1000) - 2000 + _install(monkeypatch, ts) + assert len(dark_follow.follow_targets()) == 1 + + def test_two_rejected_solves_drop_the_key(self, monkeypatch): + self._armed(monkeypatch) + + dark_follow.record_outcome(_KEY, False) + dark_follow._expire_targets_for_tests() + assert len(dark_follow.follow_targets()) == 1 + + dark_follow.record_outcome(_KEY, False) + dark_follow._expire_targets_for_tests() + assert dark_follow.follow_targets() == [] + assert state.dark_follow_dropped == 1 + + def test_a_good_solve_clears_the_streak(self, monkeypatch): + self._armed(monkeypatch) + + dark_follow.record_outcome(_KEY, False) + dark_follow.record_outcome(_KEY, True) + dark_follow.record_outcome(_KEY, False) + dark_follow._expire_targets_for_tests() + + assert len(dark_follow.follow_targets()) == 1 + assert state.dark_follow_dropped == 0 + + def test_the_cooldown_expires(self, monkeypatch): + self._armed(monkeypatch) + monkeypatch.setattr(dark_follow, "DARK_FOLLOW_COOLDOWN_S", 0.0) + + dark_follow.drop_target(_KEY, "test") + dark_follow._expire_targets_for_tests() + + assert len(dark_follow.follow_targets()) == 1 + + def test_rejected_history_records_feed_the_guard(self, monkeypatch): + """The verdicts arrive through _record_solve_history in binding mode — + the one place every outcome of a follow-solve passes through.""" + self._armed(monkeypatch) + s_in = {"follow_key": _KEY, "lane": "dark_follow", "n_nodes": 3} + + for _ in range(2): + solver_mod._record_solve_history("rejected_rms_delay", s_in, _reject_result()) + + dark_follow._expire_targets_for_tests() + assert dark_follow.follow_targets() == [] + assert state.dark_follow_dropped == 1 + + def test_a_published_record_counts_and_clears(self, monkeypatch): + self._armed(monkeypatch) + s_in = {"follow_key": _KEY, "lane": "dark_follow", "n_nodes": 3} + solver_mod._record_solve_history("rejected_rms_delay", s_in, _reject_result()) + solver_mod._record_solve_history("published", s_in, _reject_result(), solve_key=_KEY) + + assert state.dark_follow_published == 1 + dark_follow._expire_targets_for_tests() + assert len(dark_follow.follow_targets()) == 1 + + +class TestAnchorDeadReckoning: + """A follow input's anchor is stale by construction — its guess IS the + prediction of where the anchor drifted to — so the anchor distance check + has to dead-reckon before it measures.""" + + _TS_MS = 1_000_000 + + def _tracks(self, dt_s, speed_ms): + return { + _KEY: { + "lat": _LAT, + "lon": _LON, + "vel_east": 0.0, + "vel_north": speed_ms, + "timestamp_ms": self._TS_MS - int(dt_s * 1000), + "n_nodes": 3, + "solve_count": 5, + } + } + + def _result(self, north_km): + lat, lon = offset_latlon_m(_LAT, _LON, east_m=0.0, north_m=north_km * 1000.0) + return {"lat": lat, "lon": lon, "timestamp_ms": self._TS_MS} + + def test_the_flat_gate_refuses_a_fast_anchor(self): + """15 s of coasting at 270 m/s is 4.05 km of travel; a solve 2 km past + that is 6.05 km from where the entry was last STORED — outside the flat + 6 km gate, purely because the aircraft moved.""" + key, how, _d = solver_mod.multinode_key_decision( + self._tracks(15.0, 270.0), + self._result(6.05), + None, + _KEY, + learned_vel_fn=lambda _k: None, + ) + assert how != "anchor" + + def test_dead_reckoning_honours_it(self): + key, how, dist = solver_mod.multinode_key_decision( + self._tracks(15.0, 270.0), + self._result(6.05), + None, + _KEY, + learned_vel_fn=lambda _k: None, + anchor_dr=True, + ) + assert (key, how) == (_KEY, "anchor") + assert dist == pytest.approx(2.0, abs=0.1) + + def test_dead_reckoning_still_refuses_a_far_solve(self): + """The check's job is unchanged: an anchor whose solve converged + somewhere else entirely is not honoured just because it was named.""" + key, how, _d = solver_mod.multinode_key_decision( + self._tracks(15.0, 270.0), + self._result(30.0), + None, + _KEY, + learned_vel_fn=lambda _k: None, + anchor_dr=True, + ) + assert how != "anchor" diff --git a/docs/solverflow.md b/docs/solverflow.md index c626e5bf..009fe3c8 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -32,7 +32,9 @@ flowchart LR Solid arrows are the live path. Dashed arrows and the grey style mark branches that exist in code but are switched off in production today (inline CV fit, -the bottom-up doppler gate, every mode flag except `KNOWN_LANE_MODE`). Diamonds +the bottom-up doppler gate, every mode flag except `KNOWN_LANE_MODE` and +`DARK_FOLLOW_MODE`, the latter in `shadow` — it claims, solves and records, but +publishes nothing and takes nothing away from the dark lane). Diamonds are gates; a failed gate either drops the item or routes it to a fallback — labeled on the arrow. @@ -285,6 +287,127 @@ LM's SNR weighting maps to a uniform weight of 1.0. | `_ATTEMPT_TTL_S` | 600 s | `known_lane.py:110` | | `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py:205` | +### 3d. Dark track following (`DARK_FOLLOW_MODE`) + +The same inversion, applied to aircraft that have **no transponder**. Where the +known lane claims detections against a dead-reckoned ADS-B fix, this lane +claims them against an established `mn-dark-*` track's Kalman state — the only +identity a dark aircraft has. It exists for two measured problems in the +bottom-up dark lane (§4): + +- **Continuity.** Consecutive solves of one aircraft less than 5 s apart land + on a different key 15% of the time, any gap in solving re-mints the key from + scratch, and neighbours 3 km apart can share one. The key is chosen *after* + the solve, by proximity (`multinode_key_decision`), so it is a guess. +- **The initial guess.** Nothing tells the solver where the aircraft is + expected to be; a dark input starts from a quantised 3 km grid centroid. + +Following fixes both at once, because the pseudo-state that predicts the +observation is also the key: the follow input carries `anchor_key` = the +followed key, and `multinode_key_decision`'s anchor branch keys the solve back +onto the same track. Continuity is then *by construction* rather than by the +proximity scan happening to pick the right neighbour. + +Node-track ids were the obvious cheaper mechanism and are not safe: attaching +each solve to the newest key sharing a `source_track_ids` entry linked the +**wrong aircraft 12%** of the time in a dense metro cluster — the same reason +`_supersession_match` (§6) stopped trusting a bare shared id. + +```mermaid +flowchart TD + build["dark_follow.follow_targets()
rebuilt at most 1/s, TTL-cached"] + build --> gkey{"key starts mn-dark-?"} + gkey -->|"no"| skip0["ADS-B tracks are never followed"]:::inert + gkey --> gcool{"in cooldown?"} + gcool -->|"yes"| skip1["dropped key, waiting out
DARK_FOLLOW_COOLDOWN_S"]:::inert + gcool --> gelig{"age <= MAX_AGE_S 20s
AND solve_count >= 3
AND n_nodes >= 3"} + gelig -->|"no"| skip2["not established enough"]:::inert + gelig --> gkf{"track_filter.learned_velocity
has state?"} + gkf -->|"no"| skip3["nothing to dead-reckon with"]:::inert + gkf --> gsig{"vel sigma <= 60 m/s?"} + gsig -->|"no"| drop["drop_target + cooldown
dark_follow_dropped"]:::inert + gsig -->|"yes"| target["pseudo-state:
lat/lon/alt, KF velocity,
pos+vel sigma, world"] + + target --> claim["known_claiming path 3,
per frame per node"] + claim --> gfree{"detections left by
ADS-B paths 1+2?"} + gfree -->|"none"| skip4["ADS-B always wins"]:::inert + gfree --> gworld{"same world as the node?"} + gworld --> gvis{"_point_in_beam on the
dead-reckoned position"} + gvis --> gate{"Hungarian one-to-one under
widened gates (below)"} + gate -->|"infeasible"| skip5["stays in the dark pool"]:::inert + gate -->|"claimed"| rec["state.known_claims[mn-dark-key]
dark_follow: True, follow_fix
dark_follow_claims"] + + rec --> mode{"DARK_FOLLOW_MODE"} + mode -->|"off"| m0["no targets built at all"]:::inert + mode -->|"shadow"| m1["frame untouched;
pass solves + records
outcome dark_follow_shadow"] + mode -->|"binding"| m2["strip_claimed_detections;
input onto solver_queue"] + + m2 --> gates["normal dark gate stack (§5)
anchor_dr keying, KF smoothing,
published under the SAME key"] + m1 --> guard{"solve rejected?"} + gates --> guard + guard -->|"2 in a row"| drop + guard -->|"published"| streakclear["streak cleared"] + + classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 +``` + +**The widened gates** (`dark_follow.follow_gates`). A dark pseudo-state carries +its own uncertainty, where an ADS-B fix is treated as truth, so the claim gate +is the known lane's gate **plus** that uncertainty projected into observation +space: + +``` +d_gate_us = 10.0 * _gate_scale(dt) + 2 * (pos_sigma_m + vel_sigma_ms * dt) / c_m_per_us +f_gate_hz = 25.0 * _gate_scale(dt) + 2 * vel_sigma_ms * fc_hz / c_m_per_s +``` + +A position error of `s` metres moves the bistatic range by at most `2s` (the +target can be displaced toward both transmitter and receiver); a velocity error +of `u` m/s moves the bistatic Doppler by at most `2u/λ`. Both are worst-case +projections — the true geometry factor is a cosine ≤ 1 — which is the safe +direction for a gate. Both are capped (40 µs / 100 Hz, 4× the base gates): past +there the prediction is not constraining anything and the aircraft should be +re-found bottom-up. + +**The ghost lock-in guard** is load-bearing, not tidiness. Following is a +positive feedback loop — the solve keeps the key alive, the key keeps claiming +detections, and in binding mode those detections never reach the lane that +would disagree. A followed key is therefore dropped for +`DARK_FOLLOW_COOLDOWN_S` on **two consecutive rejected follow-solves** or a +velocity sigma past the ceiling, and the bottom-up lane has to re-find it. +Every follow-solve outcome reaches the guard through one hook in +`solver._record_solve_history` (published, every `rejected_*`, unconverged, and +the shadow pass's own record). + +**Anchor dead-reckoning.** The anchor branch's flat 6 km check compares the +solve against where the entry was last *stored*. For a follow input that is +wrong by construction — its guess IS a prediction of where the anchor drifted +to — and the numbers bite: the dark displacement cap is 6.0 km and the flat +anchor gate is 6.0 km, so a solve at the edge of the gate that let it through +is at the edge of the gate that must key it, before any drift is added; at the +lane's 20 s staleness limit a 270 m/s target adds 5.4 km more. Follow inputs +therefore pass `anchor_dr=True`, which dead-reckons the anchor and applies the +proximity scan's own age-scaled gate. Every other anchored input is unchanged. + +| Constant | Value | File | +|---|---|---| +| `DARK_FOLLOW_MODE` | `shadow` (env) | `core/state.py` | +| `DARK_FOLLOW_MAX_AGE_S` | 20 s (env) | `services/dark_follow.py` | +| `DARK_FOLLOW_MIN_SOLVES` / `DARK_FOLLOW_MIN_NODES` | 3 / 3 | `services/dark_follow.py` | +| `DARK_FOLLOW_MAX_VEL_SIGMA_MS` | 60 m/s (env) | `services/dark_follow.py` | +| `DARK_FOLLOW_INTERVAL_S` | 2.0 s (env) | `services/dark_follow.py` | +| `DARK_FOLLOW_COOLDOWN_S` | 30 s (env) | `services/dark_follow.py` | +| `_MAX_CONSECUTIVE_REJECTS` | 2 | `services/dark_follow.py` | +| `_TARGETS_TTL_S` (pseudo-state cache) | 1.0 s | `services/dark_follow.py` | +| Gate caps `_MAX_DELAY_GATE_US` / `_MAX_DOPPLER_GATE_HZ` | 40 µs / 100 Hz | `services/dark_follow.py` | + +Observability: `/api/test/solver-stats` `counters` carries the funnel +`dark_follow_targets` (a live gauge) → `dark_follow_claims` → +`dark_follow_inputs` → `dark_follow_published`, plus `dark_follow_dropped`. +Records are classified `lane: "dark_follow"` in `lane_split` and kept out of +the bottom-up dark funnel, and each carries `guess_source: "prediction"` and +`follow_key`. + --- ## 4. The dark lane From 376c07dd3f4f1d0d71ea88bfc1b35e572b9eac76 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 17:35:39 +0000 Subject: [PATCH 21/25] Extend the solver-stats payload pins to the follow lane lane_split and the counters block are asserted by exact equality, which is what makes them a contract; the new lane's five counters and its lane_split bucket move those pins. Adds the case they exist for: a follow record is keyed mn-dark-* by design, so only the lane stamp keeps it out of a bottom-up funnel it is not one of. Co-Authored-By: Claude Fable 5.1 --- backend/services/tasks/known_lane.py | 9 +++++++++ backend/services/tasks/solver.py | 12 ++++++++--- backend/tests/test_solver_stats.py | 30 +++++++++++++++++++++++++--- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/backend/services/tasks/known_lane.py b/backend/services/tasks/known_lane.py index c8630e40..2aa8ba7b 100644 --- a/backend/services/tasks/known_lane.py +++ b/backend/services/tasks/known_lane.py @@ -540,6 +540,15 @@ def _build_follow_solver_input(key: str, claims: dict[str, dict]) -> dict | None ``lane``/``guess_source``/``follow_key`` ride through to the history record (solver._record_solve_history stamps all three) so the lane is separable in /api/test/solver-stats without inferring it from the key. + + A CONSEQUENCE WORTH KNOWING: with no cv_epochs on the input, an n=2 follow + solve cannot pass the n=2 confirmation gate and is always withheld, so the + lane publishes at n>=3 only. Left that way on purpose — at n=2 the solver + fits five unknowns to four measurements and returns a zero residual for a + WRONG claim exactly as it does for a right one, which is the same blindness + the gate exists for and is not made safer by the claim having come from a + prediction. A follow target already needs n>=3 to be followed at all; this + just means it needs three claiming nodes as well. """ newest = max(claims.values(), key=lambda c: int(c["ts_ms"])) newest_ts_ms = int(newest["ts_ms"]) diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 4ff017aa..a91948df 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -2253,9 +2253,15 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus elif _key_how == "proximity": state.bump_counter("solver_key_proximity_dark") if _anchor_key: - # Only an anchored solver input (top-down claiming, active - # mode) ever sets s_in["anchor_key"] — this whole block is - # inert off/shadow, by construction, with no mode read here. + # s_in["anchor_key"] is set by exactly two producers: top-down + # claiming in active mode, and a dark-follow input in binding + # mode (services/dark_follow.py). Both are off by default, so + # this block stays inert by construction with no mode read + # here. The counters do not split the two, deliberately: they + # measure the same thing either way — how often an anchor + # named the track the solve actually landed on — and + # follow_key on the history record separates them after the + # fact for anyone who needs it. state.bump_counter("solver_anchored_published") state.bump_counter("solver_anchor_hits" if _key_how == "anchor" else "solver_anchor_fallbacks") # Raw solve position, before smoothing — the history record keeps diff --git a/backend/tests/test_solver_stats.py b/backend/tests/test_solver_stats.py index 67bc599c..b66c8028 100644 --- a/backend/tests/test_solver_stats.py +++ b/backend/tests/test_solver_stats.py @@ -37,6 +37,7 @@ def _rec( adsb_hex=None, known_lane=False, displacement_km=None, + lane=None, ): return { "ts_ms": int((time.time() - age_s) * 1000), @@ -48,6 +49,7 @@ def _rec( "adsb_hex": adsb_hex, "known_lane": known_lane, "displacement_km": displacement_km, + "lane": lane, } @@ -257,6 +259,11 @@ def test_consensus_and_counters_reflect_state(self): state.solver_consensus_fallback = 9 state.solver_consensus_shadow = 10 state.solver_vel_untrusted_published = 11 + state.dark_follow_targets = 13 + state.dark_follow_claims = 14 + state.dark_follow_inputs = 15 + state.dark_follow_published = 16 + state.dark_follow_dropped = 17 out = _solver_window_stats(10.0) assert out["counters"] == { "successes": 5, @@ -268,6 +275,11 @@ def test_consensus_and_counters_reflect_state(self): "queue_drops": 6, "worker_errors": 0, "vel_untrusted_published": 11, + "dark_follow_targets": 13, + "dark_follow_claims": 14, + "dark_follow_inputs": 15, + "dark_follow_published": 16, + "dark_follow_dropped": 17, } assert out["consensus"]["selected"] == 7 assert out["consensus"]["filtered"] == 8 @@ -520,7 +532,7 @@ def test_empty_state_no_division_errors(self): assert out["position_error_km"] == {"median": None, "p90": None, "n": 0} assert out["ghosts"]["live_tracks"] == 0 assert out["ghosts"]["precision_pct"] is None - assert out["lane_split"] == {"dark": 0, "adsb": 0, "known": 0} + assert out["lane_split"] == {"dark": 0, "adsb": 0, "known": 0, "dark_follow": 0} assert out["window_effective_minutes"] == 0.0 @@ -618,7 +630,7 @@ def test_known_records_leave_the_funnel_and_land_in_lane_split(self): _push(_rec("known_truth_match", known_lane=True, displacement_km=0.5)) _push(_rec("known_ghost", known_lane=True, displacement_km=8.0)) out = _solver_window_stats(10.0) - assert out["lane_split"] == {"dark": 2, "adsb": 0, "known": 4} + assert out["lane_split"] == {"dark": 2, "adsb": 0, "known": 4, "dark_follow": 0} assert out["attempts"] == 2 assert out["published"]["total"] == 1 assert out["rejects"] == {"total": 1, "by_reason": {"displacement": 1}} @@ -626,12 +638,24 @@ def test_known_records_leave_the_funnel_and_land_in_lane_split(self): assert "known_truth_match" not in out["rejects"]["by_reason"] assert "known_ghost" not in out["rejects"]["by_reason"] + def test_follow_records_leave_the_dark_funnel(self): + """A dark-follow solve (services/dark_follow.py) is keyed mn-dark-* by + design — it is the same aircraft, reached top-down — so only the + ``lane`` stamp keeps it out of a bottom-up funnel it is not part of.""" + _push(_rec("published", solve_key="mn-dark-1")) + _push(_rec("published", solve_key="mn-dark-2", lane="dark_follow")) + _push(_rec("rejected_displacement", lane="dark_follow")) + out = _solver_window_stats(10.0) + assert out["lane_split"] == {"dark": 1, "adsb": 0, "known": 0, "dark_follow": 2} + assert out["attempts"] == 1 + assert out["rejects"]["total"] == 0 + def test_adsb_lane_records_are_counted_but_not_funnelled(self): _push(_rec("published", solve_key="mn-adsb-a1b2c3")) _push(_rec("rejected_displacement", adsb_hex="a1b2c3")) _push(_rec("published", solve_key="mn-dark-1")) out = _solver_window_stats(10.0) - assert out["lane_split"] == {"dark": 1, "adsb": 2, "known": 0} + assert out["lane_split"] == {"dark": 1, "adsb": 2, "known": 0, "dark_follow": 0} assert out["attempts"] == 1 assert out["rejects"]["total"] == 0 From 0ad8d81e9c82ddc6627d19e88b02c2c2ceda8fa8 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 18:07:08 +0000 Subject: [PATCH 22/25] Sweep and snapshot multinode_tracks under the solver's track lock aircraft_feed's feed build snapshotted state.multinode_tracks and popped expired entries without _MN_TRACKS_LOCK while the solver worker iterates the same dict inside multinode_key_decision under that lock. Live on the test droplet (2026-09-05) this raised "dictionary changed size during iteration" and lost the solve; it surfaced once the dark expiry fell to 30 s and the follow lane raised the publish rate, but the race predates both. Co-Authored-By: Claude Fable 5.1 --- backend/services/aircraft_feed.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/services/aircraft_feed.py b/backend/services/aircraft_feed.py index 1befcf04..1641b373 100644 --- a/backend/services/aircraft_feed.py +++ b/backend/services/aircraft_feed.py @@ -340,7 +340,17 @@ def build_combined_aircraft_json(default_pipeline: PassiveRadarPipeline) -> dict # 3. Multi-node solver stale_mn = [] - for key, r in list(state.multinode_tracks.items()): + # Snapshot and evict under the solver's track lock: the solver worker + # iterates state.multinode_tracks inside multinode_key_decision while + # holding it, and a pop from this thread mid-iteration raised + # "dictionary changed size during iteration" live (2026-09-05). Lazy + # import: solver.py owns the lock and importing it at module level here + # would create a cycle through the task modules. + from services.tasks import solver as _solver_mod + + with _solver_mod._MN_TRACKS_LOCK: + _mn_snapshot = list(state.multinode_tracks.items()) + for key, r in _mn_snapshot: age_s = now - r.get("timestamp_ms", 0) / 1000 if age_s > 60: stale_mn.append(key) @@ -410,7 +420,8 @@ def build_combined_aircraft_json(default_pipeline: PassiveRadarPipeline) -> dict # without bound — enough to trip the anomaly_flood health check. with state.anomaly_lock: state.anomaly_hexes.discard(multinode_hex_from_key(k)) - state.multinode_tracks.pop(k, None) + with _solver_mod._MN_TRACKS_LOCK: + state.multinode_tracks.pop(k, None) # 3b. Singly-claimed ADS-B targets — no seen_hex guard on purpose. A # partially-claimed aircraft can still carry a tracker track keyed by the From 1787b7a0ea330a274c91fd5781db929dbab78aea Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 18:48:14 +0000 Subject: [PATCH 23/25] Let the dark-follow lane own the keys it follows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following an established dark track fixed the lane's own continuity, but the bottom-up lane kept solving the same sky and keying by proximity onto whatever entry was nearest. Measured on test with the lane binding (20 min, 625 six-plus-node dark samples): 224 follow solves published, all anchor-keyed, position error median 0.24 km / p90 1.35 km, and 6+-node dark aircraft on the map 89% of the time against 76% before. The residual misses are all the other lane — of 425 bottom-up solves keyed by proximity onto an existing key, 90 landed on a key owned by a DIFFERENT aircraft (21%), 12 of them onto a key the follow lane had published on within the previous 6 s. A cross-keyed solve moves the entry 5+ km, corrupts the KF velocity it feeds, and can supersede the right key. A tighter spatial gate cannot separate the two cases: same-aircraft re-key distances are p50 1.5 km / p90 4.3 km, mostly n=2 bottom-up solves whose own error is ~2.4 km median, which overlaps the wrong-aircraft population entirely. What can separate them is that the follow lane already supplies every solve an established track needs, so a bottom-up solve arriving at a freshly-followed key is either a duplicate of that aircraft — competing with the anchored solve and dragging the filter — or a different one stealing the key. Neither should join. So in binding mode a key the lane published on within DARK_FOLLOW_OWN_S (6 s, three follow-solve intervals) leaves the proximity scan's candidate set, and a solve landing within DARK_FOLLOW_SHADOW_KM (2.0 km) of the nearest such key is refused outright rather than keyed: outcome shadowed_by_follow, counter dark_bottomup_shadowed, no publish, no KF update, no entry written. Farther out the solve falls through to the non-followed candidates and mints as before. The anchor and ADS-B branches are untouched, which is what keeps the lane's own solves — they are anchored — from shadowing themselves off the map. The ghost guard is deliberately not fed from these records. A shadowed record names a followed key but was produced by the bottom-up lane, and two of them in a row would otherwise drop the very track that refused them, so the guard now reads the solver INPUT's follow_key rather than the record's. Inert unless DARK_FOLLOW_MODE is binding; the default is still shadow. Co-Authored-By: Claude Fable 5.1 --- backend/core/state.py | 9 +- backend/routes/test.py | 7 + backend/services/dark_follow.py | 85 +++++++- backend/services/tasks/solver.py | 319 ++++++++++++++++++----------- backend/tests/test_dark_follow.py | 219 +++++++++++++++++++- backend/tests/test_solver_stats.py | 2 + docs/solverflow.md | 58 +++++- 7 files changed, 573 insertions(+), 126 deletions(-) diff --git a/backend/core/state.py b/backend/core/state.py index 5be36271..50e4b1d2 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -533,6 +533,12 @@ def _adsb_for_seeding() -> dict[str, dict]: dark_follow_inputs: int = 0 dark_follow_published: int = 0 dark_follow_dropped: int = 0 +# Bottom-up dark solves refused at keying because the follow lane owns the key +# they landed on (solver.multinode_key_decision's "shadowed" verdict, binding +# mode only). They passed every gate, so they are counted in solver_successes +# too — this is the difference between a success and a publication on the dark +# lane, and the matching rejects.by_reason entry is "shadowed_by_follow". +dark_bottomup_shadowed: int = 0 # n=2 solves withheld from the map because their track pairing has not (yet) # passed the constant-velocity fit. Counted separately from solver_failures: # the solve succeeded, it simply has not earned publication, and a real target @@ -775,7 +781,7 @@ def _reset_for_tests() -> None: global known_claims_made, known_claim_contentions, known_claims_bound global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects global dark_follow_targets, dark_follow_claims, dark_follow_inputs - global dark_follow_published, dark_follow_dropped + global dark_follow_published, dark_follow_dropped, dark_bottomup_shadowed global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes global coverage_rebuild_backlog global solver_queue_drops, solver_stale_drops, solver_resolve_skips @@ -867,6 +873,7 @@ def _reset_for_tests() -> None: known_claims_world_rejects = 0 dark_follow_targets = dark_follow_claims = dark_follow_inputs = 0 dark_follow_published = dark_follow_dropped = 0 + dark_bottomup_shadowed = 0 coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0 coverage_rebuild_backlog = 0 solver_stale_drops = 0 diff --git a/backend/routes/test.py b/backend/routes/test.py index d9edcfe3..6632f687 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -1212,6 +1212,13 @@ def _solver_window_stats(minutes: float) -> dict: "dark_follow_inputs": state.dark_follow_inputs, "dark_follow_published": state.dark_follow_published, "dark_follow_dropped": state.dark_follow_dropped, + # The other side of the lane: bottom-up dark solves refused at + # keying because the follow lane owns the key they landed on. It + # belongs beside the funnel because it is the same trade — the + # lane keeps a key only if it also stops the bottom-up lane from + # corrupting it — and the matching per-solve records are the + # "shadowed_by_follow" entries in rejects.by_reason. + "dark_bottomup_shadowed": state.dark_bottomup_shadowed, }, } diff --git a/backend/services/dark_follow.py b/backend/services/dark_follow.py index b21459b8..e949195e 100644 --- a/backend/services/dark_follow.py +++ b/backend/services/dark_follow.py @@ -46,7 +46,10 @@ binding — claimed detections leave the dark pool (the same strip_claimed_detections the known lane uses) and the solve goes onto the normal solver queue, keyed onto the followed track by - its anchor. + its anchor. Binding also gives the lane OWNERSHIP of the keys it + follows: a bottom-up solve may not join a key this lane published + on in the last DARK_FOLLOW_OWN_S — see that constant for the + measurement, and recently_followed for the reader. """ import logging @@ -94,6 +97,37 @@ # a follow-solve every 2 s is four refreshes inside the map's 60 s expiry. DARK_FOLLOW_INTERVAL_S = float(os.getenv("DARK_FOLLOW_INTERVAL_S", "2.0")) +# ── Key ownership ──────────────────────────────────────────────────────────── +# How long after a follow-solve publishes on a key that key stays the follow +# lane's, i.e. un-joinable by a bottom-up solve (solver.multinode_key_decision, +# binding mode only). Three follow-solve intervals: a followed track is solved +# every DARK_FOLLOW_INTERVAL_S, so a key still inside this window is one the +# lane is actively refreshing and does not need help keeping alive, while a key +# that has missed three turns is one the lane has stopped answering for and the +# bottom-up lane should be free to claim again. +# +# WHY OWNERSHIP AT ALL. Measured on test with the lane binding (20 min, 625 +# six-plus-node dark samples): of 425 bottom-up solves keyed by proximity onto +# an existing key, 90 landed on a key belonging to a DIFFERENT aircraft (21%), +# 12 of them onto a key the follow lane had published on within the previous +# 6 s. A cross-keyed solve moves the entry 5+ km, corrupts the KF velocity it +# feeds, and can supersede the right key. A tighter spatial gate cannot +# separate the two cases: same-aircraft re-key distances are p50 1.5 km / +# p90 4.3 km, overlapping the wrong-aircraft population entirely. What CAN +# separate them is that the follow lane already supplies every solve an +# established track needs — so near a freshly-followed key a bottom-up solve is +# either a duplicate (it competes with the anchored solve and drags the filter) +# or a different aircraft (it steals the key). Neither should join. +DARK_FOLLOW_OWN_S = float(os.getenv("DARK_FOLLOW_OWN_S", "6.0")) +# How close a bottom-up solve has to land to a followed key before it is +# refused outright rather than merely kept off that key. Inside this radius +# the two are the same aircraft often enough that minting a second key would +# just fragment the track; outside it the solve is plausibly a neighbour the +# follow lane knows nothing about and deserves a key of its own. Well under +# the 6 km proximity gate on purpose — this is a "these are the same target" +# radius, not an association gate. +DARK_FOLLOW_SHADOW_KM = float(os.getenv("DARK_FOLLOW_SHADOW_KM", "2.0")) + # Consecutive rejected follow-solves that drop a key. Two, not one: a single # reject is routinely a bad epoch (one node's contaminated measurement trips # the rms gate), while two in a row is the prediction itself being wrong. @@ -138,6 +172,16 @@ _reject_streak: dict[str, int] = {} _cooldown_until: dict[str, float] = {} +# Ownership state: key → the measurement epoch of the newest follow-solve that +# published on it (see note_follow_publish for why the measurement clock and +# not wall time). Dict-level TTL, same shape and reason as the guard maps +# above: mn-dark-* keys churn for the process lifetime, so nothing keyed by one +# may grow unbounded. 60 s is the map's own expiry — a key with no follow +# publish for that long has no entry left to own. +_FOLLOWED_TTL_S = 60.0 +_FOLLOWED_LOCK = threading.Lock() +_last_follow_publish: dict[str, float] = {} + def _reset_for_tests() -> None: """Drop the target cache and the guard state. Tests only.""" @@ -148,6 +192,8 @@ def _reset_for_tests() -> None: with _GUARD_LOCK: _reject_streak.clear() _cooldown_until.clear() + with _FOLLOWED_LOCK: + _last_follow_publish.clear() state.dark_follow_targets = 0 @@ -212,6 +258,43 @@ def record_outcome(key: str, ok: bool) -> None: drop_target(key, f"{streak} consecutive rejected follow-solves") +def note_follow_publish(key: str, ts_s: float) -> None: + """Record that a follow-lane solve published on ``key`` at epoch ``ts_s``. + + ``ts_s`` is the solve's MEASUREMENT epoch, not wall time, and that is not + an accident: the only reader is solver.multinode_key_decision, which is + deliberately clock-free — every time-of-day it uses arrives on the solve it + is judging — so that the keying rule stays replayable against recorded + history. Feeding it wall time here would make the one gate that decides + key ownership the one thing a replay could not reproduce. + + Called from the single point every follow-solve outcome passes through + (solver._record_solve_history), so there is no path that publishes on a + followed key without marking it. + """ + if not key or ts_s <= 0.0: + return + with _FOLLOWED_LOCK: + _last_follow_publish[key] = ts_s + cutoff = ts_s - _FOLLOWED_TTL_S + for k in [k for k, t in _last_follow_publish.items() if t < cutoff]: + del _last_follow_publish[k] + + +def recently_followed(key: str, now_s: float, within_s: float = DARK_FOLLOW_OWN_S) -> bool: + """Did the follow lane publish on ``key`` within ``within_s`` of ``now_s``? + + Symmetric in time on purpose. Solves reach the worker out of order (three + worker threads, a queue, and a per-key rate limit that batches claims), so + a bottom-up solve whose epoch sits just BEFORE the follow-solve's is + looking at the same instant of the same aircraft as one just after it, and + the ownership answer has to be the same for both. + """ + with _FOLLOWED_LOCK: + last = _last_follow_publish.get(key) + return last is not None and abs(now_s - last) <= within_s + + def _in_cooldown(key: str, now_mono: float) -> bool: with _GUARD_LOCK: return _cooldown_until.get(key, 0.0) > now_mono diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index a91948df..1e2d03ca 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -757,11 +757,14 @@ def multinode_key_decision( Caller holds _MN_TRACKS_LOCK — it reads `tracks` and the caller writes back into it under the same lock. Returns (key, how, dist_km) with - how in {"adsb", "anchor", "proximity", "minted"}; dist_km is how far this - solve landed from the entry it was keyed onto (dead-reckoned, for the - proximity branch) and None where nothing was matched — "adsb" and - "minted". The caller stamps both onto the solve-history record, which is - the only way to tell a re-key apart from a fragment after the fact. + how in {"adsb", "anchor", "proximity", "shadowed", "minted"}; dist_km is + how far this solve landed from the entry it was keyed onto (dead-reckoned, + for the proximity and shadowed branches) and None where nothing was + matched — "adsb" and "minted". The caller stamps both onto the + solve-history record, which is the only way to tell a re-key apart from a + fragment after the fact. "shadowed" is the one verdict that is NOT a key: + it names the followed key this solve was refused in favour of, and the + caller must treat it as a rejection (see _process_solver_item). Order: 1. ADS-B-tagged solves key on the transponder hex — unconditional, and @@ -787,6 +790,18 @@ def multinode_key_decision( does; candidates compete on distance normalised by their own gate, so a fresh close entry beats an old far one rather than the scan simply taking whichever is nearer in kilometres. + + KEY OWNERSHIP (DARK_FOLLOW_MODE=binding only). A key the follow + lane published on within dark_follow.DARK_FOLLOW_OWN_S is removed + from this scan's candidates entirely, and if the nearest such key is + within DARK_FOLLOW_SHADOW_KM the solve is refused ("shadowed") + instead of keyed at all. The follow lane already supplies every + solve an established track needs, so a bottom-up solve arriving at + one of its keys is either a duplicate — competing with the anchored + solve and dragging the filter — or a different aircraft stealing the + key; 21% of proximity joins measured on test were the latter. See + dark_follow.DARK_FOLLOW_OWN_S for why a tighter gate cannot separate + the two. 4. Mint. This key only needs to be unique at birth; every later solve associates to it above (by proximity, or by anchor once a claim forms), so it stays stable. @@ -848,6 +863,13 @@ def multinode_key_decision( best_key: str | None = None best_score = 1.0 best_dist: float | None = None + # Key ownership: the nearest key the follow lane is currently answering + # for, and how far this solve landed from it. Only collected for a + # bottom-up solve in binding mode — an anchored or ADS-B solve names the + # aircraft it is of, and neither branch above reaches this scan. + shadow_scan = not anchor_key and dark_follow.mode() == "binding" + shadow_key: str | None = None + shadow_dist: float | None = None for key, prev in tracks.items(): # Only dark tracks are claimable; an untagged solve must never steal the @@ -871,10 +893,25 @@ def multinode_key_decision( north_m=vel_north_ms * dt, ) d = _haversine_km(lat, lon, p_lat, p_lon) + # A key the follow lane just published on is not joinable bottom-up, + # whatever the distance says — see dark_follow.DARK_FOLLOW_OWN_S for + # the measurement. It still competes to SHADOW this solve below, so + # the scan has to remember the nearest one rather than skipping it. + if shadow_scan and dark_follow.recently_followed(key, ts_s, dark_follow.DARK_FOLLOW_OWN_S): + if shadow_dist is None or d < shadow_dist: + shadow_key, shadow_dist = key, d + continue score = d / _mn_assoc_gate_km(dt, max_dist_km) if score < best_score: best_key, best_score, best_dist = key, score, d + # Close enough to a followed key that this solve is the same aircraft the + # follow lane is already solving: refuse it outright rather than mint a + # second key for a target that already has one. Farther away it falls + # through to the non-followed candidates and, failing those, mints — the + # one thing it may never do is join the followed key. + if shadow_key is not None and shadow_dist <= dark_follow.DARK_FOLLOW_SHADOW_KM: + return shadow_key, "shadowed", shadow_dist if best_key is not None: return best_key, "proximity", best_dist # No claimant — a genuinely new target. @@ -1459,6 +1496,7 @@ def _record_solve_history( chi2_per_dof: float | None = None, key_how: str | None = None, key_dist_km: float | None = None, + follow_key: str | None = None, superseded_keys: list[str] | None = None, superseded_blocked: int | None = None, extra: dict | None = None, @@ -1475,7 +1513,16 @@ def _record_solve_history( solve — which branch produced solve_key, and how far the solve landed from the entry it was keyed onto. Only the publish path has run the keying rule, so both are None on every reject (the key is minted after the - gates, which is also why solver_hex is None there). + gates, which is also why solver_hex is None there) — with one exception: + a ``shadowed_by_follow`` reject IS the keying rule's verdict, and carries + key_how/key_dist_km plus ``follow_key`` naming the followed key it was + refused in favour of. + + ``follow_key`` overrides the input's own follow_key for the record only. + The ghost guard below is deliberately NOT fed from it: the guard judges + solves the FOLLOW lane produced, and a shadowed record is a bottom-up + solve that merely names a followed key — feeding it there would let the + bottom-up lane's refusals drop the very track that refused them. ``superseded_keys``/``superseded_blocked`` are the other side of that decision: which existing entries this publish popped as the same aircraft @@ -1521,7 +1568,7 @@ def _record_solve_history( # carries — and follow_key names the track that predicted it. "lane": s.get("lane"), "guess_source": s.get("guess_source"), - "follow_key": s.get("follow_key"), + "follow_key": follow_key or s.get("follow_key"), "raw_lat": round(float(raw_lat), 6) if raw_lat is not None else None, "raw_lon": round(float(raw_lon), 6) if raw_lon is not None else None, "lat": round(float(r["lat"]), 6) if outcome == "published" else None, @@ -1586,7 +1633,10 @@ def _record_solve_history( } if extra: rec.update(extra) - _follow_key = rec.get("follow_key") + # The INPUT's follow_key, not the record's: a shadowed_by_follow reject + # names a followed key it was refused in favour of, and that key's guard + # must not hear about a solve the follow lane never made. + _follow_key = s.get("follow_key") if _follow_key: # The dark-follow ghost guard (services/dark_follow.py) needs a verdict # for every follow-solve, and this is the one place all of them pass @@ -1599,6 +1649,14 @@ def _record_solve_history( dark_follow.record_outcome(_follow_key, bool(rec.get("follow_ok", outcome == "published"))) if outcome == "published": state.bump_counter("dark_follow_published") + # ...and the lane now owns the key it published on, for + # DARK_FOLLOW_OWN_S. Stamped with the MEASUREMENT epoch, because + # the reader (multinode_key_decision) compares it against another + # solve's measurement epoch and is deliberately clock-free. The + # key published on, not the anchor: on the rare anchor fallback + # the lane's solve went somewhere else, and that is the entry it + # is now refreshing. + dark_follow.note_follow_publish(solve_key or _follow_key, rec["measurement_ts_ms"] / 1000.0) if raw_lat is not None and raw_lon is not None: meas_ts_s = (rec["measurement_ts_ms"] or now_ms) / 1000.0 rec.update(_gt_for_record(rec["adsb_hex"], float(raw_lat), float(raw_lon), meas_ts_s)) @@ -2237,117 +2295,140 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus # the distance check it must be judged by. anchor_dr=bool(isinstance(s_in, dict) and s_in.get("follow_key")), ) - # Dark-lane key births vs re-keys. The fragmentation question is - # "how often does one aircraft get a second key", and the only - # place that is decided is right here — solver_successes counts - # solves, distinct_keys counts survivors, neither counts the - # decision. Dark only: the ADS-B lane keys off the transponder - # hex unconditionally and has no decision to observe. Anchor - # hits are deliberately in neither counter; solver_anchor_hits - # already carries them, and double-counting them here would make - # minted + proximity stop summing to the dark decisions this - # gate actually made. - if key.startswith("mn-dark-"): - if _key_how == "minted": - state.bump_counter("solver_key_minted_dark") - elif _key_how == "proximity": - state.bump_counter("solver_key_proximity_dark") - if _anchor_key: - # s_in["anchor_key"] is set by exactly two producers: top-down - # claiming in active mode, and a dark-follow input in binding - # mode (services/dark_follow.py). Both are off by default, so - # this block stays inert by construction with no mode read - # here. The counters do not split the two, deliberately: they - # measure the same thing either way — how often an anchor - # named the track the solve actually landed on — and - # follow_key on the history record separates them after the - # fact for anyone who needs it. - state.bump_counter("solver_anchored_published") - state.bump_counter("solver_anchor_hits" if _key_how == "anchor" else "solver_anchor_fallbacks") - # Raw solve position, before smoothing — the history record keeps - # both so display-side drift can be separated from solver error. - _raw_lat, _raw_lon = result["lat"], result["lon"] - # Multi-epoch averaging cuts single-frame noise by ~√K. Originally - # n=2-with-ADS-B only — production showed dark targets (where MLAT - # is the only position source) were the one population left raw. - # Smoothing now runs through the env-gated KF in - # services/track_filter.py, with this module's EWMA kept as the - # TRACK_SMOOTHER=ewma fallback. - result = track_filter.smooth_solve(result, key, _adsb_hex, ewma_fn=_ewma_smooth_track) - prev = state.multinode_tracks.get(key) - if prev: - # Latch: a tracker flag raised on an earlier solve holds for - # the multinode track's lifetime (≤60 s expiry) even if the - # contributing track has since despawned or gone quiet. - result["is_anomalous"] = bool(result.get("is_anomalous")) or bool(prev.get("is_anomalous")) - result["anomaly_types"] = sorted( - set(result.get("anomaly_types", [])) | set(prev.get("anomaly_types", [])) - ) - # Source-track identity: the single-node track ids this solve was - # built from. Used below for supersession and carried into the - # history record so a bad map marker can be traced to its inputs. - result["source_track_ids"] = sorted(s_in.get("track_ids") or []) if isinstance(s_in, dict) else [] - - # Supersession: an earlier entry that is THIS aircraft, under a - # key the proximity match (multinode_key_decision) missed, is - # replaced now rather than left rendering beside the new one for - # up to 60 s. solve_count carries forward so the re-solved - # aircraft does not fall back under the n=2 gate below. - # - # A shared source track id is the cheap filter, not the rule. The - # premise this block used to carry — "one aircraft is one set of - # source tracks" — is false: single-node tracker tracks are shared - # between the association candidates of DIFFERENT aircraft (74 of - # 178 track ids in a 6 min live window appeared in published solves - # of more than one ground-truth aircraft), so popping on the shared - # id alone destroyed a live neighbour's key 36 times in 44 - # supersessions — 41 of them beyond the association gate, 43 under - # 15 s old — and the victim's next solve minted a fresh key (dark - # keys churning at 7.4/min with a 7 s median lifetime). Now - # _supersession_match has to agree: the old entry dead-reckons - # into the gate, or its inputs are a subset of this solve's. - # Replayed over the same solves that cuts mints 47 -> 22 and - # cross-aircraft pops 36 -> 7. Refusals are counted - # (mn_superseded_blocked), not - # silent — the shared-id signal is mostly contamination and the - # panel has to be able to see that. - # - # Unchanged by anchor honoring: `old_key == key: continue` below - # already protects an anchor from superseding itself, and a - # proximity-minted fragment built from exactly the anchor's source - # tracks merging INTO the anchor (old_key != key, key == - # anchor_key) is the identical-inputs branch (b) of the predicate — - # exactly the fragmentation-collapse this whole feature exists for. - max_superseded_count = 0 - _superseded_keys: list[str] = [] - _superseded_blocked = 0 - if result["source_track_ids"]: - new_ids = set(result["source_track_ids"]) - _ts_ms = result.get("timestamp_ms") or 0 - for old_key, old_r in list(state.multinode_tracks.items()): - if old_key == key: - continue - if not new_ids.intersection(old_r.get("source_track_ids") or ()): - continue - matched, _ = _supersession_match(old_key, old_r, new_ids, _raw_lat, _raw_lon, _ts_ms) - if not matched: - _superseded_blocked += 1 - state.bump_counter("mn_superseded_blocked") - continue - state.multinode_tracks.pop(old_key, None) - with state.anomaly_lock: - state.anomaly_hexes.discard(multinode_hex_from_key(old_key)) - with _MN_POS_HISTORY_LOCK: - _MN_POS_HISTORY.pop(old_key, None) - track_filter.drop_key(old_key) - max_superseded_count = max(max_superseded_count, old_r.get("solve_count", 0)) - _superseded_keys.append(old_key) - state.bump_counter("mn_superseded") - - result["solve_count"] = max(prev.get("solve_count", 0) if prev else 0, max_superseded_count) + 1 - state.multinode_tracks[key] = result - if trim_meta: - state.bump_counter("solver_trimmed") + # Key ownership (DARK_FOLLOW_MODE=binding). A bottom-up solve + # that landed on a key the follow lane is answering for is not + # keyed at all: it is either a duplicate of the anchored solve + # already refreshing that key or a different aircraft about to + # steal it, and both drag the entry and its filter. The verdict + # is taken here, under the same lock as the decision, so nothing + # about the entry can change between deciding and refusing; the + # record and the counter are emitted outside it, as every other + # outcome's are. + _shadow_key = key if _key_how == "shadowed" else None + if _shadow_key is None: + # Dark-lane key births vs re-keys. The fragmentation question is + # "how often does one aircraft get a second key", and the only + # place that is decided is right here — solver_successes counts + # solves, distinct_keys counts survivors, neither counts the + # decision. Dark only: the ADS-B lane keys off the transponder + # hex unconditionally and has no decision to observe. Anchor + # hits are deliberately in neither counter; solver_anchor_hits + # already carries them, and double-counting them here would make + # minted + proximity stop summing to the dark decisions this + # gate actually made. + if key.startswith("mn-dark-"): + if _key_how == "minted": + state.bump_counter("solver_key_minted_dark") + elif _key_how == "proximity": + state.bump_counter("solver_key_proximity_dark") + if _anchor_key: + # s_in["anchor_key"] is set by exactly two producers: top-down + # claiming in active mode, and a dark-follow input in binding + # mode (services/dark_follow.py). Both are off by default, so + # this block stays inert by construction with no mode read + # here. The counters do not split the two, deliberately: they + # measure the same thing either way — how often an anchor + # named the track the solve actually landed on — and + # follow_key on the history record separates them after the + # fact for anyone who needs it. + state.bump_counter("solver_anchored_published") + state.bump_counter("solver_anchor_hits" if _key_how == "anchor" else "solver_anchor_fallbacks") + # Raw solve position, before smoothing — the history record keeps + # both so display-side drift can be separated from solver error. + _raw_lat, _raw_lon = result["lat"], result["lon"] + # Multi-epoch averaging cuts single-frame noise by ~√K. Originally + # n=2-with-ADS-B only — production showed dark targets (where MLAT + # is the only position source) were the one population left raw. + # Smoothing now runs through the env-gated KF in + # services/track_filter.py, with this module's EWMA kept as the + # TRACK_SMOOTHER=ewma fallback. + result = track_filter.smooth_solve(result, key, _adsb_hex, ewma_fn=_ewma_smooth_track) + prev = state.multinode_tracks.get(key) + if prev: + # Latch: a tracker flag raised on an earlier solve holds for + # the multinode track's lifetime (≤60 s expiry) even if the + # contributing track has since despawned or gone quiet. + result["is_anomalous"] = bool(result.get("is_anomalous")) or bool(prev.get("is_anomalous")) + result["anomaly_types"] = sorted( + set(result.get("anomaly_types", [])) | set(prev.get("anomaly_types", [])) + ) + # Source-track identity: the single-node track ids this solve was + # built from. Used below for supersession and carried into the + # history record so a bad map marker can be traced to its inputs. + result["source_track_ids"] = sorted(s_in.get("track_ids") or []) if isinstance(s_in, dict) else [] + + # Supersession: an earlier entry that is THIS aircraft, under a + # key the proximity match (multinode_key_decision) missed, is + # replaced now rather than left rendering beside the new one for + # up to 60 s. solve_count carries forward so the re-solved + # aircraft does not fall back under the n=2 gate below. + # + # A shared source track id is the cheap filter, not the rule. The + # premise this block used to carry — "one aircraft is one set of + # source tracks" — is false: single-node tracker tracks are shared + # between the association candidates of DIFFERENT aircraft (74 of + # 178 track ids in a 6 min live window appeared in published solves + # of more than one ground-truth aircraft), so popping on the shared + # id alone destroyed a live neighbour's key 36 times in 44 + # supersessions — 41 of them beyond the association gate, 43 under + # 15 s old — and the victim's next solve minted a fresh key (dark + # keys churning at 7.4/min with a 7 s median lifetime). Now + # _supersession_match has to agree: the old entry dead-reckons + # into the gate, or its inputs are a subset of this solve's. + # Replayed over the same solves that cuts mints 47 -> 22 and + # cross-aircraft pops 36 -> 7. Refusals are counted + # (mn_superseded_blocked), not + # silent — the shared-id signal is mostly contamination and the + # panel has to be able to see that. + # + # Unchanged by anchor honoring: `old_key == key: continue` below + # already protects an anchor from superseding itself, and a + # proximity-minted fragment built from exactly the anchor's source + # tracks merging INTO the anchor (old_key != key, key == + # anchor_key) is the identical-inputs branch (b) of the predicate — + # exactly the fragmentation-collapse this whole feature exists for. + max_superseded_count = 0 + _superseded_keys: list[str] = [] + _superseded_blocked = 0 + if result["source_track_ids"]: + new_ids = set(result["source_track_ids"]) + _ts_ms = result.get("timestamp_ms") or 0 + for old_key, old_r in list(state.multinode_tracks.items()): + if old_key == key: + continue + if not new_ids.intersection(old_r.get("source_track_ids") or ()): + continue + matched, _ = _supersession_match(old_key, old_r, new_ids, _raw_lat, _raw_lon, _ts_ms) + if not matched: + _superseded_blocked += 1 + state.bump_counter("mn_superseded_blocked") + continue + state.multinode_tracks.pop(old_key, None) + with state.anomaly_lock: + state.anomaly_hexes.discard(multinode_hex_from_key(old_key)) + with _MN_POS_HISTORY_LOCK: + _MN_POS_HISTORY.pop(old_key, None) + track_filter.drop_key(old_key) + max_superseded_count = max(max_superseded_count, old_r.get("solve_count", 0)) + _superseded_keys.append(old_key) + state.bump_counter("mn_superseded") + + result["solve_count"] = max(prev.get("solve_count", 0) if prev else 0, max_superseded_count) + 1 + state.multinode_tracks[key] = result + if trim_meta: + state.bump_counter("solver_trimmed") + if _shadow_key is not None: + state.bump_counter("dark_bottomup_shadowed") + _record_solve_history( + "shadowed_by_follow", + s_in, + result, + follow_key=_shadow_key, + key_how=_key_how, + key_dist_km=_key_dist_km, + extra=_extra, + ) + return result # Append a snapshot to the track-archive buffer for Parquet persistence. # solve_ts_ms records when the solve completed (server wallclock) so # analysts can measure end-to-end latency vs. result["timestamp_ms"]. diff --git a/backend/tests/test_dark_follow.py b/backend/tests/test_dark_follow.py index 07a547f7..5d1b5d68 100644 --- a/backend/tests/test_dark_follow.py +++ b/backend/tests/test_dark_follow.py @@ -18,7 +18,10 @@ - binding mode removing the claimed detections from the frame the dark lane sees (frame_processor); - the ghost guard: two rejected follow-solves drop the key for the cooldown, - including when the verdicts arrive through _record_solve_history. + including when the verdicts arrive through _record_solve_history; +- key ownership in binding mode: a bottom-up solve may not join a key the lane + just published on, is refused outright inside DARK_FOLLOW_SHADOW_KM, and + those refusals are invisible to the guard. Style follows test_known_claiming.py (registered associator geometry, frames built around a real predicted observation) and test_solver_anchor.py. @@ -626,3 +629,217 @@ def test_dead_reckoning_still_refuses_a_far_solve(self): anchor_dr=True, ) assert how != "anchor" + + +class TestKeyOwnership: + """A key the follow lane just published on is not the bottom-up lane's to + join. + + Binding mode only, and the reason is measured rather than aesthetic: 21% of + bottom-up proximity joins on test landed on a key belonging to a DIFFERENT + aircraft, and same-aircraft re-key distances (p50 1.5 km) overlap the + wrong-aircraft ones entirely, so no tighter spatial gate separates them. + See dark_follow.DARK_FOLLOW_OWN_S. + """ + + _TS_MS = 2_000_000 + _TS_S = _TS_MS / 1000.0 + + def setup_method(self): + dark_follow._reset_for_tests() + + def teardown_method(self): + dark_follow._reset_for_tests() + + def _tracks(self, dt_s=1.0): + """One live dark entry at the reference position, last solved dt_s ago + and not moving — so the distance below is exactly the offset.""" + return { + _KEY: { + "lat": _LAT, + "lon": _LON, + "vel_east": 0.0, + "vel_north": 0.0, + "timestamp_ms": self._TS_MS - int(dt_s * 1000), + "n_nodes": 3, + "solve_count": 5, + } + } + + def _result(self, north_km): + lat, lon = offset_latlon_m(_LAT, _LON, east_m=0.0, north_m=north_km * 1000.0) + return {"lat": lat, "lon": lon, "timestamp_ms": self._TS_MS} + + def _decide(self, north_km, anchor_key=None, dt_s=1.0): + return solver_mod.multinode_key_decision( + self._tracks(dt_s), + self._result(north_km), + None, + anchor_key, + learned_vel_fn=lambda _k: None, + ) + + def test_a_solve_next_to_a_freshly_followed_key_is_shadowed(self, monkeypatch): + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding") + dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0) + key, how, dist = self._decide(1.0) + assert (key, how) == (_KEY, "shadowed") + assert dist == pytest.approx(1.0, abs=0.05) + + def test_a_solve_further_out_mints_rather_than_joining(self, monkeypatch): + """4 km is well inside the 6 km proximity gate — without ownership + this solve joins the followed key, which is the bug. It gets its own + key instead: too far to be the same aircraft, and never the followed + one's.""" + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding") + dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0) + key, how, _dist = self._decide(4.0) + assert how == "minted" + assert key != _KEY + + def test_ownership_expires(self, monkeypatch): + """Past DARK_FOLLOW_OWN_S the lane has stopped answering for the key + (three missed follow-solve intervals), so the bottom-up lane may have + it back.""" + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding") + dark_follow.note_follow_publish(_KEY, self._TS_S - 20.0) + key, how, _dist = self._decide(1.0) + assert (key, how) == (_KEY, "proximity") + + @pytest.mark.parametrize("mode", ["shadow", "off"]) + def test_the_inert_modes_key_exactly_as_before(self, monkeypatch, mode): + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", mode) + dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0) + key, how, _dist = self._decide(1.0) + assert (key, how) == (_KEY, "proximity") + + def test_the_follow_lanes_own_solve_still_lands_on_its_key(self, monkeypatch): + """Ownership is a rule about BOTTOM-UP solves. The follow lane's own + solves are anchored and return from the anchor branch, which never + reaches the proximity scan — otherwise the lane would shadow itself + off the map.""" + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding") + dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0) + key, how, dist = self._decide(1.0, anchor_key=_KEY) + assert (key, how) == (_KEY, "anchor") + assert dist == pytest.approx(1.0, abs=0.05) + + def test_an_n2_solve_cannot_join_a_followed_n3_key(self, monkeypatch): + """The population the ownership rule is aimed at: an n=2 bottom-up + solve, whose own position error is ~2.4 km median, arriving at a key + the follow lane is refreshing from n>=3 measurements.""" + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding") + dark_follow.note_follow_publish(_KEY, self._TS_S - 2.0) + result = self._result(1.5) + result["n_nodes"] = 2 + key, how, _dist = solver_mod.multinode_key_decision( + self._tracks(), + result, + None, + None, + learned_vel_fn=lambda _k: None, + ) + assert (key, how) == (_KEY, "shadowed") + + +class TestShadowedSolveIsARejection: + """What the solver worker does with a "shadowed" verdict: record it, count + it, and touch nothing else.""" + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + dark_follow._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + dark_follow._reset_for_tests() + + def _entry(self, ts_ms): + return { + "success": True, + "lat": _LAT, + "lon": _LON, + "alt_m": _ALT_M, + "vel_east": 0.0, + "vel_north": 0.0, + "n_nodes": 3, + "solve_count": 5, + "timestamp_ms": ts_ms - 1000, + "contributing_node_ids": [_NODE_ID], + } + + def _solve_fn(self, ts_ms, north_km): + lat, lon = offset_latlon_m(_LAT, _LON, east_m=0.0, north_m=north_km * 1000.0) + + def fn(_s_in, _cfgs): + return { + "success": True, + "lat": lat, + "lon": lon, + "alt_m": _ALT_M, + "vel_east": 0.0, + "vel_north": 0.0, + "rms_delay": 1.0, + "rms_doppler": 5.0, + "n_nodes": 3, + "n_measurements": 3, + "timestamp_ms": ts_ms, + "contributing_node_ids": ["n1", "n2", "n3"], + } + + return fn + + def _run(self, monkeypatch, north_km=1.0, follow_age_s=2.0): + monkeypatch.setattr(state, "DARK_FOLLOW_MODE", "binding") + ts_ms = int(time.time() * 1000) + state.multinode_tracks[_KEY] = self._entry(ts_ms) + dark_follow.note_follow_publish(_KEY, ts_ms / 1000.0 - follow_age_s) + solver_mod._process_solver_item( + ({"n_nodes": 3}, {}, time.time()), + self._solve_fn(ts_ms, north_km), + ) + return ts_ms + + def test_the_solve_is_recorded_counted_and_not_published(self, monkeypatch): + ts_ms = self._run(monkeypatch) + assert state.dark_bottomup_shadowed == 1 + # The followed entry is byte-for-byte what it was: no new position, no + # solve_count bump, no smoothing, and no second key minted beside it. + assert list(state.multinode_tracks) == [_KEY] + assert state.multinode_tracks[_KEY] == self._entry(ts_ms) + assert len(state.mlat_solve_history) == 1 + rec = state.mlat_solve_history[0] + assert rec["outcome"] == "shadowed_by_follow" + assert rec["follow_key"] == _KEY + assert rec["key_how"] == "shadowed" + assert rec["key_dist_km"] == pytest.approx(1.0, abs=0.05) + # A reject has no key of its own, shadowed or otherwise. + assert rec["solve_key"] is None + assert rec["lat"] is None + + def test_a_solve_the_lane_does_not_own_still_publishes(self, monkeypatch): + """The same solve with the ownership window expired — the control that + says the assertions above are about ownership and not about the + harness.""" + self._run(monkeypatch, follow_age_s=30.0) + assert state.dark_bottomup_shadowed == 0 + assert state.mlat_solve_history[0]["outcome"] == "published" + + def test_refusals_never_reach_the_follow_ghost_guard(self, monkeypatch): + """A shadowed record names a followed key but was not produced BY the + follow lane, so the guard must not hear about it — otherwise the + bottom-up lane's refusals would drop the very track that refused + them, twice in a row being enough.""" + _kf(monkeypatch) + ts_ms = self._run(monkeypatch) + state.multinode_tracks[_KEY] = self._entry(ts_ms) + dark_follow.note_follow_publish(_KEY, ts_ms / 1000.0 - 2.0) + solver_mod._process_solver_item( + ({"n_nodes": 3}, {}, time.time()), + self._solve_fn(ts_ms, 1.0), + ) + assert state.dark_bottomup_shadowed == 2 + # Two rejects in a row is exactly what drops a followed key. It is + # still a target, so the guard never saw them. + assert [t["key"] for t in dark_follow.follow_targets()] == [_KEY] diff --git a/backend/tests/test_solver_stats.py b/backend/tests/test_solver_stats.py index b66c8028..d75d0d9c 100644 --- a/backend/tests/test_solver_stats.py +++ b/backend/tests/test_solver_stats.py @@ -264,6 +264,7 @@ def test_consensus_and_counters_reflect_state(self): state.dark_follow_inputs = 15 state.dark_follow_published = 16 state.dark_follow_dropped = 17 + state.dark_bottomup_shadowed = 18 out = _solver_window_stats(10.0) assert out["counters"] == { "successes": 5, @@ -280,6 +281,7 @@ def test_consensus_and_counters_reflect_state(self): "dark_follow_inputs": 15, "dark_follow_published": 16, "dark_follow_dropped": 17, + "dark_bottomup_shadowed": 18, } assert out["consensus"]["selected"] == 7 assert out["consensus"]["filtered"] == 8 diff --git a/docs/solverflow.md b/docs/solverflow.md index 009fe3c8..847c727a 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -389,9 +389,56 @@ lane's 20 s staleness limit a 270 m/s target adds 5.4 km more. Follow inputs therefore pass `anchor_dr=True`, which dead-reckons the anchor and applies the proximity scan's own age-scaled gate. Every other anchored input is unchanged. +**Key ownership.** Following an aircraft is not enough on its own — the +bottom-up lane keeps solving the same sky, and its solves are still keyed by +proximity onto whatever entry is nearest. Measured on test with the lane +binding (20 min, 625 six-plus-node dark samples, `DARK_FOLLOW_MODE=binding`): + +| | | +|---|---| +| follow solves published | 224, **all** anchor-keyed | +| follow-solve position error | median 0.24 km, p90 1.35 km | +| dark aircraft with 6+ nodes in cone shown on the map | **89%** (76% before the lane) | +| bottom-up solves keyed by proximity onto an existing key | 425 | +| ...that landed on a key owned by a **different** aircraft | 90 (**21%**) | +| ...onto a key the follow lane had published on within 6 s | 12 | + +A cross-keyed solve moves the entry 5+ km, corrupts the KF velocity it feeds, +and can supersede the right key. A tighter spatial gate cannot separate the two +populations: same-aircraft re-key distances are p50 1.5 km / p90 4.3 km (mostly +n=2 bottom-up solves whose own position error is ~2.4 km median), which overlaps +the wrong-aircraft distances entirely. What *can* separate them is that the +follow lane already supplies every solve an established track needs — so a +bottom-up solve arriving at a freshly-followed key is either a duplicate of that +aircraft (harmful: it competes with the anchored solve and drags the filter) or +a different one (harmful: it steals the key). Neither should join. + +So in **binding mode only**, `multinode_key_decision`'s proximity scan gives the +follow lane ownership of the keys it follows: + +- a key the lane published on within `DARK_FOLLOW_OWN_S` (6 s, three follow-solve + intervals) is **not a join candidate** for a bottom-up (non-anchored, + non-ADS-B) solve; +- if the nearest such key is within `DARK_FOLLOW_SHADOW_KM` (2.0 km) of the + solve — same dead-reckoning and same distance as `key_dist_km` — the solve is + refused outright: `how == "shadowed"`, history outcome `shadowed_by_follow` + carrying `follow_key` and the distance, counter `dark_bottomup_shadowed`, no + publish, no KF update, no entry written; +- farther away it falls through to the non-followed candidates and, failing + those, mints a key of its own as before. + +The anchor and ADS-B branches are untouched, which is what keeps the follow +lane's own solves landing on their key (they are anchored, and the anchor branch +returns before the scan). The `n>=3` case rule 2 also covers implicitly: an n=2 +bottom-up solve can no longer join a recently-followed key whose last published +solve had `n>=3`. `shadow` and `off` modes never consult ownership at all, so +the keying rule is byte-identical to before there. + | Constant | Value | File | |---|---|---| | `DARK_FOLLOW_MODE` | `shadow` (env) | `core/state.py` | +| `DARK_FOLLOW_OWN_S` | 6.0 s (env) | `services/dark_follow.py` | +| `DARK_FOLLOW_SHADOW_KM` | 2.0 km (env) | `services/dark_follow.py` | | `DARK_FOLLOW_MAX_AGE_S` | 20 s (env) | `services/dark_follow.py` | | `DARK_FOLLOW_MIN_SOLVES` / `DARK_FOLLOW_MIN_NODES` | 3 / 3 | `services/dark_follow.py` | | `DARK_FOLLOW_MAX_VEL_SIGMA_MS` | 60 m/s (env) | `services/dark_follow.py` | @@ -403,10 +450,13 @@ proximity scan's own age-scaled gate. Every other anchored input is unchanged. Observability: `/api/test/solver-stats` `counters` carries the funnel `dark_follow_targets` (a live gauge) → `dark_follow_claims` → -`dark_follow_inputs` → `dark_follow_published`, plus `dark_follow_dropped`. -Records are classified `lane: "dark_follow"` in `lane_split` and kept out of -the bottom-up dark funnel, and each carries `guess_source: "prediction"` and -`follow_key`. +`dark_follow_inputs` → `dark_follow_published`, plus `dark_follow_dropped` and +`dark_bottomup_shadowed` (the ownership refusals above). Records are classified +`lane: "dark_follow"` in `lane_split` and kept out of the bottom-up dark funnel, +and each carries `guess_source: "prediction"` and `follow_key`. A shadowed +record stays in the **bottom-up** funnel — it is a bottom-up solve — and shows +up as `rejects.by_reason["shadowed_by_follow"]`, with `follow_key` naming the +track that refused it. --- From 1cdb64754d8ed74140b8cee009a559a877bec308 Mon Sep 17 00:00:00 2001 From: "Claude_Fable_5.1" Date: Sun, 6 Sep 2026 02:26:50 +0000 Subject: [PATCH 24/25] geolocator: point the submodule at main now that free-altitude (#24) is merged there Co-Authored-By: Claude Fable 5.1 --- libs/retina-geolocator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/retina-geolocator b/libs/retina-geolocator index b87b4707..6979943a 160000 --- a/libs/retina-geolocator +++ b/libs/retina-geolocator @@ -1 +1 @@ -Subproject commit b87b4707cecb331fea2a8895f3e63f256183f4a9 +Subproject commit 6979943aa7ae3d05e61a9cb355490eafcc8b45b3 From 0bb06c02b38050dd9ab8ae7aaa05a89c9c51640c Mon Sep 17 00:00:00 2001 From: "Claude_Fable_5.1" Date: Sun, 6 Sep 2026 02:36:00 +0000 Subject: [PATCH 25/25] mlat-history: the lane list knows dark_follow Records the follow lane writes carry lane="dark_follow"; the history endpoint's per-lane cap and lane_counts index by that name, so without it a single follow record turned /api/test/mlat-history into a KeyError (seen live on the test droplet the first time the lane ran in binding mode). Co-Authored-By: Claude Fable 5.1 --- backend/routes/test.py | 2 +- backend/tests/test_mlat_history.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/routes/test.py b/backend/routes/test.py index 5fc4bb40..b01ccd4f 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -767,7 +767,7 @@ def _record_lane(rec: dict) -> str: return "adsb" if hexn and is_transponder_hex(hexn) else "dark" -_LANES = ("dark", "known", "adsb") +_LANES = ("dark", "known", "adsb", "dark_follow") def _cap_per_lane(records: list[dict], limit: int) -> list[dict]: diff --git a/backend/tests/test_mlat_history.py b/backend/tests/test_mlat_history.py index a406d1b4..30e8495b 100644 --- a/backend/tests/test_mlat_history.py +++ b/backend/tests/test_mlat_history.py @@ -960,7 +960,7 @@ def test_default_lane_is_all_and_counts_every_lane(self): self._adsb() data = self._client().get("/api/test/mlat-history?all=1").json() assert data["lane"] == "all" - assert data["lane_counts"] == {"dark": 1, "known": 1, "adsb": 1} + assert data["lane_counts"] == {"dark": 1, "known": 1, "adsb": 1, "dark_follow": 0} assert data["n_records"] == 3 def test_lane_dark_returns_only_dark_records(self): @@ -969,7 +969,7 @@ def test_lane_dark_returns_only_dark_records(self): self._adsb(1) data = self._client().get("/api/test/mlat-history?all=1&lane=dark").json() assert data["n_records"] == 2 - assert data["lane_counts"] == {"dark": 2, "known": 0, "adsb": 0} + assert data["lane_counts"] == {"dark": 2, "known": 0, "adsb": 0, "dark_follow": 0} assert all(r["solve_key"] == "mn-dark-1" for r in data["records"]) def test_lane_known_returns_only_known_records(self): @@ -993,7 +993,7 @@ def test_known_volume_cannot_evict_dark_records_from_the_response(self): assert sorted(lanes) == ["dark", "dark", "known", "known"] # n_records / lane_counts stay pre-cap so truncation is legible. assert data["n_records"] == 22 - assert data["lane_counts"] == {"dark": 2, "known": 20, "adsb": 0} + assert data["lane_counts"] == {"dark": 2, "known": 20, "adsb": 0, "dark_follow": 0} def test_limit_is_clamped_to_the_maximum(self): self._dark(3) @@ -1062,6 +1062,7 @@ def test_lane_filter_applies_to_skips(self): "dark": 1, "known": 0, "adsb": 1, + "dark_follow": 0, } def test_unknown_kind_is_rejected(self):