From 664462819fe1d381edc4b101af2c334ad8ae8e03 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 06:25:08 +0000 Subject: [PATCH 01/15] Gate overlap zones on node world, and report the skipped pairs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps libs/retina-analytics to the branch commit that stops register_node and rebuild_zones_for building an overlap zone between two nodes in different worlds (offworldlabs/retina-analytics#25). This fleet is the reason: 50 synthetic nodes and 8 receivers share one footprint over Greenville, so the associator held a zone for all 400 sim/real pairs — 39 of them a live adjacency edge — and a pairing drawn from one of those grids can only ever match a simulated tracklet against a real echo. Real node ids showed up in 48 of 156 synthetic dark solver records over a 7-minute window. node_world is already the single authority for the question and is already injected as node_world_provider, so nothing new is wired here. /api/radar/association/status gains assoc_world_skipped_pairs next to overlap_zones: without it a fleet whose cross-world pairs are being refused reads exactly like a fleet whose pairs never overlapped. Co-Authored-By: Claude Fable 5.1 --- backend/routes/analytics.py | 8 +++++++ backend/tests/test_adsb_seed_backend.py | 29 +++++++++++++++++++++++++ backend/tests/test_analytics_routes.py | 12 ++++++++++ libs/retina-analytics | 2 +- 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/backend/routes/analytics.py b/backend/routes/analytics.py index 07d83a9c..23225ccf 100644 --- a/backend/routes/analytics.py +++ b/backend/routes/analytics.py @@ -125,6 +125,14 @@ async def association_status(): return { "registered_nodes": len(_a.node_geometries), "overlap_zones": len(_a.overlap_zones), + # Node pairs that got no grid because the two nodes are in different + # worlds (node_world above). Read next to overlap_zones: on a fleet + # of 50 synthetic nodes over the same city as 8 receivers it is the + # 400 sim/real pairs whose grids could only ever have paired a + # simulated echo with a real one. Counted per pair considered, so it + # keeps rising as nodes re-register — zero means the fleet is single- + # world (or untagged), not that the gate is off. + "assoc_world_skipped_pairs": getattr(_a, "assoc_world_skipped_pairs", 0), # Confirmed single-node tracks each node last submitted; these are what # pairings are drawn from. "pending_tracks": {nid: len(tracks) for nid, tracks in list(_a._pending_tracks.items())}, diff --git a/backend/tests/test_adsb_seed_backend.py b/backend/tests/test_adsb_seed_backend.py index cba3bb36..77b3fc86 100644 --- a/backend/tests/test_adsb_seed_backend.py +++ b/backend/tests/test_adsb_seed_backend.py @@ -605,3 +605,32 @@ def test_associator_gets_the_state_world_resolver(self): must consult the same resolver claiming and the auto-tag filter use, or one consumer accepts what another rejects.""" assert state.node_associator.node_world_provider is state.node_world + + def test_a_sim_and_a_real_node_over_one_footprint_get_no_overlap_zone(self): + """The same resolver, one level down: bottom-up pairing must not build + a grid across worlds either. Registering a synthetic node and a + hardware node on overlapping coverage used to leave a zone whose only + possible pairing was a simulated echo against a real one — which is how + real node ids reached the synthetic fleet's dark solves.""" + _a = state.node_associator + try: + _a.register_node("synth-GVL-9001", dict(_NODE_CFG)) + _a.register_node("hw-9001", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36)) + + assert _a.overlap_zones == {} + assert _a._neighbors.get("synth-GVL-9001", set()) == set() + assert _a.assoc_world_skipped_pairs == 1 + finally: + state._reset_for_tests() + + def test_two_synthetic_nodes_over_one_footprint_still_pair(self): + """The gate is the world difference, not the registration.""" + _a = state.node_associator + try: + _a.register_node("synth-GVL-9001", dict(_NODE_CFG)) + _a.register_node("synth-GVL-9002", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36)) + + assert _a.overlap_zones + assert _a.assoc_world_skipped_pairs == 0 + finally: + state._reset_for_tests() diff --git a/backend/tests/test_analytics_routes.py b/backend/tests/test_analytics_routes.py index 22a821e0..73c691d4 100644 --- a/backend/tests/test_analytics_routes.py +++ b/backend/tests/test_analytics_routes.py @@ -129,6 +129,18 @@ def test_status_returns_expected_fields(self, client): assert "overlap_zones" in body assert "overlaps" in body + def test_status_reports_world_skipped_pairs(self, client): + """The world gate on overlap zones is otherwise invisible: a fleet + whose sim/real pairs are being refused looks exactly like a fleet whose + pairs never overlapped, and only this counter separates them.""" + _a = state.node_associator + _a.assoc_world_skipped_pairs += 7 + try: + body = client.get("/api/radar/association/status").json() + assert body["assoc_world_skipped_pairs"] == 7 + finally: + state._reset_for_tests() + def test_status_includes_claiming_block(self, client): """Top-down claiming (ASSOC_CLAIM_MODE) since boot — off by default in tests, so this pins the shape rather than any particular mode.""" diff --git a/libs/retina-analytics b/libs/retina-analytics index 14504176..4439c2c7 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 145041767422723d89d98ec003f843347ddbb880 +Subproject commit 4439c2c7d2d4fb49ca6646000f3bedc25f9701c4 From b488400597455bfb8cbe8db2459e090ba447d2f7 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 06:37:45 +0000 Subject: [PATCH 02/15] Measure candidate contamination in the association bench The ghost rate cannot see the failure the cluster-merge rework targets. A solver input that merges two aircraft usually still solves within MATCH_KM of one of them, so it counts as *matched* while carrying 4-5 km of position error and a node that never saw the aircraft it was published as. Nothing in the bench reported that, so the fix had no acceptance criterion. --mode track now scores every solver input association emits, ahead of the solve and ahead of every gate below it, against a truth side-channel: which aircraft actually produced each detection. That channel cannot come from the frame's own ADS-B list -- the simulator appends None there for every aircraft with has_adsb False, and dark aircraft are exactly the population under study -- so each detection is matched back to the aircraft whose noiseless (delay, doppler) it is nearest, at a gate ~5 sigma wider than the simulator's own measurement noise. It is built before _strip_adsb and never reaches association, so the blind discipline is intact. Reports contaminated_inputs_pct and foreign_nodes_per_input, per seed and pooled, alongside the ghost rate and matched error already there. Also adds the cluster-merge knobs (--merge-dist-km, --pair-vel-exclusive, --merge-vel-consistent and the two velocity thresholds) so each sub-step of the rework can be swept on its own; each defaults to None, meaning "leave the library's default alone", so a plain run measures what the library currently ships rather than freezing today's values into the bench. Pins retina-analytics at fix/cluster-contamination. Co-Authored-By: Claude Fable 5.1 --- backend/scripts/association_bench.py | 232 +++++++++++++++++++++++++++ libs/retina-analytics | 2 +- 2 files changed, 233 insertions(+), 1 deletion(-) diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py index e08999bb..14dc6cdc 100644 --- a/backend/scripts/association_bench.py +++ b/backend/scripts/association_bench.py @@ -69,6 +69,7 @@ # superseded detection path that --mode detection measures as the baseline, so # constructing it unconditionally leaves --mode track unaffected. import retina_analytics.association as _assoc_module # noqa: E402 +from retina_analytics.association import predict_observation # noqa: E402 from retina_analytics.detection_association import DetectionAssociator # noqa: E402 from retina_analytics.manager import NodeAnalyticsManager # noqa: E402 from retina_geolocator.consensus import solve_consensus # noqa: E402 @@ -248,6 +249,121 @@ def _frame_to_detections(frame: dict) -> list[dict]: return dets +# ── Contamination scoring (truth side-channel) ─────────────────────────── +# How far a detection may sit from an aircraft's noiseless (delay, doppler) +# and still be attributed to it. The simulator's own measurement noise is +# gauss(0, 0.1-0.2 us) in delay and gauss(0, 2-4 Hz) in Doppler +# (world.generate_detections_for_node), so these are ~5 sigma: wide enough +# that a real echo is never mistaken for clutter, tight enough that clutter +# — uniform over tens of us — almost never lands on an aircraft. +_TRUTH_DELAY_GATE_US = 1.0 +_TRUTH_DOPPLER_GATE_HZ = 25.0 +# Sentinel for "this measurement matched two different aircraft equally +# well". Neither foreign nor own — excluded from the numerator so an +# ambiguity in the scorer is never reported as a contamination. +_TRUTH_AMBIGUOUS = "?ambiguous" + + +def _index_detection_truth(det_truth: dict, geo, node_id: str, frame: dict, aircraft: list) -> None: + """Record which aircraft produced each detection in one node's frame. + + Truth side-channel, for the CONTAMINATION metric only: it is built from + the frame BEFORE _strip_adsb and never reaches association, so the blind + discipline is intact. It cannot be read off the frame's own ``adsb`` + list either — the simulator appends None there for every aircraft with + has_adsb False, and dark aircraft are exactly the population this metric + exists to score. So each detection is instead matched back to the + aircraft whose noiseless observation it is nearest. + + Keyed on the (delay, doppler) floats themselves because that is the only + handle the metric gets downstream: a solver input's measurement carries + its track's latest delay/doppler verbatim (history[-1] -> the detection + dict -> here), and the simulator rounds both to 2 dp, so the equality is + exact rather than approximate. + """ + delays = frame.get("delay") or [] + if not delays: + return + dopplers = frame.get("doppler") or [] + preds = [] + for ac in aircraft: + d_us, f_hz = predict_observation( + geo, + ac.lat, + ac.lon, + ac.alt_km, + ac.vel_east * 1000.0, + ac.vel_north * 1000.0, + ac.vel_up * 1000.0, + ) + preds.append((d_us, f_hz, ac.object_id)) + for d, f in zip(delays, dopplers): + best = best2 = None + for d_us, f_hz, oid in preds: + dd, df = abs(d - d_us), abs(f - f_hz) + if dd > _TRUTH_DELAY_GATE_US or df > _TRUTH_DOPPLER_GATE_HZ: + continue + # Normalised so the two axes are comparable at their own gates. + cost = (dd / _TRUTH_DELAY_GATE_US) ** 2 + (df / _TRUTH_DOPPLER_GATE_HZ) ** 2 + if best is None or cost < best[0]: + best, best2 = (cost, oid), best + elif best2 is None or cost < best2[0]: + best2 = (cost, oid) + if best is None: + continue # clutter: left absent, which the scorer reads as foreign + oid = best[1] + if best2 is not None and best2[0] < 4.0 * best[0]: + oid = _TRUTH_AMBIGUOUS + key = (node_id, float(d), float(f)) + prev = det_truth.get(key) + # The same (node, delay, doppler) recurring for a different aircraft + # later in the run would silently relabel an earlier measurement, so + # a collision demotes the key rather than overwriting it. + det_truth[key] = oid if (prev is None or prev == oid) else _TRUTH_AMBIGUOUS + + +def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) -> None: + """Count the nodes in one solver input that are not looking at its aircraft. + + The input's own aircraft is the plurality of its measurements' true + aircraft — the honest reading of "what is this candidate mostly about", + and the one that does not assume the (possibly contaminated) initial + guess is anywhere near a target. Ties are broken by the nearest ground + truth to the initial guess, which is the criterion the ghost/matched + split already uses. + + A node is foreign when its measurement belongs to a different aircraft, + or to no aircraft at all (clutter that survived the tracker's M-of-N and + the delay grid). Ambiguous attributions are counted in neither. + """ + oids = [ + det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"])), None) + for m in s_in.get("measurements") or [] + ] + if not oids: + return + counts = Counter(o for o in oids if o is not None and o != _TRUTH_AMBIGUOUS) + if not counts: + return + top_n = max(counts.values()) + contenders = sorted(o for o, c in counts.items() if c == top_n) + if len(contenders) > 1: + guess = s_in.get("initial_guess") or {} + contenders.sort( + key=lambda o: min( + (_haversine_km(guess.get("lat", 0.0), guess.get("lon", 0.0), a, b) for a, b, oid, _ in truth if oid == o), + default=float("inf"), + ) + ) + own = contenders[0] + foreign = sum(1 for o in oids if o != own and o != _TRUTH_AMBIGUOUS) + res.inputs_scored += 1 + res.input_nodes += len(oids) + res.foreign_nodes += foreign + if foreign: + res.inputs_contaminated += 1 + + def _strip_adsb(frame: dict) -> dict: """Return the frame as a real receiver would see it. @@ -482,6 +598,9 @@ class Result: gate_accepted: int = 0 gate_unfitted: int = 0 gate_superseded: int = 0 + # Position clusters that held two different tracks of one node and were + # split into one solver input each, straight off the associator. + cluster_splits: int = 0 # Deferred mode only: what the *solver-side* n=2 gate did. In production the # associator emits unscored pairings and this gate is the one that runs, so # without these the shipped configuration's selection is invisible. @@ -531,6 +650,20 @@ class Result: keys_real: int = 0 keys_ghost: int = 0 + # ── Candidate contamination (--mode track) ──────────────────────────── + # Scored on every solver input association emits, BEFORE the solve and + # before every downstream gate: the question is what association handed + # the solver, not what survived it. A contaminated input is one whose + # measurements do not all belong to the same aircraft — the failure the + # cluster-merge rework targets, and the one the ghost rate cannot see + # (a two-aircraft merge usually still solves within MATCH_KM of one of + # them, so it counts as matched while carrying 4-5 km of position + # error). See _score_contamination. + inputs_scored: int = 0 + inputs_contaminated: int = 0 + foreign_nodes: int = 0 + input_nodes: int = 0 + # Stone-Soup GOSPA/SIAP scalars for this one run (--ss-metrics), or None # when it was off, stonesoup wasn't available, or the recorder had # nothing to score (see stonesoup_metrics.MetricRecorder.compute). Not @@ -634,6 +767,11 @@ def keys_per_object(self): "distinct_keys", "keys_real", "keys_ghost", + "inputs_scored", + "inputs_contaminated", + "foreign_nodes", + "input_nodes", + "cluster_splits", ) _EXTEND_FIELDS = ( "errors_km", @@ -685,6 +823,14 @@ def merge(self, other: Result, tag: str) -> None: def ghost_pct(self): return 100.0 * self.ghosts / self.total if self.total else 0.0 + @property + def contaminated_inputs_pct(self): + return 100.0 * self.inputs_contaminated / self.inputs_scored if self.inputs_scored else 0.0 + + @property + def foreign_nodes_per_input(self): + return self.foreign_nodes / self.inputs_scored if self.inputs_scored else 0.0 + def build_scene( seed: int, @@ -789,6 +935,7 @@ def run( ss_metric_dt=5.0, ss_hold_s=12.0, smoother_legs=None, + cluster_opts=None, ) -> Result: import random @@ -861,12 +1008,17 @@ def fov_provider(node_id): # emits unscored pairings and the solver worker fits and arbitrates. The # two are different code paths, so they need separate baselines. deferred = mode == "track" and cv_fit_mode == "deferred" + # The cluster-merge knobs are passed only when the caller overrode them, + # so a plain run measures whatever the library currently ships rather than + # freezing today's defaults into the bench. + _cluster_kwargs = {k: v for k, v in (cluster_opts or {}).items() if v is not None} assoc = DetectionAssociator( grid_step_km=3.0, cv_fit=(fit_constant_velocity if (mode == "track" and not deferred) else None), cv_chi2_max=chi2_max, cv_min_span_s=min_span_s, cv_exclusive=exclusive, + **_cluster_kwargs, ) n2_gate = DeferredN2Gate(chi2_max, claim_ttl_s=claim_ttl_s, claim_policy=claim_policy) if deferred else None # One tracker per node, driven by every frame — mirrors @@ -915,6 +1067,10 @@ def fov_provider(node_id): _BENCH_MN_MAX_AGE_MS = 60_000 res = Result() + # (node_id, delay_us, doppler_hz) -> the aircraft that produced that + # detection. Truth side-channel for the contamination metric only, built + # from the un-stripped frame below — see _index_detection_truth. + det_truth: dict = {} _all_keys_seen: set = set() _keys_real: set = set() _keys_ghost: set = set() @@ -960,6 +1116,10 @@ def _geo_key(nid): for nid in due_nodes: next_send[nid] += frame_interval frame = world.generate_detections_for_node(nid, ts_ms) + if mode == "track": + # Before _strip_adsb, and never fed to association: the + # contamination metric's truth channel. + _index_detection_truth(det_truth, assoc.node_geometries[nid], nid, frame, world.aircraft) if fov_analytics is not None: # The truth channel, not the (possibly blind) association # stream below -- a real node's ADS-B calibration reaches @@ -1001,6 +1161,11 @@ def _geo_key(nid): res.cluster_sizes[(_k, len(s_in.get("track_ids") or []))] += 1 if s_in.get("n_nodes", 0) < 2: continue + if mode == "track": + # Scored here, ahead of the solve and every gate below: + # this measures what association emitted, which is the + # thing the cluster-merge rework changes. + _score_contamination(res, s_in, det_truth, truth) try: _t0 = time.perf_counter() out = solve_fn(s_in, node_cfgs) @@ -1153,6 +1318,7 @@ def _geo_key(nid): res.gate_accepted = assoc.track_pairs_accepted res.gate_unfitted = assoc.track_pairs_unfitted res.gate_superseded = assoc.track_pairs_superseded + res.cluster_splits = getattr(assoc, "cluster_splits", 0) res.claims_matched = assoc.claims_matched res.claim_conflicts = assoc.claim_conflicts res.anchored_inputs = assoc.anchored_inputs_emitted @@ -1329,6 +1495,15 @@ def report(label: str, r: Result, truth_max_kt: float | None = None): f" solves faster than any real aircraft ({truth_max_kt:.0f} kt): " f"{over} ({100 * over / len(r.speeds_kt):.0f}%)" ) + if r.inputs_scored: + print( + f" CONTAMINATION: {r.inputs_contaminated}/{r.inputs_scored} solver inputs carry a foreign node" + f" -> {r.contaminated_inputs_pct:5.1f}% " + f"foreign nodes/input {r.foreign_nodes_per_input:.2f}" + f" ({r.foreign_nodes}/{r.input_nodes} nodes)" + ) + if r.cluster_splits: + print(f" cluster splits (same-node track conflict): {r.cluster_splits}") if r.gate_gated: print( f" CV gate: {r.gate_gated} pairings past the delay grid " @@ -1498,6 +1673,33 @@ def main(): help="track mode: disable one-to-one hypothesis selection " "(each pairing then answers only to the chi2 threshold)", ) + # Cluster-merge knobs (track mode). Each defaults to None, meaning "leave + # the library's own default alone", so the bench does not silently pin a + # value the library later changes — and so a sweep leg reads as exactly + # the deviation it is testing. + p.add_argument( + "--merge-dist-km", + type=float, + default=None, + help="track mode: how close two pairings must be to merge into one " + "solver input (association._MERGE_DIST_KM)", + ) + p.add_argument( + "--pair-vel-exclusive", + choices=("on", "off"), + default=None, + help="track mode, deferred only: drop a pairing whose implied velocity " + "contradicts a better-scoring pairing that claims the same track", + ) + p.add_argument( + "--merge-vel-consistent", + choices=("on", "off"), + default=None, + help="track mode: require implied-velocity agreement, not just " + "proximity, before two pairings are merged into one cluster", + ) + p.add_argument("--pair-vel-dv-ms", type=float, default=None, help="velocity-conflict speed threshold (m/s)") + p.add_argument("--pair-vel-dtheta-deg", type=float, default=None, help="velocity-conflict heading threshold (deg)") p.add_argument("--min-aircraft", type=int, default=10, help="matches FLEET_AIRCRAFT lower bound") p.add_argument("--max-aircraft", type=int, default=20) p.add_argument("--metro-traffic-frac", type=float, default=0.85, help="matches FLEET_METRO_TRAFFIC_FRAC") @@ -1568,6 +1770,14 @@ def main(): ) args = p.parse_args() + cluster_opts = { + "merge_dist_km": args.merge_dist_km, + "pair_vel_exclusive": None if args.pair_vel_exclusive is None else args.pair_vel_exclusive == "on", + "merge_vel_consistent": None if args.merge_vel_consistent is None else args.merge_vel_consistent == "on", + "pair_vel_dv_ms": args.pair_vel_dv_ms, + "pair_vel_dtheta_deg": args.pair_vel_dtheta_deg, + } + # --ss-metrics auto/on/off resolution. "on" without stonesoup installed # is a hard error (the user explicitly asked for numbers this image # cannot produce); "auto" degrades quietly except for one notice line so @@ -1622,6 +1832,7 @@ def main(): + f", fov={args.fov}" + f", ss-metrics={'on' if ss_metrics_enabled else 'off'}" + (f", smoother-legs={','.join(lbl for lbl, _, _ in smoother_legs)}" if smoother_legs else "") + + "".join(f", {k.replace('_', '-')}={v}" for k, v in sorted(cluster_opts.items()) if v is not None) ) # chi2 only means anything in track mode; keep one pass otherwise. @@ -1632,6 +1843,7 @@ def main(): solve_fn = _ESTIMATORS[estimator_name] rates, solve_rates, reals, fakes, speed_errs = [], [], [], [], [] n2_rates = [] + contam_rates, foreign_rates, med_errs = [], [], [] agg = Result() last = None for k in range(args.repeat): @@ -1666,6 +1878,7 @@ def main(): ss_metric_dt=args.ss_metric_dt, ss_hold_s=args.ss_hold_s, smoother_legs=smoother_legs, + cluster_opts=cluster_opts, ) agg.merge(last, tag=f"s{args.seed + k}") # Track-level is the comparable metric — solve-level and @@ -1678,6 +1891,9 @@ def main(): fakes.append(len(last.ghost_tracks)) if last.speed_err_ms: speed_errs.append(statistics.median(last.speed_err_ms)) + contam_rates.append(last.contaminated_inputs_pct) + foreign_rates.append(last.foreign_nodes_per_input) + med_errs.append(statistics.median(last.errors_km) if last.errors_km else float("nan")) label = f"assoc_interval={interval:g}s" if chi2_max is not None: label += f" chi2/dof<={chi2_max:g}" @@ -1700,6 +1916,22 @@ def main(): f"({', '.join(f'{x:.0f}%' for x in n2_rates)})" ) print(f" by solve: {', '.join(f'{x:.1f}%' for x in solve_rates)}") + if any(contam_rates): + print( + f" contaminated inputs per seed: " + f"{', '.join(f'{x:.0f}%' for x in contam_rates)}" + f" mean {statistics.mean(contam_rates):.1f}%" + ) + print( + f" foreign nodes/input per seed: " + f"{', '.join(f'{x:.2f}' for x in foreign_rates)}" + f" mean {statistics.mean(foreign_rates):.2f}" + ) + print( + f" median matched error per seed: " + f"{', '.join(f'{x:.2f}' for x in med_errs)} km" + f" real tracks {', '.join(str(x) for x in reals)}" + ) if speed_errs: print( f" median speed error per seed: " diff --git a/libs/retina-analytics b/libs/retina-analytics index 14504176..4ff48fca 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 145041767422723d89d98ec003f843347ddbb880 +Subproject commit 4ff48fcaa3abdaad1d42eb576f8573342addad4e From a1395caa259187123d41657ab9c0b696d2d171a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 06:37:04 +0000 Subject: [PATCH 03/15] solver: SOLVER_ALT_MODE=free, and ship only the configs a candidate uses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent changes to the same enqueue-and-solve path. 1. SOLVER_ALT_MODE (core/state.py, default "sweep" — nothing changes until it is set). The n>=3 altitude sweep solves once per fixed layer and keeps the lowest rms_delay. Its layers are 2 km apart, so the altitude it pins is systematically up to 1 km wrong, and on noise-free replay of this fleet's geometry that quantisation ALONE left rms_delay at a 1.76 us median against the 3.0 us reject gate — most of the gate's budget spent on an altitude the measurements themselves determine, and the residual left over then blamed on nodes, so _trim_and_resolve drops measurements that were never the problem. In "free" mode _solve_best_altitude makes ONE pool call to the geolocator's solve_multinode_multistart with three start layers (the one nearest the association guess and its two neighbours), altitude solved as a sixth unknown — against the sweep's six round trips, each of which pickles the node configs for a child. n=2 is untouched: altitude is unobservable there, so _solve_best_altitude_n2 keeps its single pinned solve and the geolocator pins regardless of the flag. Trimming re-solves through _solve_best_altitude, so a trim round inherits the mode its first solve used — mixing a swept altitude with a free one would make the rms it compares between rounds a different quantity. Both modes stamp altitude_mode on every history record, published or rejected, and free adds the per-start residuals and z_saturated. That is the whole comparison channel: one deploy of each, read off /api/test/mlat-history. There is deliberately no shadow mode — the two produce the same shape of result, so running both would double the solver's cost to learn what a deploy of each already says. 2. The frame path shipped ALL 58 node configs with every solver input. The pool is a spawn pool, so that whole set is pickled per solve while a candidate carries 2-8 measurements. configs_for_solver_input restricts it to the measurement node ids; nothing downstream needs the rest — the solver builds NodeSetups from the measurements, trimming and consensus only ever narrow that set, the beam gate iterates contributing_node_ids, and cv_epochs is built from the same matched nodes in all three input shapes association emits. The known lane fetches its own configs and is unaffected. Co-Authored-By: Claude Fable 5.1 --- backend/.env.example | 10 + backend/core/state.py | 20 ++ backend/services/frame_processor.py | 23 ++- backend/services/tasks/solver.py | 118 +++++++++++- backend/tests/test_solver_alt_mode.py | 265 ++++++++++++++++++++++++++ docs/solverflow.md | 43 ++++- libs/retina-geolocator | 2 +- 7 files changed, 470 insertions(+), 11 deletions(-) create mode 100644 backend/tests/test_solver_alt_mode.py diff --git a/backend/.env.example b/backend/.env.example index 20a107b4..488ecbdc 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -129,9 +129,19 @@ MENDER_PAT= # from its association guess before it is rejected; the default is 6 km = 2 x # the 3 km association grid step, since a dark guess is a quantised grid point # rather than an ADS-B fix. ADS-B-anchored solves keep the fixed 2 km cap. +# +# SOLVER_ALT_MODE is how an n>=3 solve gets its altitude. sweep (default) +# solves once per fixed altitude layer and keeps the lowest rms_delay — six +# process-pool round trips per candidate, and an altitude quantised to a +# ladder whose 2 km spacing puts up to 1 km of error straight into the +# residual the reject gate reads. free makes one pool call to the geolocator's +# multi-start helper, which solves altitude as a sixth unknown. Both modes +# stamp altitude_mode on the solve-history record, so /api/test/mlat-history +# can compare them across a deploy of each. See docs/solverflow.md. # SOLVER_WORKERS=2 # SOLVER_RESOLVE_INTERVAL_S=12 # SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0 +# SOLVER_ALT_MODE=sweep # Detection mirror. Production only. Every accepted v1 detection frame is # forwarded to another environment's /api/radar/detections/bulk, batched once a diff --git a/backend/core/state.py b/backend/core/state.py index 634aa212..f1239459 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -74,6 +74,26 @@ if KNOWN_LANE_MODE not in ("off", "shadow", "binding"): KNOWN_LANE_MODE = "shadow" +# How the n>=3 solve gets its altitude (see services/tasks/solver.py's +# _solve_best_altitude). sweep/free, read here rather than in that module so +# it sits with its sibling mode flags and a test can monkeypatch it without +# reimporting the solver. +# sweep (default) — solve once per fixed altitude layer, keep the lowest +# rms_delay. Six pool round trips per candidate, and an altitude +# quantised to the ladder: layers are 2 km apart, so the pin is +# systematically up to 1 km wrong and that error lands in the +# residual the reject gate reads. +# free — one pool call to retina_geolocator's multi-start helper, which +# solves altitude as a sixth unknown from three start layers. +# Not off/shadow/active: there is no shadow here, because the two modes +# produce the same shape of result and the history record carries +# altitude_mode either way — running both would double the solver's cost to +# learn what one deploy of each already says. An unrecognised value falls +# back to "sweep", the same degrade-to-inert rule the sibling flags use. +SOLVER_ALT_MODE = os.getenv("SOLVER_ALT_MODE", "sweep").lower() +if SOLVER_ALT_MODE not in ("sweep", "free"): + SOLVER_ALT_MODE = "sweep" + node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE) diff --git a/backend/services/frame_processor.py b/backend/services/frame_processor.py index 0fb2bbd5..fe7b086d 100644 --- a/backend/services/frame_processor.py +++ b/backend/services/frame_processor.py @@ -156,6 +156,27 @@ def get_node_configs() -> dict[str, dict]: return configs +def configs_for_solver_input(node_cfgs: dict[str, dict], s_in: dict) -> dict[str, dict]: + """The subset of ``node_cfgs`` a solver input can actually reach. + + The solver runs in a *spawn* process pool, so everything queued with an + input is pickled and shipped to a child on every call — and the fleet is + 58 nodes while a candidate carries 2-8 measurements. Sending the whole + set meant ~50 configs per solve that no code path could look at. + + Nothing downstream needs the rest. The solver builds NodeSetups from the + measurements only; trimming and consensus both narrow that set further + (_filter_s_in_to_nodes) and never widen it; the beam gate iterates the + result's contributing_node_ids, which are measurement node ids by + construction; and cv_epochs is built from the same matched nodes as the + measurements in all three input shapes association emits. The known lane + is unaffected — it fetches its own configs (known_lane.run_known_lane_pass) + rather than reusing what was queued here. + """ + wanted = {m.get("node_id") for m in (s_in.get("measurements") or ())} + return {nid: cfg for nid, cfg in node_cfgs.items() if nid in wanted} + + # ── Per-node pipeline factory ───────────────────────────────────────────────── @@ -427,7 +448,7 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP if s_in["n_nodes"] < 2: continue try: - state.solver_queue.put_nowait((s_in, node_cfgs, time.time())) + state.solver_queue.put_nowait((s_in, configs_for_solver_input(node_cfgs, s_in), time.time())) except Exception: state.bump_counter("solver_queue_drops") if state.solver_queue_drops % 100 == 1: diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 7c6b947c..41bb53ec 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -100,6 +100,24 @@ def _pool_call(fn, *args): return fn(*args) +def _pool_solve_multistart(s_in, node_cfgs, alt_starts_km): + """solve_multinode_multistart via the process pool (inline when none). + + Defined here rather than beside _pool_solve_multinode at the foot of this + module for the reason _pool_select_consensus is: it is a default argument + value, resolved when the ``def`` executes, so it has to be bound before + _solve_best_altitude's signature is reached. + + A module-level function taking only picklable arguments, because the pool + is a *spawn* pool — a child imports retina_geolocator and nothing of the + backend, so what crosses is this function's qualified name plus the input + dicts. + """ + from retina_geolocator.multinode_solver import solve_multinode_multistart + + return _pool_call(solve_multinode_multistart, s_in, node_cfgs, alt_starts_km, True) + + # Altitude layers (km) tried when n_nodes ≥ 3. For an overdetermined system # (3+ delay equations, 2 unknowns after altitude pinning) only the correct # altitude layer yields rms_delay ≈ 0; wrong layers give rms > 0, so picking @@ -320,17 +338,72 @@ def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[floa return best_result -def _solve_best_altitude(s_in: dict, node_cfgs: dict, solve_fn) -> dict | None: - """Altitude sweep for n≥3: pick by minimum rms_delay. +# How many start altitudes the free mode hands the multi-start helper: the +# layer nearest the guess and its two neighbours. Freeing z removes the +# ladder's quantisation but not the LM's locality, so the starts are still +# what stops a solve settling on the wrong side of a bistatic ellipse — but +# three of them, in one pool call, rather than the sweep's six. +_FREE_ALT_N_STARTS = 3 +# Fewest measurements the free mode is used at. Below this altitude is not +# observable and retina_geolocator pins it anyway; the sweep is left in place +# so the n=2 path keeps its documented behaviour exactly. +_FREE_ALT_MIN_NODES = 3 + + +def _free_alt_starts(ig_alt_km, layers: list[float]) -> list[float]: + """The layer nearest ``ig_alt_km`` and its neighbours — _FREE_ALT_N_STARTS + of them, clamped to the ends of the ladder so the count never shrinks + there (the top and bottom layers are where a wrong start is least + recoverable, not most). + """ + if not layers: + return [] + alt = float(ig_alt_km) if ig_alt_km is not None else 7.0 + nearest = min(range(len(layers)), key=lambda i: abs(layers[i] - alt)) + lo = max(0, min(nearest - 1, len(layers) - _FREE_ALT_N_STARTS)) + return layers[lo : lo + _FREE_ALT_N_STARTS] + - If the initial_guess already carries an ADS-B altitude (not one of the fixed - grid layers), include it in the sweep so the correct exact altitude is tried. +def _solve_best_altitude( + s_in: dict, + node_cfgs: dict, + solve_fn, + multistart_fn=_pool_solve_multistart, +) -> dict | None: + """Altitude for n≥3, by whichever rule state.SOLVER_ALT_MODE names. + + sweep (default): solve once per layer, pick by minimum rms_delay. If the + initial_guess already carries an ADS-B altitude (not one of the fixed grid + layers), include it in the sweep so the correct exact altitude is tried. + + free: one call to the multi-start helper, which solves altitude as a sixth + unknown from _free_alt_starts. The sweep cannot do better than half its + 2 km layer spacing, and on noise-free replay of this fleet's geometry that + quantisation alone left rms_delay at a 1.76 µs median against the 3.0 µs + reject gate — spending most of the gate's budget on an altitude the + measurements themselves determine, and provoking _trim_and_resolve to drop + nodes that were never the problem. Costs one pool round trip per + candidate instead of six. + + The mode is read per call rather than captured at import, so a test (and a + live config reload) sees the value it set. Read here and not inside + _process_solver_item because _trim_and_resolve re-enters through this same + function: a trim must re-solve under the mode its first solve used, or the + residuals it is comparing are not the same quantity. """ ig_alt = s_in.get("initial_guess", {}).get("alt_km") if ig_alt is not None and ig_alt not in _SOLVER_ALT_LAYERS_KM: layers = sorted(set(_SOLVER_ALT_LAYERS_KM + [round(float(ig_alt), 3)])) else: layers = _SOLVER_ALT_LAYERS_KM + n_meas = len({m.get("node_id") for m in (s_in.get("measurements") or [])}) + if state.SOLVER_ALT_MODE == "free" and n_meas >= _FREE_ALT_MIN_NODES: + # No fall back to the sweep when this returns None: a helper that got + # no solve out of three starts is reporting the same thing the sweep + # reports when every layer fails, and sweeping anyway would cost the + # six round trips this mode exists to avoid on exactly the candidates + # that are least likely to repay them. + return multistart_fn(s_in, node_cfgs, _free_alt_starts(ig_alt, layers)) return _sweep_altitudes(s_in, node_cfgs, solve_fn, layers, "rms_delay") @@ -386,6 +459,7 @@ def _trim_and_resolve( node_cfgs: dict, solve_fn, result: dict, + multistart_fn=_pool_solve_multistart, ) -> tuple[dict, dict, dict | None]: """Drop the worst-residual node(s) and re-solve, down to _TRIM_MIN_NODES. @@ -395,6 +469,11 @@ def _trim_and_resolve( so re-solving on the survivors after dropping the offending node recovers a solve the blanket gate would otherwise discard outright. + Re-solves through _solve_best_altitude, so it inherits whichever altitude + mode is in force — the loop compares this round's rms against the previous + round's, and mixing a swept altitude with a free one would make that + comparison meaningless. + Returns (final_result, final_s_in, trim_meta). trim_meta is None only when no round ever produced a successful re-solve — i.e. no trimming was actually performed — never when trimming ran but rms stayed high (that @@ -439,7 +518,7 @@ def _trim_and_resolve( s_next = _filter_s_in_to_nodes(s_in, survivors) try: - new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn) + new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn, multistart_fn) except Exception: logging.exception("Solver trim re-solve failed") break @@ -1693,7 +1772,12 @@ def fov_gate_verdict(fov, n_nodes: int, brg: float, dist_km: float, range_rule_p return range_rule_pass or fov_pass -def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus) -> dict | None: +def _process_solver_item( + item: tuple, + solve_fn, + select_fn=_pool_select_consensus, + multistart_fn=_pool_solve_multistart, +) -> dict | None: """Process a single solver queue entry. Returns the solver result (or None). Extracted from the worker loop so the success/failure/latency bookkeeping @@ -1704,6 +1788,10 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus initial_guess and _CONSENSUS_MODE != "off" — n=2 (mirror-disambiguation is the displacement/beam gates' job, not consensus's) and detection-level inputs (no initial_guess to pin an altitude with) never call it. + + multistart_fn is the free-altitude solve (_pool_solve_multistart by + default; tests substitute a stub), reached only when + state.SOLVER_ALT_MODE is "free" — see _solve_best_altitude. """ s_in, node_cfgs = item[0], item[1] enqueued_at: float | None = item[2] if len(item) > 2 else None @@ -1736,7 +1824,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus if _CONSENSUS_MODE != "off": s_in, consensus_meta = _consensus_select(s_in, node_cfgs, select_fn) n_nodes = s_in.get("n_nodes", n_nodes) - result = _solve_best_altitude(s_in, node_cfgs, solve_fn) + result = _solve_best_altitude(s_in, node_cfgs, solve_fn, multistart_fn) else: result = _solve_best_altitude_n2(s_in, node_cfgs, solve_fn) except Exception: @@ -1762,7 +1850,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus and (result.get("rms_delay") or 0) > _SOLVER_RMS_DELAY_MAX_US and result.get("per_node_delay_res_us") ): - result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result) + result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result, multistart_fn) n_nodes = result.get("n_nodes", n_nodes) # Built once and threaded through every history record below @@ -1771,6 +1859,20 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus _extra: dict | None = dict(trim_meta) if trim_meta else {} if consensus_meta is not None: _extra["consensus_meta"] = consensus_meta + # How this solve got its altitude, and — in free mode — what each + # start altitude fitted to. Stamped on every record, published or + # rejected, and in BOTH modes (the sweep's solves report + # altitude_mode "pinned"), because the only way to judge SOLVER_ALT_MODE + # live is to compare the two lanes' rms_delay and gt_error_km over the + # same history buffer. The per-start list is what says whether the + # three starts were worth keeping or one would have done. + if result.get("altitude_mode"): + _extra["altitude_mode"] = result["altitude_mode"] + if result.get("rms_by_start") is not None: + _extra["alt_starts_km"] = result.get("alt_starts_km") + _extra["alt_start_rms_us"] = [None if v is None else round(float(v), 3) for v in result["rms_by_start"]] + if result.get("z_saturated"): + _extra["z_saturated"] = True _extra = _extra or None rms_delay = result.get("rms_delay", 0) or 0 diff --git a/backend/tests/test_solver_alt_mode.py b/backend/tests/test_solver_alt_mode.py new file mode 100644 index 00000000..8a374dbc --- /dev/null +++ b/backend/tests/test_solver_alt_mode.py @@ -0,0 +1,265 @@ +"""SOLVER_ALT_MODE: how the n>=3 solve gets its altitude. + +sweep (the default) calls the LM once per fixed altitude layer and keeps the +lowest rms_delay — six process-pool round trips, and an altitude quantised to +a ladder 2 km wide, which puts up to 1 km of error into the residual the +reject gate reads. free makes ONE call to the geolocator's multi-start +helper, which solves altitude as a sixth unknown from three start layers. + +These tests are about the routing, not the physics: the geolocator's own +suite (tests/test_free_altitude.py there) measures what the free solve +actually fits. What matters here is that the default is byte-identical to +the sweep, that free spends one call and not six, that trimming re-solves +under the same mode, and that both modes leave enough on the history record +to be compared live. +""" + +import time + +import pytest + +from core import state +from services import frame_processor +from services.tasks import solver as solver_mod + +LAT, LON = 35.0, -82.0 + + +def _s_in(node_ids, alt_km=9.0, **overrides): + s_in = { + "initial_guess": {"lat": LAT, "lon": LON, "alt_km": alt_km}, + "measurements": [{"node_id": nid, "delay_us": 10.0, "doppler_hz": 1.0, "snr": 15.0} for nid in node_ids], + "n_nodes": len(node_ids), + "timestamp_ms": int(time.time() * 1000), + } + s_in.update(overrides) + return s_in + + +def _stub_result(node_ids, rms_delay=0.5, **overrides): + result = { + "success": True, + "lat": LAT, + "lon": LON, + "alt_m": 9000.0, + "timestamp_ms": int(time.time() * 1000), + "vel_east": 0.0, + "vel_north": 0.0, + "rms_delay": rms_delay, + "rms_doppler": 5.0, + "n_nodes": len(node_ids), + "n_measurements": len(node_ids), + "contributing_node_ids": list(node_ids), + } + result.update(overrides) + return result + + +class _Recorder: + """A solve_fn / multistart_fn that records every call it is given.""" + + def __init__(self, result_for): + self.calls: list[tuple] = [] + self._result_for = result_for + + def __call__(self, s_in, node_cfgs, *rest): + self.calls.append((s_in, node_cfgs, rest)) + nodes = tuple(m["node_id"] for m in s_in["measurements"]) + return self._result_for(nodes, s_in, *rest) + + +class _AltModeBase: + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + +class TestFreeAltStarts: + """The three starts handed to the multi-start helper.""" + + @pytest.mark.parametrize( + "alt_km,expected", + [ + (9.0, [7.0, 9.0, 11.0]), + (7.0, [5.0, 7.0, 9.0]), + (8.2, [7.0, 9.0, 11.0]), + # Clamped at the ends: the ladder's first and last layers still get + # three starts, not one or two. + (1.5, [1.5, 3.0, 5.0]), + (0.4, [1.5, 3.0, 5.0]), + (11.0, [7.0, 9.0, 11.0]), + (40.0, [7.0, 9.0, 11.0]), + ], + ) + def test_window_around_the_nearest_layer(self, alt_km, expected): + starts = solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) + assert starts == expected + assert len(starts) == solver_mod._FREE_ALT_N_STARTS + + def test_an_adsb_altitude_in_the_ladder_is_a_start(self): + """_solve_best_altitude splices an ADS-B altitude into the layers, and + the window is taken over that spliced list — otherwise the one exact + altitude available would never be started from.""" + layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4])) + assert solver_mod._free_alt_starts(8.4, layers) == [7.0, 8.4, 9.0] + + def test_no_layers_gives_no_starts(self): + assert solver_mod._free_alt_starts(9.0, []) == [] + + +class TestSweepIsTheDefault(_AltModeBase): + def test_sweep_calls_the_lm_once_per_layer_and_never_the_multistart(self): + nodes = ["n1", "n2", "n3"] + solve = _Recorder(lambda n, s, *r: _stub_result(n)) + multistart = _Recorder(lambda n, s, *r: pytest.fail("multistart called in sweep mode")) + + result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) + + assert result is not None and result["success"] + assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM) + assert [c[0]["initial_guess"]["alt_km"] for c in solve.calls] == solver_mod._SOLVER_ALT_LAYERS_KM + assert multistart.calls == [] + assert state.SOLVER_ALT_MODE == "sweep" + + +class TestFreeMode(_AltModeBase): + def setup_method(self): + super().setup_method() + self._saved_mode = state.SOLVER_ALT_MODE + state.SOLVER_ALT_MODE = "free" + + def teardown_method(self): + state.SOLVER_ALT_MODE = self._saved_mode + super().teardown_method() + + def test_one_multistart_call_with_three_starts(self): + nodes = ["n1", "n2", "n3"] + solve = _Recorder(lambda n, s, *r: pytest.fail("sweep ran in free mode")) + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[1.2, 0.4, 0.9])) + + result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) + + assert result is not None and result["success"] + assert solve.calls == [] + assert len(multistart.calls) == 1 + (_, _, rest) = multistart.calls[0] + assert rest == ([7.0, 9.0, 11.0],) + + def test_n2_keeps_the_sweep(self): + """Altitude is unobservable at n=2 — the free path is not entered even + with the mode on.""" + nodes = ["n1", "n2"] + solve = _Recorder(lambda n, s, *r: _stub_result(n)) + multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken at n=2")) + + result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) + + assert result is not None + assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM) + assert multistart.calls == [] + + def test_a_failed_multistart_is_a_failed_solve(self): + """No silent fall back to the sweep: three starts producing nothing is + the same verdict as every layer producing nothing.""" + solve = _Recorder(lambda n, s, *r: pytest.fail("swept after a failed multistart")) + multistart = _Recorder(lambda n, s, *r: None) + + assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart) is None + + def test_history_carries_the_mode_and_the_per_start_residuals(self): + nodes = ["n1", "n2", "n3"] + multistart = _Recorder( + lambda n, s, *r: _stub_result( + n, altitude_mode="free", rms_by_start=[1.2345, None, 0.4321], alt_starts_km=[5.0, 7.0, 9.0] + ) + ) + solver_mod._process_solver_item( + (_s_in(nodes), {}, time.time()), + lambda s, c: pytest.fail("sweep ran in free mode"), + multistart_fn=multistart, + ) + + rec = state.mlat_solve_history[-1] + assert rec["outcome"] == "published" + assert rec["altitude_mode"] == "free" + assert rec["alt_starts_km"] == [5.0, 7.0, 9.0] + assert rec["alt_start_rms_us"] == [1.234, None, 0.432] + + def test_z_saturation_reaches_the_history(self): + nodes = ["n1", "n2", "n3"] + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", z_saturated=True)) + solver_mod._process_solver_item((_s_in(nodes), {}, time.time()), lambda s, c: None, multistart_fn=multistart) + assert state.mlat_solve_history[-1]["z_saturated"] is True + + def test_trimming_re_solves_through_the_multistart(self): + """A trim round must use the mode its first solve used, or the rms it + compares against the previous round is a different quantity.""" + full = ["n1", "n2", "n3", "n4", "bad"] + trimmed = ["n1", "n2", "n3", "n4"] + + def _result(nodes, s_in, *rest): + if "bad" in nodes: + return _stub_result( + nodes, + rms_delay=8.0, + altitude_mode="free", + per_node_delay_res_us={n: (12.0 if n == "bad" else 0.5) for n in nodes}, + ) + return _stub_result( + nodes, + rms_delay=0.8, + altitude_mode="free", + per_node_delay_res_us={n: 0.3 for n in nodes}, + ) + + multistart = _Recorder(_result) + result = solver_mod._process_solver_item( + (_s_in(full), {}, time.time()), + lambda s, c: pytest.fail("sweep ran during a free-mode trim"), + multistart_fn=multistart, + ) + + assert result is not None and result["n_nodes"] == 4 + assert sorted(m["node_id"] for m in multistart.calls[-1][0]["measurements"]) == trimmed + rec = state.mlat_solve_history[-1] + assert rec["outcome"] == "published" + assert rec["trimmed_node_ids"] == ["bad"] + assert rec["altitude_mode"] == "free" + + def test_an_unrecognised_mode_would_sweep(self): + """The flag degrades to the inert mode, like its siblings — asserted on + the resolution rule rather than by re-importing core.state.""" + state.SOLVER_ALT_MODE = "definitely-not-a-mode" + solve = _Recorder(lambda n, s, *r: _stub_result(n)) + multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken for a bad mode")) + assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart) + assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM) + + +class TestConfigsForSolverInput: + """Only the configs a candidate can reach are queued with it. + + The pool is a spawn pool, so whatever is queued is pickled and shipped on + every solve — 58 fleet configs against a candidate's 2-8 measurements. + """ + + _FLEET = {f"n{i}": {"rx_lat": 35.0 + i, "rx_lon": -82.0} for i in range(8)} + + def test_restricted_to_the_measurement_nodes(self): + s_in = _s_in(["n1", "n3", "n5"]) + cfgs = frame_processor.configs_for_solver_input(self._FLEET, s_in) + assert sorted(cfgs) == ["n1", "n3", "n5"] + assert cfgs["n3"] is self._FLEET["n3"] + + def test_unknown_measurement_nodes_are_simply_absent(self): + """A measurement from a node with no config is the case + solve_multinode already handles by skipping it — not an error here.""" + cfgs = frame_processor.configs_for_solver_input(self._FLEET, _s_in(["n1", "ghost"])) + assert sorted(cfgs) == ["n1"] + + def test_no_measurements_gives_nothing(self): + assert frame_processor.configs_for_solver_input(self._FLEET, {"measurements": []}) == {} + assert frame_processor.configs_for_solver_input(self._FLEET, {}) == {} diff --git a/docs/solverflow.md b/docs/solverflow.md index c626e5bf..50101e85 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -81,6 +81,7 @@ own. Everything that reaches a solve passes through one gate stack | `FRAME_WORKERS` | 4 (compose sets 6) | `main.py:164`, `docker-compose.yml:54` | | `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py:31,67` | | `KNOWN_LANE_MODE` default | `binding` | `core/state.py:72-74` | +| `SOLVER_ALT_MODE` default | `sweep` | `core/state.py:77-95` | --- @@ -453,12 +454,52 @@ flowchart TD m6 -->|"no"| m7["vz_saturated if vz on bound;
rms recomputed unweighted;
cov_en_km2 from s^2(J^T J)^-1"] m7 --> alt{"n_nodes >= 3?"} - alt -->|"yes"| sweep["_solve_best_altitude wrapper:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"] + alt -->|"yes"| mode{"SOLVER_ALT_MODE"} + mode -->|"sweep (default)"| sweep["_solve_best_altitude:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"] + mode -->|"free"| freealt["_solve_best_altitude:
ONE pool call to
solve_multinode_multistart,
3 start layers, z solved"] alt -->|"no, n=2"| single["_solve_best_altitude_n2:
one LM call at the
association altitude"] classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` +#### `SOLVER_ALT_MODE` — how the n>=3 solve gets its altitude + +`solve_multinode` pins altitude from `initial_guess.alt_km`, so the fix is only +as good as the altitude the caller found for it. `sweep`, the default, searches +the six fixed layers of `_SOLVER_ALT_LAYERS_KM` — 2 km apart, so the pin is +systematically up to 1 km wrong. On noise-free replay of this fleet's geometry +that quantisation alone left `rms_delay` at a 1.76 us median against the 3.0 us +gate at 6.5, while a solve at the true altitude reaches 0. Most of the gate's +budget is spent on the ladder, and the residual left over gets blamed on nodes: +trimming (6.4) drops measurements that were never the problem. + +`free` instead calls `solve_multinode_multistart`, which runs the LM with +altitude as a sixth unknown (state `[x, y, z, vx, vy, vz]`, z bounded +0.05–20 km, the `vz` bound unchanged) from three start layers — the one nearest +the association guess and its two neighbours — and keeps the lowest `rms_delay`. +Freeing z removes the ladder's quantisation but not the LM's locality, which is +what the several starts are still for. It is also cheaper: **one** process-pool +round trip per candidate instead of six, each of which pickles the node configs +the input needs. + +At n=2 the mode is inert — four residuals cannot support six unknowns, so the +geolocator pins altitude regardless and `_solve_best_altitude_n2` is unchanged. +Trimming re-solves through `_solve_best_altitude`, so a trim round inherits +whichever mode its first solve used. + +Both modes stamp `altitude_mode` (`"free"` / `"pinned"`) on every +`mlat_solve_history` record, published or rejected; `free` adds `alt_starts_km`, +`alt_start_rms_us` (each start's residual) and `z_saturated` (the altitude +analogue of `vz_saturated` — z stopped on a bound rather than converging, so +`alt_m` is the bound and not a fit). That is the comparison channel: deploy one +mode per environment and read the two lanes' `rms_delay` and `gt_error_km` off +`/api/test/mlat-history`. + +| Mode | Pool calls per n>=3 candidate | Altitude | +|---|---|---| +| `sweep` (default) | 6 (one per layer) | quantised to the nearest layer | +| `free` | 1 (three starts inside it) | solved, 0.05–20 km | + | Constant | Value | File:line | |---|---|---| | `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py:701` | diff --git a/libs/retina-geolocator b/libs/retina-geolocator index 2da39822..b87b4707 160000 --- a/libs/retina-geolocator +++ b/libs/retina-geolocator @@ -1 +1 @@ -Subproject commit 2da3982220374f05dd622633576108e57bd9e34f +Subproject commit b87b4707cecb331fea2a8895f3e63f256183f4a9 From a82e6b25d30fe16c26fcb6cd8eb700d0e1fb8f57 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 06:49:45 +0000 Subject: [PATCH 04/15] Satisfy the lint gate on the bench contamination scorer ruff format wrapped the truth-channel lookup and SIM910 wanted the redundant None default off dict.get. Co-Authored-By: Claude Fable 5.1 --- backend/scripts/association_bench.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py index 14dc6cdc..9e5792db 100644 --- a/backend/scripts/association_bench.py +++ b/backend/scripts/association_bench.py @@ -337,7 +337,7 @@ def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) the delay grid). Ambiguous attributions are counted in neither. """ oids = [ - det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"])), None) + det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"]))) for m in s_in.get("measurements") or [] ] if not oids: @@ -351,7 +351,11 @@ def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) guess = s_in.get("initial_guess") or {} contenders.sort( key=lambda o: min( - (_haversine_km(guess.get("lat", 0.0), guess.get("lon", 0.0), a, b) for a, b, oid, _ in truth if oid == o), + ( + _haversine_km(guess.get("lat", 0.0), guess.get("lon", 0.0), a, b) + for a, b, oid, _ in truth + if oid == o + ), default=float("inf"), ) ) @@ -1681,8 +1685,7 @@ def main(): "--merge-dist-km", type=float, default=None, - help="track mode: how close two pairings must be to merge into one " - "solver input (association._MERGE_DIST_KM)", + help="track mode: how close two pairings must be to merge into one solver input (association._MERGE_DIST_KM)", ) p.add_argument( "--pair-vel-exclusive", From 1c9e5def9b8988ee22214727ca211eec0938d627 Mon Sep 17 00:00:00 2001 From: jehanazad Date: Sat, 5 Sep 2026 06:56:57 +0000 Subject: [PATCH 05/15] Make the dark lane's blind spots readable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three of the signals the dark-lane recovery work is judged on could not be read from a running server, so every judgement about it has been offline replay. This adds no behaviour: nothing about which candidates solve, which solves publish, or what reaches the map changes. 1. /api/test/mlat-history caps per lane. The known lane writes ~16x the dark lane's volume, so the flat records[:1000] answered a 30 min request with the newest ~6 min of dark records and the rest of the window read as a quiet period — the same failure the shared deque had before #289 split it, moved into the reader. ?lane= narrows to one lane, ?limit= (max 5 000) is applied per lane, and lane_counts is reported pre-cap so truncation stays legible. 2. Resolve-slot skips are recorded, not just counted. Live the rule refuses ~1 537 candidates per 646 dark attempts per 30 min, and nothing said WHICH claim did it — so a skip that suppressed a duplicate and one that suppressed a different aircraft (tracker track ids are shared across candidates) were indistinguishable. A separate 500-entry deque carries the blocking claims; solver-stats windows it as resolve_skips, and ?kind=resolve_skips dumps it. Its own deque and not the solve history: skips outrun dark records two to one and would evict them. 3. Contamination is measured live. A dark record matched to ground truth now says which of its own contributing nodes could not see that aircraft (foreign_node_ids/contaminated), using the associator's own visibility predicate — the same gate known-lane claiming uses, so the two cannot drift apart. The GT trail lookup already happened; this costs one cone test per node. Records nothing could be asked about stay out of the denominator rather than counting as clean. 4. NODE_FRAME_MIN_INTERVAL_S drops are counted. frames_dropped is published but is the queue-saturation counter and reads zero throughout; the frames the per-node rate limiter refuses were uncounted entirely, so "the tracker sees what this node sent" looked true from every metric. docs/solverflow.md's file:line references are replaced with file + symbol — every one of them had drifted — and gain a section on reading these endpoints. Co-Authored-By: Claude Fable 5.1 --- backend/core/state.py | 35 +++- backend/routes/test.py | 135 ++++++++++++- backend/services/tasks/solver.py | 132 ++++++++++++- backend/services/tcp_handler.py | 6 + backend/tests/test_mlat_history.py | 288 ++++++++++++++++++++++++++++ backend/tests/test_solver_stats.py | 119 ++++++++++++ backend/tests/test_solver_worker.py | 44 +++++ docs/solverflow.md | 231 ++++++++++++---------- 8 files changed, 887 insertions(+), 103 deletions(-) diff --git a/backend/core/state.py b/backend/core/state.py index 634aa212..246907c9 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -475,6 +475,15 @@ def _adsb_for_seeding() -> dict[str, dict]: # Monotonic counter for dropped frames (useful for monitoring) frames_dropped: int = 0 +# Frames the per-node rate limiter refused before they ever reached +# frame_queue (tcp_handler's NODE_FRAME_MIN_INTERVAL_S gate). A different +# event from frames_dropped, which is queue saturation: this one is the +# pipeline deliberately sampling a node down to ~1 Hz, and a node streaming at +# 22 fps therefore reports a large number here while dropping nothing. It was +# uncounted, so "how much of a node's evidence does the tracker actually see" +# had no answer at all — the frames_dropped that IS published +# (/api/admin/metrics) says zero throughout. +node_frames_rate_limited: int = 0 frames_processed: int = 0 solver_successes: int = 0 solver_failures: int = 0 @@ -543,6 +552,25 @@ def _adsb_for_seeding() -> dict[str, dict]: # solver_stale_drops that means work was lost. solver_resolve_skips: int = 0 +# The dark-lane share of the counter above, split out because the two lanes +# read completely differently: an ADS-B-anchored duplicate that is skipped +# costs nothing (the transponder keeps the track alive anyway), while a +# skipped dark candidate may be the only chance that aircraft had of reaching +# the map this window. Lane is decided by solver._is_dark_solver_input, the +# same predicate routes.test._record_lane falls back to for a record that +# never got a key — and a skip never gets one. +solver_resolve_skips_dark: int = 0 + +# The last few hundred resolve-slot skips, with the claims that blocked them. +# Deliberately NOT the solve-history deque: a skip is not a solve outcome, and +# writing one record per skip into mlat_solve_history would evict the real +# records at roughly twice their rate (live: ~1 537 skips per 646 dark +# attempts per 30 min). Small and separate, read by +# /api/test/solver-stats' resolve_skips block and dumped by +# /api/test/mlat-history?kind=resolve_skips. ~250 B/entry. +SOLVER_RESOLVE_SKIPS_RECENT_MAX = 500 +solver_resolve_skips_recent: deque = deque(maxlen=SOLVER_RESOLVE_SKIPS_RECENT_MAX) + # Multinode entries removed because a later solve shared a source single-node # track with them AND the spatial/identical-inputs guard in solver.py's # _supersession_match agreed they are the same aircraft — the age-scaled @@ -746,12 +774,14 @@ def _reset_for_tests() -> None: global latest_mlat_accuracy_bytes, latest_mlat_verification_bytes global latest_storage_bytes, simulation_config global frames_dropped, frames_processed, solver_successes, solver_failures + global node_frames_rate_limited global adsb_seed_frames_autotagged, adsb_capture_ts_fallback global known_claims_made, known_claim_contentions, known_claims_bound global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes global coverage_rebuild_backlog global solver_queue_drops, solver_stale_drops, solver_resolve_skips + global solver_resolve_skips_dark global mn_superseded, mn_superseded_blocked, solver_trimmed global solver_consensus_selected, solver_consensus_filtered global solver_consensus_fallback, solver_consensus_shadow @@ -801,6 +831,7 @@ def _reset_for_tests() -> None: track_archive_buffer.clear() mlat_solve_history.clear() mlat_solve_history_known.clear() + solver_resolve_skips_recent.clear() accuracy_samples.clear() mlat_samples.clear() for q in (frame_queue, solver_queue): @@ -832,7 +863,7 @@ def _reset_for_tests() -> None: simulation_config = dict(_SIMULATION_CONFIG_DEFAULTS) with counters_lock: - frames_dropped = frames_processed = 0 + frames_dropped = frames_processed = node_frames_rate_limited = 0 solver_successes = solver_failures = n2_unconfirmed = 0 adsb_seed_frames_autotagged = adsb_capture_ts_fallback = 0 known_claims_made = known_claim_contentions = known_claims_bound = 0 @@ -841,7 +872,7 @@ def _reset_for_tests() -> None: coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0 coverage_rebuild_backlog = 0 solver_stale_drops = 0 - solver_resolve_skips = 0 + solver_resolve_skips = solver_resolve_skips_dark = 0 mn_superseded = mn_superseded_blocked = 0 solver_trimmed = 0 solver_consensus_selected = solver_consensus_filtered = 0 diff --git a/backend/routes/test.py b/backend/routes/test.py index e4018b36..5909ff9d 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -752,11 +752,40 @@ def _record_lane(rec: dict) -> str: return "adsb" if hexn and is_transponder_hex(hexn) else "dark" +_LANES = ("dark", "known", "adsb") + + +def _cap_per_lane(records: list[dict], limit: int) -> list[dict]: + """Keep the ``limit`` newest records OF EACH LANE, newest first. + + ``records`` must already be newest-first. A single flat ``[:limit]`` + made the cap a race between lanes rather than a retention rule, exactly + as the shared deque did before PR #289 split it: the known lane writes + ~16x the dark lane's volume, so a flat 1 000-record answer to a 30 min + request held only the newest ~6 min of dark records and the rest of the + window read as a quiet period. Capping per lane means known-lane volume + can never evict a dark record from a response. + """ + kept: list[dict] = [] + counts: dict[str, int] = {} + for r in records: + lane = _record_lane(r) + n = counts.get(lane, 0) + if n >= limit: + continue + counts[lane] = n + 1 + kept.append(r) + return kept + + @router.get("/api/test/mlat-history") async def mlat_history( hex: str | None = None, all: int = 0, minutes: float = 30.0, + lane: str = "all", + limit: int = 1000, + kind: str = "solves", ): """Per-solve MLAT history from the last ~30 minutes. @@ -770,22 +799,73 @@ async def mlat_history( merged here, so both lanes answer either query exactly as they did when they shared a deque. + ``?lane=dark|known|adsb`` narrows the answer to one lane (default + ``all``, classified by ``_record_lane``); ``?limit=`` caps the record + list (default 1 000, max 5 000) and is applied PER LANE, so a known-lane + burst can never push dark records out of an ``all`` response — see + _cap_per_lane. + + ``?kind=resolve_skips`` dumps a different store entirely: the solver's + recent resolve-slot refusals (state.solver_resolve_skips_recent), each + with the claims that blocked it. Those are not solve outcomes and + deliberately do not live in the solve-history deques. + ``window_effective_minutes`` is how much of the requested window the stores actually hold — below ``window_minutes`` the answer is truncated. """ + if lane not in ("all", *_LANES): + return Response( + content=orjson.dumps({"error": f"lane must be one of all,{','.join(_LANES)}"}), + media_type="application/json", + status_code=400, + ) + if kind not in ("solves", "resolve_skips"): + return Response( + content=orjson.dumps({"error": "kind must be solves or resolve_skips"}), + media_type="application/json", + status_code=400, + ) minutes = max(0.0, min(minutes, 35.0)) + limit = max(1, min(int(limit), 5000)) cutoff_ms = int((time.time() - minutes * 60.0) * 1000) + + if kind == "resolve_skips": + skips = [ + s + for s in list(state.solver_resolve_skips_recent) + if s["ts_ms"] >= cutoff_ms and (lane == "all" or s["lane"] == lane) + ] + skips.reverse() # newest first + payload = { + "kind": "resolve_skips", + "window_minutes": minutes, + "lane": lane, + "lane_counts": {ln: sum(1 for s in skips if s["lane"] == ln) for ln in _LANES}, + "n_records": len(skips), + "records": skips[:limit], + } + return Response(content=orjson.dumps(payload), media_type="application/json") + merged = _merged_solve_history() effective_minutes = _window_effective_minutes(merged, minutes) records = [r for r in merged if r["ts_ms"] >= cutoff_ms] records.reverse() # newest first + if lane != "all": + records = [r for r in records if _record_lane(r) == lane] + lane_counts = dict.fromkeys(_LANES, 0) + for r in records: + lane_counts[_record_lane(r)] += 1 if all: payload = { "window_minutes": minutes, "window_effective_minutes": effective_minutes, + "lane": lane, + # Pre-cap, so a truncated `records` can be read against what the + # window actually held. + "lane_counts": lane_counts, "n_records": len(records), - "records": records[:1000], + "records": _cap_per_lane(records, limit), } return Response(content=orjson.dumps(payload), media_type="application/json") @@ -813,6 +893,8 @@ async def mlat_history( "hex": norm, "window_minutes": minutes, "window_effective_minutes": effective_minutes, + "lane": lane, + "lane_counts": lane_counts, "n_solves": len(solves), "solves": solves[:500], "rejects_nearby": { @@ -914,6 +996,46 @@ def _solver_window_stats(minutes: float) -> dict: reason = outcome[len("rejected_") :] if outcome.startswith("rejected_") else outcome by_reason[reason] = by_reason.get(reason, 0) + 1 + # ── cluster contamination (dark, windowed) ────────────────────────────── + # Of the dark records this window that matched ground truth, how many + # carried a node that could not see the aircraft they were matched to — + # the live version of the offline number Phase 2 exists to move (~60 %). + # Records without the stamp are records nothing could be asked about (no + # GT match, or no registered geometry for any contributing node) and stay + # out of the denominator rather than counting as clean; see + # solver._stamp_foreign_nodes. + judged = [r for r in records if r.get("foreign_node_ids") is not None] + contaminated = [r for r in judged if r.get("contaminated")] + n_judged = len(judged) + contamination = { + "records_with_gt": n_judged, + "contaminated": len(contaminated), + "pct": round(100.0 * len(contaminated) / n_judged, 1) if n_judged else None, + "foreign_nodes_per_record": ( + round(sum(len(r["foreign_node_ids"]) for r in judged) / n_judged, 2) if n_judged else None + ), + } + + # ── resolve-slot skips (windowed, from the skip deque) ────────────────── + # The counter in "counters" below is since-boot; these are the skips that + # happened inside this window, so they can be read against the attempts in + # the same window. attempts_ratio is (all-lane skips / DARK attempts) — + # the shape the acceptance target for the claim-on-publish fix is quoted + # in (live baseline ~1 537 / 646 = 2.4), not a per-lane rate. The dark + # numerator is published beside it for anyone who wants one. + all_skips = list(state.solver_resolve_skips_recent) + skips = [s for s in all_skips if s["ts_ms"] >= cutoff_ms] + resolve_skips = { + "total": len(skips), + "dark": sum(1 for s in skips if s["lane"] == "dark"), + "attempts_ratio": round(len(skips) / attempts, 3) if attempts else None, + # The skip deque is 500 entries against a live rate of ~50/min, so a + # long window IS truncated here even when the solve-history stores + # cover it. Same honesty rule as window_effective_minutes above: read + # it before reading total as a window count. + "window_effective_minutes": _window_effective_minutes(all_skips, minutes), + } + pos_errors.sort() n_err = len(pos_errors) median_err = pos_errors[n_err // 2] if n_err else None @@ -1066,6 +1188,9 @@ def _solver_window_stats(minutes: float) -> dict: "published": {"total": n2 + n3plus, "n2": n2, "n3plus": n3plus}, "rejects": {"total": reject_total, "by_reason": by_reason}, "position_error_km": {"median": median_err, "p90": p90_err, "n": n_err}, + # Both windowed and both DARK-lane, like the funnel above them. + "contamination": contamination, + "resolve_skips": resolve_skips, "ghosts": { # Scoped to dark tracks: precision_pct's denominator is # dark_tracks, and it is None (not 100.0) when there are none. @@ -1192,7 +1317,15 @@ def _solver_window_stats(minutes: float) -> dict: "solver_trimmed": state.solver_trimmed, "stale_drops": state.solver_stale_drops, "resolve_skips": state.solver_resolve_skips, + # Dark share of the line above. The windowed version, with the + # blocking claims, is the "resolve_skips" block further up. + "resolve_skips_dark": state.solver_resolve_skips_dark, "queue_drops": state.solver_queue_drops, + # Frames the per-node rate limiter refused before the tracker ever + # saw them (tcp_handler's NODE_FRAME_MIN_INTERVAL_S). Not the + # same event as /api/admin/metrics' frames_dropped, which is + # frame_queue saturation and normally reads zero. + "node_frames_rate_limited": state.node_frames_rate_limited, "worker_errors": state.solver_worker_errors, "vel_untrusted_published": state.solver_vel_untrusted_published, }, diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 7c6b947c..5faf7bc5 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -11,6 +11,8 @@ from collections import deque from concurrent.futures.process import BrokenProcessPool +from retina_analytics.association import _point_in_beam + from config.constants import ( ARC_ONLY_ANOMALY_ALLOWLIST, ASSOC_GRID_STEP_KM, @@ -1036,6 +1038,64 @@ def _claim_resolve_slot(s_in, now_s: float) -> bool: return True +def _resolve_slot_blockers(track_ids, now_s: float) -> list[dict]: + """The live claims covering ``track_ids``, for a skip record. + + Read-only, and taken after the refusal rather than during it: the check + itself must stay one atomic test-and-claim, and a skip is rare enough + (relative to the queue drain rate) that a second lock acquisition on that + path costs nothing. Any claim that moves between the two is a claim the + diagnosis would have wanted to name anyway. + """ + cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S + out: list[dict] = [] + with _RECENT_SOLVES_LOCK: + for tid in track_ids: + held = _RECENT_SOLVES.get(tid) + if held is not None and held[0] > cutoff: + out.append({"track_id": tid, "held_ts": round(held[0], 3), "held_n": held[1]}) + return out + + +def _record_resolve_skip(s_in, now_s: float, blocking: list[dict] | None = None) -> None: + """Count and remember one resolve-slot refusal. + + The counter alone could not answer the question the suppression rule + raises — *whose* claim blocked this, and was it even the same aircraft. + Live on the test droplet the rule refuses ~1 537 candidates per 646 dark + attempts per 30 min, and nothing recorded which claim did it, so a skip + that suppressed a genuinely different aircraft (tracker track ids are + shared across candidates — see _supersession_match) was indistinguishable + from one that suppressed a duplicate. The deque carries the blocking + claims and the candidate's own guess position so the two can be told apart + after the fact. + + Deliberately NOT a solve-history record: skips outrun real dark records + roughly two to one, and writing them into that deque would evict the + solves the same investigation needs (see state.solver_resolve_skips_recent). + """ + s = s_in if isinstance(s_in, dict) else {} + track_ids = list(s.get("track_ids") or []) + dark = _is_dark_solver_input(s) + state.bump_counter("solver_resolve_skips") + if dark: + state.bump_counter("solver_resolve_skips_dark") + ig = s.get("initial_guess") or {} + state.solver_resolve_skips_recent.append( + { + "ts_ms": int(now_s * 1000), + # No key is minted for a candidate that never solves, so lane is + # the same fallback routes.test._record_lane uses for a reject. + "lane": "dark" if dark else "adsb", + "track_ids": track_ids, + "n_nodes": int(s.get("n_nodes") or 0), + "blocking": _resolve_slot_blockers(track_ids, now_s) if blocking is None else blocking, + "guess_lat": round(float(ig["lat"]), 6) if ig.get("lat") else None, + "guess_lon": round(float(ig["lon"]), 6) if ig.get("lon") else None, + } + ) + + # Which single-node track pair currently owns a published n=2 track, and how # well it fitted. One track is one aircraft, so two pairings sharing a track # are mutually exclusive; the better chi2 wins and the loser is withheld. @@ -1384,6 +1444,59 @@ def _is_dark_solver_input(s_in) -> bool: return not (hx and is_transponder_hex(hx)) +def _stamp_foreign_nodes(rec: dict) -> None: + """Stamp which of a dark record's own nodes could not see the aircraft. + + Cluster contamination is the dark lane's largest known defect — a + candidate assembled by format_track_pairs_for_solver can carry a node + whose track belongs to a *different* aircraft, and the solver then fits a + geometry no single aircraft ever occupied. Offline the audit measured it + at ~60 % of dark candidates; this makes the same number live. + + The test is the associator's own visibility predicate applied whole + (retina_analytics.association._point_in_beam against the registered + NodeGeometry), which is the same gate known-lane claiming uses — claiming + and the dark lane must mean the same thing by "this node can see there", + and a second bespoke rule here would let the two disagree. Two + consequences worth knowing: it is a ground-projected bearing/footprint + test with no altitude term, and under FOV_MODE=active it is the learned + FOV rather than the theoretical wedge. Both are exactly what the rest of + the pipeline believes about coverage, which is the point. + + Position is the matched ground-truth point already stamped on the record + (gt_lat/gt_lon at the solve epoch), so this costs no extra trail lookup — + only one cone test per contributing node. Nodes trimmed out by + _trim_and_resolve are included: a node dropped for a bad residual is + precisely the contamination this measures, and leaving it out would hide + every case trimming already rescued. + + A node with no registered geometry is not judged either way. When that + leaves nothing judgeable the record is left unstamped rather than stamped + clean, so contamination_pct never counts an abstention as innocence. + """ + lat, lon = rec.get("gt_lat"), rec.get("gt_lon") + if lat is None or lon is None: + return + node_ids = list(rec.get("contributing_node_ids") or []) + node_ids += [nid for nid in (rec.get("trimmed_node_ids") or []) if nid not in node_ids] + if not node_ids: + return + geometries = state.node_associator.node_geometries + judged = 0 + foreign: list[str] = [] + for nid in node_ids: + geo = geometries.get(nid) + if geo is None: + continue + judged += 1 + if not _point_in_beam(lat, lon, geo): + foreign.append(nid) + if not judged: + return + rec["foreign_node_ids"] = foreign + rec["contaminated"] = bool(foreign) + + def _record_dark_accuracy_sample(rec: dict) -> None: """Offer one published DARK solve to the rolling accuracy store. @@ -1460,7 +1573,14 @@ def _record_solve_history( ``extra`` merges caller-supplied fields (trim metadata, beam-rejection diagnostics) into the record. Applied before the GT stamp so it can - never clobber gt_hex/gt_error_km/gt_lat/gt_lon. + never clobber gt_hex/gt_error_km/gt_lat/gt_lon — and so the trimmed node + ids it carries are in hand for the contamination stamp below. + + ``foreign_node_ids``/``contaminated`` are stamped on DARK records that + matched ground truth: which of this candidate's own nodes could not see + the aircraft it was matched to (see _stamp_foreign_nodes). Absent on + every other record, which is what /api/test/solver-stats' contamination + block counts as "not judged" rather than as clean. """ r = result if isinstance(result, dict) else {} s = s_in if isinstance(s_in, dict) else {} @@ -1574,6 +1694,11 @@ def _record_solve_history( rec["vel_err_ms"] = round(math.hypot(ve - gt_ve, vn - gt_vn), 1) else: rec["vel_err_ms"] = None + # Live cluster-contamination metric, dark lane only and only where ground + # truth actually matched — without a truth position there is nothing to + # ask "could this node see it?" about. See _stamp_foreign_nodes. + if _dark and rec.get("gt_hex"): + _stamp_foreign_nodes(rec) if rec["outcome"] == "published" and _dark and rec.get("gt_error_km") is not None: _record_dark_accuracy_sample(rec) # Route by lane: the known lane's per-hex-per-pass volume would otherwise @@ -1724,8 +1849,9 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus # here rather than at enqueue: the frame path must not carry solver state, # and a copy that queued before its twin was solved can only be recognised # once it reaches a worker. - if not _claim_resolve_slot(s_in, time.time()): - state.bump_counter("solver_resolve_skips") + _now_s = time.time() + if not _claim_resolve_slot(s_in, _now_s): + _record_resolve_skip(s_in, _now_s) return None n_nodes = s_in.get("n_nodes", 0) if isinstance(s_in, dict) else 0 consensus_meta: dict | None = None diff --git a/backend/services/tcp_handler.py b/backend/services/tcp_handler.py index bbe82197..6b21be77 100644 --- a/backend/services/tcp_handler.py +++ b/backend/services/tcp_handler.py @@ -529,6 +529,12 @@ def _enqueue_detection(msg: dict, node_id: str | None): if node_id: last = _per_node_last_enqueue.get(node_id, 0.0) if (now_m - last) < _NODE_MIN_INTERVAL_S: + # Counted, not silent: this is the only place a node's detections + # are discarded on purpose, and until now nothing said how many. + # state.frames_dropped is the queue-saturation counter and reads + # zero throughout, so "the tracker sees every frame this node + # sent" looked true from every published metric. + state.bump_counter("node_frames_rate_limited") return # position already updated; skip expensive queue work _per_node_last_enqueue[node_id] = now_m diff --git a/backend/tests/test_mlat_history.py b/backend/tests/test_mlat_history.py index e03d9da4..e8e91bbe 100644 --- a/backend/tests/test_mlat_history.py +++ b/backend/tests/test_mlat_history.py @@ -777,3 +777,291 @@ def test_known_lane_records_are_not_sampled_by_this_path(self): extra={"known_lane": True, "label": "truth_match", "published": True}, ) assert not state.accuracy_samples + + +def _register_geo(node_id, beam_azimuth_deg, rx_lat=LAT, rx_lon=LON, max_range_km=50.0): + """Register one node geometry with the associator, aimed as given. + + The contamination stamp asks the associator's own visibility predicate, + so a test node has to exist there rather than in a config dict. + """ + from retina_analytics.association import NodeGeometry + + geo = NodeGeometry( + node_id=node_id, + rx_lat=rx_lat, + rx_lon=rx_lon, + rx_alt_km=0.0, + tx_lat=rx_lat + 0.5, + tx_lon=rx_lon + 0.5, + tx_alt_km=0.3, + beam_azimuth_deg=beam_azimuth_deg, + beam_width_deg=41.0, + max_range_km=max_range_km, + ) + state.node_associator.node_geometries[node_id] = geo + return geo + + +class TestForeignNodeStamp: + """A dark record matched to ground truth says which of its own nodes + could not have seen that aircraft. + + Cluster contamination — a solver candidate assembled from tracks of two + different aircraft — is the dark lane's largest known defect, and until + now it was measurable only offline. The verdict is the associator's own + visibility predicate, the same one known-lane claiming gates on. + """ + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + def _run(self, contributing=("n_in", "n_out"), **extra): + return solver_mod._process_solver_item( + (dict(_CONFIRMED_N2), {}, time.time()), + _solve_fn(contributing_node_ids=list(contributing), **extra), + ) + + def test_a_node_aimed_away_is_named_foreign(self): + # Ground truth sits due north of both nodes; n_in is aimed at it and + # n_out at the opposite bearing. + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=180.0) + self._run() + rec = state.mlat_solve_history[0] + assert rec["gt_hex"] == "abc123" + assert rec["foreign_node_ids"] == ["n_out"] + assert rec["contaminated"] is True + + def test_all_nodes_in_cone_is_not_contaminated(self): + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=10.0) + self._run() + rec = state.mlat_solve_history[0] + assert rec["foreign_node_ids"] == [] + assert rec["contaminated"] is False + + def test_a_node_out_of_range_is_foreign(self): + """Range, not only bearing: the predicate applies whole.""" + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=0.0, max_range_km=1.0) + self._run() + assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_out"] + + def test_trimmed_nodes_are_judged_too(self): + """A node dropped by _trim_and_resolve is exactly the contamination + this measures — excluding it would hide every case trimming already + rescued.""" + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_trimmed", beam_azimuth_deg=180.0) + solver_mod._record_solve_history( + "published", + dict(_CONFIRMED_N2), + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2, "contributing_node_ids": ["n_in"]}, + solve_key="mn-dark-1", + raw_lat=LAT, + raw_lon=LON, + extra={"trimmed_node_ids": ["n_trimmed"], "trim_rounds": 1}, + ) + assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_trimmed"] + + def test_no_ground_truth_means_no_stamp(self): + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=180.0) + self._run() + rec = state.mlat_solve_history[0] + assert "foreign_node_ids" not in rec + assert "contaminated" not in rec + + def test_unregistered_nodes_are_not_stamped_clean(self): + """Nothing judgeable is an abstention, not innocence.""" + _put_gt(lat=LAT + 0.05, lon=LON) + self._run() + rec = state.mlat_solve_history[0] + assert rec["gt_hex"] == "abc123" + assert "foreign_node_ids" not in rec + + def test_an_adsb_record_is_not_stamped(self): + """Dark lane only — the tagged lane's identity is not in doubt.""" + _put_gt(lat=LAT + 0.05, lon=LON) + _register_geo("n_in", beam_azimuth_deg=0.0) + _register_geo("n_out", beam_azimuth_deg=180.0) + s_in = dict(_CONFIRMED_N2, adsb_hex="abc123") + solver_mod._process_solver_item( + (s_in, {}, time.time()), + _solve_fn(contributing_node_ids=["n_in", "n_out"]), + ) + assert "foreign_node_ids" not in state.mlat_solve_history[0] + + +class TestLaneFilterAndPerLaneCap: + """?lane= and ?limit= on /api/test/mlat-history. + + The flat records[:1000] cap made the response a race between lanes: the + known lane writes ~16x the dark lane's volume, so a 30 min request held + only the newest ~6 min of dark records and the rest of the window read as + a quiet period. The cap is now per lane. + """ + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + def _client(self): + from main import app + + return TestClient(app) + + def _dark(self, n=1): + for _ in range(n): + solver_mod._record_solve_history( + "published", + {"n_nodes": 3}, + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3}, + solve_key="mn-dark-1", + raw_lat=LAT, + raw_lon=LON, + ) + + def _known(self, n=1): + for _ in range(n): + solver_mod._record_solve_history( + "known_truth_match", + {"n_nodes": 2, "adsb_hex": "abc123", "initial_guess": {"lat": LAT, "lon": LON}}, + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2}, + extra={"known_lane": True, "label": "truth_match", "published": False}, + ) + + def _adsb(self, n=1): + for _ in range(n): + solver_mod._record_solve_history( + "published", + {"n_nodes": 3, "adsb_hex": "abc123"}, + {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3}, + solve_key="mn-adsb-abc123", + raw_lat=LAT, + raw_lon=LON, + ) + + def test_default_lane_is_all_and_counts_every_lane(self): + self._dark() + self._known() + self._adsb() + data = self._client().get("/api/test/mlat-history?all=1").json() + assert data["lane"] == "all" + assert data["lane_counts"] == {"dark": 1, "known": 1, "adsb": 1} + assert data["n_records"] == 3 + + def test_lane_dark_returns_only_dark_records(self): + self._dark(2) + self._known(3) + self._adsb(1) + data = self._client().get("/api/test/mlat-history?all=1&lane=dark").json() + assert data["n_records"] == 2 + assert data["lane_counts"] == {"dark": 2, "known": 0, "adsb": 0} + assert all(r["solve_key"] == "mn-dark-1" for r in data["records"]) + + def test_lane_known_returns_only_known_records(self): + self._dark(2) + self._known(3) + data = self._client().get("/api/test/mlat-history?all=1&lane=known").json() + assert data["n_records"] == 3 + assert all(r["known_lane"] for r in data["records"]) + + def test_unknown_lane_is_rejected(self): + assert self._client().get("/api/test/mlat-history?all=1&lane=bogus").status_code == 400 + + def test_known_volume_cannot_evict_dark_records_from_the_response(self): + """The bug the per-lane cap fixes, at 1/500 scale.""" + self._dark(2) + self._known(20) + data = self._client().get("/api/test/mlat-history?all=1&limit=2").json() + # 2 dark + 2 known survive the cap; the flat cap would have returned + # the 2 newest records overall, both known. + lanes = [("known" if r.get("known_lane") else "dark") for r in data["records"]] + assert sorted(lanes) == ["dark", "dark", "known", "known"] + # n_records / lane_counts stay pre-cap so truncation is legible. + assert data["n_records"] == 22 + assert data["lane_counts"] == {"dark": 2, "known": 20, "adsb": 0} + + def test_limit_is_clamped_to_the_maximum(self): + self._dark(3) + data = self._client().get("/api/test/mlat-history?all=1&limit=99999").json() + assert len(data["records"]) == 3 + + def test_hex_lookup_reports_the_lane_block_too(self): + self._dark() + rec = state.mlat_solve_history[0] + data = self._client().get(f"/api/test/mlat-history?hex={rec['solver_hex']}").json() + assert data["lane"] == "all" + assert data["lane_counts"]["dark"] == 1 + + +class TestResolveSkipDump: + """?kind=resolve_skips dumps the solver's skip deque. + + A skip is not a solve outcome and must not be written into the + solve-history deques: on the live fleet skips outrun dark records roughly + two to one and would evict exactly the records an investigation needs. + """ + + def setup_method(self): + state._reset_for_tests() + solver_mod._reset_for_tests() + + def teardown_method(self): + solver_mod._reset_for_tests() + + def _client(self): + from main import app + + return TestClient(app) + + def _skip(self, track_ids=("a1", "b1"), n_nodes=3, **s_in): + now = time.time() + s = dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids), **s_in) + solver_mod._claim_resolve_slot(s, now) + assert solver_mod._claim_resolve_slot(dict(s), now) is False + solver_mod._record_resolve_skip(dict(s), now) + + def test_skip_records_the_blocking_claim(self): + self._skip() + data = self._client().get("/api/test/mlat-history?kind=resolve_skips").json() + assert data["kind"] == "resolve_skips" + assert data["n_records"] == 1 + rec = data["records"][0] + assert rec["lane"] == "dark" + assert rec["track_ids"] == ["a1", "b1"] + assert rec["n_nodes"] == 3 + assert {b["track_id"] for b in rec["blocking"]} == {"a1", "b1"} + assert all(b["held_n"] == 3 for b in rec["blocking"]) + + def test_skips_do_not_land_in_the_solve_history(self): + self._skip() + assert not state.mlat_solve_history + assert not state.mlat_solve_history_known + + def test_lane_filter_applies_to_skips(self): + self._skip(track_ids=("a1", "b1")) + self._skip(track_ids=("a2", "b2"), adsb_hex="abc123") + assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=dark").json()["n_records"] == 1 + assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=adsb").json()["n_records"] == 1 + assert self._client().get("/api/test/mlat-history?kind=resolve_skips").json()["lane_counts"] == { + "dark": 1, + "known": 0, + "adsb": 1, + } + + def test_unknown_kind_is_rejected(self): + assert self._client().get("/api/test/mlat-history?kind=bogus").status_code == 400 diff --git a/backend/tests/test_solver_stats.py b/backend/tests/test_solver_stats.py index 67bc599c..806e8992 100644 --- a/backend/tests/test_solver_stats.py +++ b/backend/tests/test_solver_stats.py @@ -257,6 +257,8 @@ def test_consensus_and_counters_reflect_state(self): state.solver_consensus_fallback = 9 state.solver_consensus_shadow = 10 state.solver_vel_untrusted_published = 11 + state.solver_resolve_skips_dark = 9 + state.node_frames_rate_limited = 13 out = _solver_window_stats(10.0) assert out["counters"] == { "successes": 5, @@ -265,7 +267,9 @@ def test_consensus_and_counters_reflect_state(self): "solver_trimmed": 3, "stale_drops": 4, "resolve_skips": 12, + "resolve_skips_dark": 9, "queue_drops": 6, + "node_frames_rate_limited": 13, "worker_errors": 0, "vel_untrusted_published": 11, } @@ -788,3 +792,118 @@ def get(self, *a, **kw): state.multinode_tracks["mn-dark-1"] = {"lat": 35.0, "lon": -82.0} state.adsb_aircraft["real1"] = _MutatingFix({"lat": 35.009, "lon": -82.0, "last_seen_ms": now_ms}) assert _solver_window_stats(10.0)["ghosts"]["ghost_tracks"] == 0 + + +def _skip_rec(lane="dark", age_s=0.0, track_ids=("a1",), n_nodes=3): + return { + "ts_ms": int((time.time() - age_s) * 1000), + "lane": lane, + "track_ids": list(track_ids), + "n_nodes": n_nodes, + "blocking": [{"track_id": track_ids[0], "held_ts": time.time() - age_s, "held_n": n_nodes}], + "guess_lat": None, + "guess_lon": None, + } + + +class TestResolveSkipBlock: + """Resolve-slot skips are windowed from their own deque, not from the + since-boot counter, so they can be read against the attempts in the same + window — the ratio the claim-on-publish fix is judged on.""" + + def setup_method(self): + state._reset_for_tests() + + def test_totals_split_by_lane(self): + for _ in range(3): + state.solver_resolve_skips_recent.append(_skip_rec("dark")) + state.solver_resolve_skips_recent.append(_skip_rec("adsb")) + out = _solver_window_stats(10.0)["resolve_skips"] + assert out["total"] == 4 + assert out["dark"] == 3 + + def test_window_excludes_old_skips(self): + state.solver_resolve_skips_recent.append(_skip_rec(age_s=20 * 60)) + state.solver_resolve_skips_recent.append(_skip_rec(age_s=1)) + assert _solver_window_stats(10.0)["resolve_skips"]["total"] == 1 + + def test_attempts_ratio_is_skips_over_dark_attempts(self): + for _ in range(4): + state.solver_resolve_skips_recent.append(_skip_rec()) + state.mlat_solve_history.append(_rec("published")) + state.mlat_solve_history.append(_rec("rejected_beam")) + out = _solver_window_stats(10.0) + assert out["attempts"] == 2 + assert out["resolve_skips"]["attempts_ratio"] == 2.0 + + def test_attempts_ratio_is_none_without_attempts(self): + state.solver_resolve_skips_recent.append(_skip_rec()) + assert _solver_window_stats(10.0)["resolve_skips"]["attempts_ratio"] is None + + def test_window_effective_minutes_exposes_a_truncated_deque(self): + """The deque is 500 entries against ~50 skips/min live, so a long + window IS truncated here even when the solve stores cover it.""" + state.solver_resolve_skips_recent.append(_skip_rec(age_s=6 * 60)) + out = _solver_window_stats(30.0)["resolve_skips"] + assert 5.9 <= out["window_effective_minutes"] <= 6.1 + + def test_a_skip_is_not_an_attempt_or_a_reject(self): + """Skips must not leak into the funnel — they never reached a solve.""" + for _ in range(5): + state.solver_resolve_skips_recent.append(_skip_rec()) + out = _solver_window_stats(10.0) + assert out["attempts"] == 0 + assert out["rejects"]["total"] == 0 + + +def _gt_rec(foreign=(), **kw): + """A dark record carrying the contamination stamp.""" + rec = _rec("published", **kw) + rec["gt_hex"] = "abc123" + rec["foreign_node_ids"] = list(foreign) + rec["contaminated"] = bool(foreign) + return rec + + +class TestContaminationBlock: + """Live cluster contamination: of the dark records that matched ground + truth, how many carried a node that could not see the aircraft.""" + + def setup_method(self): + state._reset_for_tests() + + def test_pct_and_mean_over_judged_records(self): + state.mlat_solve_history.append(_gt_rec(foreign=["n1"])) + state.mlat_solve_history.append(_gt_rec(foreign=["n1", "n2"])) + state.mlat_solve_history.append(_gt_rec(foreign=[])) + state.mlat_solve_history.append(_gt_rec(foreign=[])) + out = _solver_window_stats(10.0)["contamination"] + assert out["records_with_gt"] == 4 + assert out["contaminated"] == 2 + assert out["pct"] == 50.0 + assert out["foreign_nodes_per_record"] == 0.75 + + def test_unstamped_records_are_out_of_the_denominator(self): + """No GT match, or no judgeable node geometry, is an abstention — not + a clean record.""" + state.mlat_solve_history.append(_gt_rec(foreign=["n1"])) + state.mlat_solve_history.append(_rec("published")) + out = _solver_window_stats(10.0)["contamination"] + assert out["records_with_gt"] == 1 + assert out["pct"] == 100.0 + + def test_empty_window_abstains_rather_than_reporting_zero(self): + out = _solver_window_stats(10.0)["contamination"] + assert out == { + "records_with_gt": 0, + "contaminated": 0, + "pct": None, + "foreign_nodes_per_record": None, + } + + def test_known_lane_records_are_not_counted(self): + """Dark lane only, like every other block in the funnel.""" + rec = _gt_rec(foreign=["n1"]) + rec["known_lane"] = True + state.mlat_solve_history_known.append(rec) + assert _solver_window_stats(10.0)["contamination"]["records_with_gt"] == 0 diff --git a/backend/tests/test_solver_worker.py b/backend/tests/test_solver_worker.py index 6f5ea7b3..a4a6c8e9 100644 --- a/backend/tests/test_solver_worker.py +++ b/backend/tests/test_solver_worker.py @@ -38,6 +38,7 @@ def _reset_state(): state.n2_unconfirmed = 0 state.solver_stale_drops = 0 state.solver_resolve_skips = 0 + state.solver_resolve_skips_dark = 0 state.multinode_tracks.clear() state.task_last_success.clear() @@ -415,6 +416,49 @@ def solve_fn(s_in, cfgs): assert state.solver_failures == 0 assert state.solver_stale_drops == 0 + def test_a_skip_is_recorded_with_the_claim_that_blocked_it(self): + """The counter alone cannot say WHOSE claim suppressed a candidate, + and tracker track ids are shared between different aircraft — so a + skip that suppressed a duplicate and one that suppressed a neighbour + looked identical. The deque carries the blocking claims.""" + _reset_state() + state.solver_resolve_skips_recent.clear() + now = time.time() + s_in = dict(self._s_in(["a1", "b1"], n_nodes=4), initial_guess={"lat": 35.0, "lon": -82.0}) + assert solver_mod._claim_resolve_slot(dict(s_in), now) is True + assert solver_mod._claim_resolve_slot(dict(s_in), now) is False + solver_mod._record_resolve_skip(dict(s_in), now) + + assert state.solver_resolve_skips == 1 + assert state.solver_resolve_skips_dark == 1 + assert len(state.solver_resolve_skips_recent) == 1 + rec = state.solver_resolve_skips_recent[0] + assert rec["lane"] == "dark" + assert rec["track_ids"] == ["a1", "b1"] + assert rec["n_nodes"] == 4 + assert rec["guess_lat"] == 35.0 + assert {b["track_id"]: b["held_n"] for b in rec["blocking"]} == {"a1": 4, "b1": 4} + + def test_a_tagged_candidate_is_counted_but_not_as_dark(self): + _reset_state() + state.solver_resolve_skips_recent.clear() + now = time.time() + s_in = dict(self._s_in(["a1"], n_nodes=3), adsb_hex="abc123") + solver_mod._record_resolve_skip(s_in, now) + assert state.solver_resolve_skips == 1 + assert state.solver_resolve_skips_dark == 0 + assert state.solver_resolve_skips_recent[0]["lane"] == "adsb" + + def test_skips_never_enter_the_solve_history(self): + """One skip per solve-history record would evict the solves the same + investigation needs — live, skips outrun dark records two to one.""" + _reset_state() + state.mlat_solve_history.clear() + s_in = self._s_in(["a1", "b1"]) + solver_mod._record_resolve_skip(s_in, time.time()) + assert not state.mlat_solve_history + assert not state.mlat_solve_history_known + class TestSolveBestAltitude: """Altitude-sweep helpers: n_nodes >= 3 uses a layer sweep, n_nodes = 2 uses initial_guess directly.""" diff --git a/docs/solverflow.md b/docs/solverflow.md index c626e5bf..c80c4673 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -11,10 +11,12 @@ beyond what publication needs, see [`pipeline.md`](pipeline.md) (its own §3 is stale on the known lane and pool fallback — this doc is the current source for those two topics). -File:line references are repo-relative to `backend/`, except the `libs/*` -paths, which are already fully qualified (those are separate submodule repos -vendored under `libs/`). All references were checked against `main` at -`0a1d30f`. +References name a **file and a symbol**, never a line number: paths are +repo-relative to `backend/`, except the `libs/*` ones, which are already fully +qualified (those are separate submodule repos vendored under `libs/`). Line +numbers were what this document used to carry, and they were stale within two +weeks of being written — every one of them had drifted by the time anyone +followed it. A symbol survives an edit above it, so grep for the name. ## Legend @@ -74,13 +76,13 @@ lane rides the solver loop's idle cycles rather than owning workers of its own. Everything that reaches a solve passes through one gate stack (`_process_solver_item`) before publication. -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py:358-359` | -| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py:365-366` | -| `FRAME_WORKERS` | 4 (compose sets 6) | `main.py:164`, `docker-compose.yml:54` | -| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py:31,67` | -| `KNOWN_LANE_MODE` default | `binding` | `core/state.py:72-74` | +| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py` | +| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py` | +| `FRAME_WORKERS` | 4 (compose sets 6) | `core/state.py` (`FRAME_WORKERS`), `docker-compose.yml` | +| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py` (`_N_SOLVER_WORKERS`, `_make_solver_pool`) | +| `KNOWN_LANE_MODE` default | `binding` | `core/state.py` (`KNOWN_LANE_MODE`) | --- @@ -89,11 +91,11 @@ own. Everything that reaches a solve passes through one gate stack ```mermaid flowchart TD subgraph producers["Five producers"] - p1["TCP (primary)
tcp_handler.py:326"] - p2["blah2 bridge
blah2_bridge.py:289"] - p3["v1 node HTTP API
node_stream.py:250"] - p4["Legacy HTTP radar routes
routes/radar.py:151,202"] - p5["Startup priming
node_pipeline.py:139"] + p1["TCP (primary)
tcp_handler._enqueue_detection"] + p2["blah2 bridge
blah2_bridge.blah2_bridge_task"] + p3["v1 node HTTP API
node_stream._file_frame"] + p4["Legacy HTTP radar routes
radar.ingest_detections(_bulk)"] + p5["Startup priming
node_pipeline.prime_pipeline"] end p1 --> gA{"Gate A: timestamp present?"} @@ -131,25 +133,25 @@ flowchart TD classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` -The ordering inside `process_one_frame` (`services/frame_processor.py:294`) is +The ordering inside `process_one_frame` (`services/frame_processor.py`) is load-bearing, not incidental: claiming (2.3) runs **before** ADS-B seeding (2.4) so a node-supplied `adsb` field is still distinguishable from a claim, and both run **before** the tracker (2.5) so that, in `binding` mode, a claimed detection never reaches the dark-lane tracker or association at all -— see the ordering comment at `services/frame_processor.py:327-337`. +— see the ordering comment at the head of `process_one_frame`'s claiming step. Frame-level gates (A/B/C on TCP, plus the connected-node check on the v1 API) sit ahead of everything else; nothing downstream sees a frame that failed one of them. -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| Gate A: timestamp required | — | `tcp_handler.py:513-516` | -| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node | `tcp_handler.py:495,527-532` | -| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler.py:536-552` | -| `process_one_frame` entry | — | `services/frame_processor.py:294` | -| Ordering rationale (claim → seed → tracker) | — | `services/frame_processor.py:327-337` | -| Gate 2.10: `n_nodes < 2` skip | — | `services/frame_processor.py:409-426` | -| blah2 poll interval | 1.0 s | `config/constants.py:266` | +| Gate A: timestamp required | — | `tcp_handler._enqueue_detection` | +| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node, counted as `node_frames_rate_limited` | `tcp_handler` (`_NODE_MIN_INTERVAL_S`, `_enqueue_detection`) | +| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler._enqueue_detection` | +| `process_one_frame` entry | — | `services/frame_processor.py` | +| Ordering rationale (claim → seed → tracker) | — | `frame_processor.process_one_frame` | +| Gate 2.10: `n_nodes < 2` skip | — | `frame_processor.process_one_frame` | +| blah2 poll interval | 1.0 s | `config/constants.py` (`BLAH2_POLL_INTERVAL_S`) | --- @@ -205,17 +207,17 @@ also reject — a differential property test in `test_known_claiming.py` failure increments the same `known_claims_visibility_rejects` counter as a gate failure: same event, same meaning, just caught cheaper. -**Mode semantics** (`KNOWN_LANE_MODE`, read once at `core/state.py:72-74`, +**Mode semantics** (`KNOWN_LANE_MODE`, read once in `core/state.py`, default `binding`; an unrecognized value falls back to `shadow`, not to the default — a typo should degrade to the inert mode, not the acting one): | Mode | Claiming | Frame the dark lane sees | Known-lane solver | Publication | |---|---|---|---|---| -| `off` | never runs | untouched | returns 0 immediately (`known_lane.py:391-392`); worker never even calls it (`solver.py:1961`) | none | +| `off` | never runs | untouched | returns 0 immediately (`known_lane.run_known_lane_pass`); worker never even calls it (`solver._run_solver_worker`) | none | | `shadow` | runs, records claims + residuals + counters | untouched | runs: solves, classifies, records accuracy samples | never | -| `binding` | runs | `strip_claimed_detections` removes claimed indices (`frame_processor.py:347`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish | +| `binding` | runs | `strip_claimed_detections` removes claimed indices (called from `frame_processor.process_one_frame`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish | -`strip_claimed_detections` (`services/known_claiming.py:343`) returns a copy +`strip_claimed_detections` (`services/known_claiming.py`) returns a copy with claimed indices removed from `delay`/`doppler`/`snr`/`adsb`; the original frame still feeds the archive and ADS-B extraction (steps 2.11-2.12) unchanged. @@ -224,7 +226,7 @@ unchanged. ```mermaid flowchart TD - arm["Solver worker loop arms known_lane
at thread start (solver.py:1961)"] + arm["Solver worker loop arms known_lane
at thread start (solver._run_solver_worker)"] arm --> drain["After every queue-drain iteration,
call maybe_run_pass"] drain --> gm{"mode == off?"} gm -->|"yes"| ret1["return"]:::inert @@ -261,7 +263,7 @@ flowchart TD gpub -->|"no"| noop["accuracy sample only,
no feed entry"]:::inert ``` -The docstring at `services/tasks/known_lane.py:19-27` calls this the "free +The module docstring of `services/tasks/known_lane.py` calls this the "free solve invariant": the ADS-B fix seeds the initial guess and pins altitude, nothing else — no regularization pulls the solve toward the truth position, so the residual (`err_km`) is a genuine measurement of radar accuracy, not a @@ -269,21 +271,21 @@ circular check. One more intentional-by-omission detail: known-lane measurements carry `snr = 0.0` (claim records have no `snr` key), which the LM's SNR weighting maps to a uniform weight of 1.0. -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `ADSB_SEED_MAX_DR_AGE_S`, `association.py:106`) | -| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `ADSB_SEED_*`, `association.py:98,99`) | -| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py:79` | -| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py:205` | -| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py:89`, applied in `known_claiming.py:_dark_global_projections` | -| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py:73,77` | -| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py:80` | -| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py:85,86` | -| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py:274-275` | -| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py:105` | -| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py:91,99` | -| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py:110` | -| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py:205` | +| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `association.ADSB_SEED_MAX_DR_AGE_S`) | +| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `association.ADSB_SEED_DELAY_GATE_US` / `_DOPPLER_GATE_HZ`) | +| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py` | +| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py` | +| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py`, applied in `known_claiming._dark_global_projections` | +| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py` | +| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py` | +| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py` | +| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py` | +| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py` | +| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py` | +| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py` | +| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py` | --- @@ -291,7 +293,7 @@ LM's SNR weighting maps to a uniform weight of 1.0. ```mermaid flowchart TD - frame["pipeline.process_frame
passive_radar.py:672"] + frame["PassiveRadarPipeline.process_frame
pipeline/passive_radar.py"] frame --> tracker["retina_tracker
Kalman + GNN"] tracker --> geo["_run_geolocation per track
with new data"] @@ -345,7 +347,7 @@ flowchart TD classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` -`compute_overlap_zone` (`libs/retina-analytics/.../association.py:578`) +`compute_overlap_zone` (`libs/retina-analytics/.../association.py`) underlies both the confirmed-track association round and the overlap-grid cache: it fast-prunes non-overlapping node pairs by receiver separation, grids the shared coverage at `ASSOC_GRID_STEP_KM` on six altitude layers that @@ -353,22 +355,22 @@ must match the solver's `_SOLVER_ALT_LAYERS_KM`, and requires each grid column to fall in **both** beams (`_point_in_beam`, FOV-aware only when `FOV_MODE=active`). -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py:194` | -| Single-node min detections | 3 | `passive_radar.py:356-361` | -| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py:59` | -| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py:220` | -| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py:22` | -| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py:23` | -| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py:31`, `association.py:1043` | -| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.py:883` | -| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.py:884` | -| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py:166` | -| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py:57-58` | -| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.py:2160` | -| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py:21` | -| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py:111` | +| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py` (applied as `_GEO_INTERVAL_S` in `_run_geolocation`) | +| Single-node min detections | 3 | `pipeline/passive_radar.py` (`_geolocate_track_event`, `min_det`) | +| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py` | +| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py` | +| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py` | +| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py` | +| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py`, `association.py` | +| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.compute_overlap_zone` (default arg) | +| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.compute_overlap_zone` (default arg) | +| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py` | +| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py` | +| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.InterNodeAssociator.format_track_pairs_for_solver` (local) | +| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py` | +| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py` | --- @@ -376,7 +378,7 @@ column to fall in **both** beams (`_point_in_beam`, FOV-aware only when The centerpiece: every candidate from either lane, once dequeued from `solver_queue`, runs through `_process_solver_item` -(`services/tasks/solver.py:1344`) as a strict, ordered chain. A failure at +(`services/tasks/solver.py`) as a strict, ordered chain. A failure at any gate stops the chain, bumps a counter, and (from 6.5 onward) writes a named record to solve history. @@ -430,14 +432,14 @@ flowchart TD ``` `SOLVER_CONSENSUS_MODE` is `off` in production (see the mode-flag table in -[`architecture.md:94-110`](architecture.md#feature-gates)), so in practice +[`architecture.md`](architecture.md#feature-gates)), so in practice this sub-branch never reaches `active` outside staging. ### The LM itself -`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py:518`, +`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py`, invoked through the process pool via `_pool_solve_multinode` -(`services/tasks/solver.py:1915`). +(`services/tasks/solver.py`). ```mermaid flowchart TD @@ -459,19 +461,19 @@ flowchart TD classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 ``` -| Constant | Value | File:line | +| Constant | Value | Defined in | |---|---|---| -| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py:701` | -| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py:744` | -| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py:160-162` | -| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py:132` | -| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py:173` | -| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py:205` | -| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py:56` | -| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py:807` | -| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py:154` | -| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py:51,52` | -| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py:57,63` | +| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py` | +| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py` (`_SOLVER_RESOLVE_INTERVAL_S`) | +| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py` | +| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py` (`_SOLVER_RMS_DELAY_MAX_US`) | +| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py` | +| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py` | +| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py` | +| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py` | +| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py` | +| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py` | +| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py` | --- @@ -525,26 +527,61 @@ flowchart TD | Value | Set at | Meaning | |---|---|---| -| `multinode_solve` | `aircraft_feed.py:132` | published multi-node solve | -| `solver_adsb_seed` | `track_gates.py:330` | single-node LM with fresh ADS-B fix | -| `solver_single_node` | `track_gates.py:330` | single-node LM, no ADS-B | -| `single_node_ellipse_arc` | `track_gates.py:378` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint | -| `adsb_single_node` | `aircraft_feed.py:_claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` | -| `known_lane_truth_match` / `known_lane_ghost` | `known_lane.py:260` | accuracy-sample-only, not a feed entry | - -| Constant | Value | File:line | +| `multinode_solve` | `aircraft_feed.multinode_to_aircraft` | published multi-node solve | +| `solver_adsb_seed` | `track_gates.track_entry` | single-node LM with fresh ADS-B fix | +| `solver_single_node` | `track_gates.track_entry` | single-node LM, no ADS-B | +| `single_node_ellipse_arc` | `track_gates.track_entry` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint | +| `adsb_single_node` | `aircraft_feed._claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` | +| `known_lane_truth_match` / `known_lane_ghost` | `known_lane._record_accuracy` | accuracy-sample-only, not a feed entry | + +| Constant | Value | Defined in | |---|---|---| | `_MN_ASSOC_MAX_DIST_KM` / `_MN_ASSOC_MAX_AGE_S` (identity step 2/3) | 6.0 km / 60.0 s | `services/tasks/solver.py` | | `_MN_ASSOC_DRIFT_KM_PER_S` / `_MN_ASSOC_MAX_DIST_CAP_KM` (step 3 only — the gate grows with the matched entry's age) | 0.13 km/s / 12.0 km | `services/tasks/solver.py` | | Supersession gate (`_supersession_match`) — the same age-scaled `_mn_assoc_gate_km` and `_MN_ASSOC_MAX_AGE_S` as step 3, applied to the solve's RAW position | 6.0 + 0.13·dt km, cap 12.0 / 60.0 s | `services/tasks/solver.py` | -| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py:77` | -| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py:63` | -| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py:66` | -| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py:37-43` | -| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py:131-139` | -| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py:49-50` | -| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py:167` | -| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py:206,213` | +| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py` | +| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py` | +| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py` | +| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py` | +| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py` | +| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py` (`_DEDUP_PROXIMITY_KM`, `_DEDUP_ALT_GATE_FT`) | +| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py` | +| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py` | + +--- + +## 7. Reading the pipeline from outside + +Three endpoints answer questions about the two lanes, and each has a shape +worth knowing before it is trusted. + +**`/api/test/mlat-history`** dumps solve records. Both lanes write their own +deque (`state.mlat_solve_history`, `state.mlat_solve_history_known`) and every +reader merges them. `?lane=dark|known|adsb|all` narrows the answer; +`?limit=` (default 1 000, max 5 000) is applied **per lane**, so a known-lane +burst can never push dark records out of the response — the flat cap that +preceded it left a 30 min request holding only the newest ~6 min of dark +records, which reads exactly like a quiet dark lane. `lane_counts` is +reported pre-cap so a truncated `records` list is legible. +`?kind=resolve_skips` dumps a different store entirely — see below. + +**`/api/test/solver-stats`** is the Solver Report panel's source. Its funnel, +error percentiles, ghosts, fragmentation, `contamination` and `resolve_skips` +are all the DARK lane; `lane_split` gives the per-lane record counts and +`known_lane` that lane's own numbers. + +| Block | Says | Watch for | +|---|---|---| +| `contamination` | Of the dark records that matched ground truth, how many carried a node that could not see the aircraft (`foreign_node_ids` on the record; verdict is the associator's own `_point_in_beam`, the same gate known-lane claiming uses) | `pct` is the live version of the offline ~60 % the cluster-splitting work exists to move. Records with no GT match, or no registered geometry for any contributing node, are **out of the denominator** — abstention, not innocence | +| `resolve_skips` | Candidates the re-solve suppression refused in this window, from `state.solver_resolve_skips_recent`, with the claims that blocked each one | `attempts_ratio` is all-lane skips over DARK attempts (live baseline ~2.4). The deque holds 500 entries against ~50 skips/min, so read `window_effective_minutes` before reading `total` as a window count | +| `counters.resolve_skips_dark` | Dark share of the since-boot skip counter | — | +| `counters.node_frames_rate_limited` | Frames `NODE_FRAME_MIN_INTERVAL_S` refused before the tracker saw them (Gate B in §2) | Not the same event as `/api/admin/metrics`' `frames_dropped`, which is `frame_queue` saturation and normally reads zero | + +A skip is deliberately **not** a solve-history record: skips outrun dark +records roughly two to one on the live fleet, so writing them into +`mlat_solve_history` would evict exactly the solves an investigation needs. +They are also not counted as attempts or rejects — a skipped candidate never +reached a solve. --- @@ -566,15 +603,15 @@ flowchart TD (`fragmentation`) and `superseded_keys` / `superseded_blocked` on each published `mlat_solve_history` record are how this is watched. - **Node-trust residuals are measure-only.** `node_bias.py` computes them but - nothing in the solver consumes them yet (`node_bias.py:33-40` docstring). + nothing in the solver consumes them yet (`node_bias.py` module docstring). - **`docs/pipeline.md` §3 is stale.** It predates the known lane and the process-pool inline fallback; this doc supersedes it for both topics. - **The bottom-up doppler gate is inert.** `doppler_gate_hz` in the dark lane's coarse pairing step is defined but the grid gate is delay-only in - practice (`libs/retina-analytics/.../association.py:884`). + practice (`association.compute_overlap_zone`'s `doppler_gate_hz`). - **Production runs with every mode flag off** except `KNOWN_LANE_MODE`, which is `binding` everywhere by code default and is set in no environment's `.env`. The in-repo statement of what each environment sets is - [`architecture.md:94-110`](architecture.md#feature-gates); the actual + [`architecture.md`](architecture.md#feature-gates); the actual values live in the gitignored `backend/.env` on each host, not in this repo. From 3a69d2dd0d1ecb95b3718a6c3cc3f966c5867097 Mon Sep 17 00:00:00 2001 From: jehanazad Date: Sat, 5 Sep 2026 07:08:01 +0000 Subject: [PATCH 06/15] Claim a re-solve slot on publication, not on admission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suppression rule reads "this aircraft is already on the map at this width, at every track it is built from". _claim_resolve_slot recorded the claim BEFORE the solve, so a candidate that never reached the map still made that statement — and never released it. Two consequences, both live: * A rejected candidate blacked out its own identical twin for the full 12 s. 24% of dark attempts are rejected, and the retry that would have published was suppressed by the failure. * Tracker track ids are shared across the association candidates of DIFFERENT aircraft (74 of 178 ids in a 6 min window appeared in solves of more than one ground-truth aircraft — the finding that forced _supersession_match's spatial guard in #290). So a contaminated superset that the gates sank also suppressed the clean subsets behind it, including its neighbour's only candidate. Measured on the test droplet: ~1,537 skips against 646 dark attempts per 30 min. The mechanism refused more than twice as many candidates as it solved, for aircraft it had put nowhere. _claim_resolve_slot splits into a pure _resolve_slot_covered (read-only, run before the solve, also returning the blocking claims for the skip record) and _record_resolve_slot, called only on the publish path with the POST-TRIM survivors — result["source_track_ids"], which _filter_s_in_to_nodes rebuilds from the surviving track_ids_by_node. A trimmed node's track is deliberately left unclaimed: it contributed nothing to the published position and was probably another aircraft's, so claiming it would suppress that aircraft on the strength of a measurement this solve threw away. The rule itself is unchanged: every track covered at no fewer nodes within _SOLVER_RESOLVE_INTERVAL_S, widest claim wins, same pruning sweep. No negative claim for rejects — measure first. The cost is that the check no longer claims under the same lock, so two workers can both solve duplicates that arrived together. That is one extra solve, arbitrated downstream by keying and supersession, against the starvation above. Co-Authored-By: Claude Fable 5.1 --- backend/core/state.py | 15 +-- backend/routes/test.py | 2 +- backend/services/tasks/solver.py | 108 ++++++++++++------- backend/tests/test_mlat_history.py | 7 +- backend/tests/test_solver_trimming.py | 73 +++++++++++++ backend/tests/test_solver_worker.py | 150 +++++++++++++++++++++----- docs/solverflow.md | 28 ++++- 7 files changed, 306 insertions(+), 77 deletions(-) diff --git a/backend/core/state.py b/backend/core/state.py index 246907c9..54c14ea4 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -544,12 +544,15 @@ def _adsb_for_seeding() -> dict[str, dict]: solver_stale_drops: int = 0 # Candidates dequeued and skipped because every single-node track they carry -# was already solved within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes (see -# solver.py's _claim_resolve_slot). Association is per-node and rate-limited -# per node, so one aircraft arrives as one candidate per node that can see it; -# this counts the copies that were never worth solving. High against -# solver_successes is normal and is the mechanism working — it is -# solver_stale_drops that means work was lost. +# was already PUBLISHED within _SOLVER_RESOLVE_INTERVAL_S at no fewer nodes +# (see solver.py's _resolve_slot_covered). Association is per-node and +# rate-limited per node, so one aircraft arrives as one candidate per node +# that can see it; this counts the copies that were never worth solving. High +# against solver_successes is normal and is the mechanism working — it is +# solver_stale_drops that means work was lost. Read it against +# solver_successes, not against attempts: while the claim was taken on +# ADMISSION rather than on publication, a rejected candidate blacked out every +# later one sharing a track id and this counter ran at ~2.4x attempts. solver_resolve_skips: int = 0 # The dark-lane share of the counter above, split out because the two lanes diff --git a/backend/routes/test.py b/backend/routes/test.py index 5909ff9d..c9431cd2 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -210,7 +210,7 @@ def _build_dashboard_data() -> bytes: # not the queue size. "stale_drops": state.solver_stale_drops, # Duplicate candidates for an aircraft already solved this - # window (see solver.py's _claim_resolve_slot). Read it + # window (see solver.py's _resolve_slot_covered). Read it # against stale_drops: skips are work correctly not done, # stale drops are work lost. "resolve_skips": state.solver_resolve_skips, diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 5faf7bc5..dbdf3ce8 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -986,6 +986,34 @@ def _supersession_match( # Sized against the map, not the association cadence: multinode_tracks expire # at 60 s, so refreshing an aircraft every 12 s leaves four solves' worth of # margin. 0 disables the suppression entirely. +# +# The claim is recorded ON PUBLICATION, not on admission, and from the +# POST-TRIM survivors. Claiming on admission made a candidate that never +# reached the map suppress every later candidate sharing any of its track ids +# for the full window — including other aircraft's, since tracker track ids +# are shared across the association candidates of different aircraft (74 of +# 178 ids in a 6 min live window appeared in solves of more than one +# ground-truth aircraft; the same finding that forced _supersession_match's +# spatial guard). A rejected candidate, or a contaminated superset that the +# gates sank, therefore blacked out the clean subsets behind it for 12 s and +# nothing was refreshed at all. Live that cost ~1 537 skips per 646 dark +# attempts per 30 min — more candidates suppressed than solved, by a factor +# of two. The rule this suppression is FOR is "an aircraft already on the map +# at this width does not need re-solving yet", and only a publication puts an +# aircraft on the map. +# +# Two consequences, both accepted deliberately: +# * the check no longer claims under the same lock, so two workers can now +# both solve duplicates of one aircraft that arrived together. The pair +# costs one extra solve and is resolved downstream by keying and +# supersession, which already handle exactly this; the alternative is the +# starvation above. +# * trimmed nodes' track ids are NOT claimed (_filter_s_in_to_nodes rebuilds +# track_ids from the surviving track_ids_by_node, so result's +# source_track_ids are the survivors). A node dropped for a bad residual +# was probably another aircraft's — claiming its track would suppress that +# aircraft's own candidate on the strength of a measurement this solve +# threw away. _SOLVER_RESOLVE_INTERVAL_S = float(os.getenv("SOLVER_RESOLVE_INTERVAL_S", "12")) _RECENT_SOLVES: dict[str, tuple[float, int]] = {} # track_id → (solved_at, n_nodes) _RECENT_SOLVES_LOCK = threading.Lock() @@ -1003,61 +1031,60 @@ def _sweep_recent_solves(now_s: float) -> None: del _RECENT_SOLVES[tid] -def _claim_resolve_slot(s_in, now_s: float) -> bool: - """False when this candidate re-solves tracks another candidate just took. +def _resolve_slot_covered(s_in, now_s: float) -> tuple[bool, list[dict]]: + """Is every track this candidate carries already ON THE MAP at this width? - Records the claim as a side effect, under one lock with the test, so two - workers cannot both admit the same aircraft's duplicates. An input with no - track provenance (detection-level, or an anchored input carrying none) is - always admitted — there is nothing to match it against. + Pure: it reads the claims and mutates nothing, so a candidate that is + admitted here and then rejected by the gate stack leaves no trace. The + claim is made afterwards by _record_resolve_slot, from the publish path + only — see the block comment above for why, and for what the loss of + atomic test-and-claim costs. + + Returns (covered, blocking). ``blocking`` is the claims that covered it, + for the skip record; it is empty whenever ``covered`` is False. An input + with no track provenance (detection-level, or an anchored input carrying + none) is never covered — there is nothing to match it against. """ if _SOLVER_RESOLVE_INTERVAL_S <= 0 or not isinstance(s_in, dict): - return True + return False, [] track_ids = s_in.get("track_ids") if not track_ids: - return True + return False, [] n_nodes = int(s_in.get("n_nodes") or 0) cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S + blocking: list[dict] = [] with _RECENT_SOLVES_LOCK: - covered = True for tid in track_ids: held = _RECENT_SOLVES.get(tid) if held is None or held[0] <= cutoff or held[1] < n_nodes: - covered = False - break - if covered: - return False - for tid in track_ids: - held = _RECENT_SOLVES.get(tid) - # Keep the widest claim of the window: a narrow candidate admitted - # after a wide one must not lower the bar the next copy is tested - # against. - held_nodes = held[1] if held is not None and held[0] > cutoff else 0 - _RECENT_SOLVES[tid] = (now_s, max(n_nodes, held_nodes)) - _sweep_recent_solves(now_s) - return True + return False, [] + blocking.append({"track_id": tid, "held_ts": round(held[0], 3), "held_n": held[1]}) + return True, blocking -def _resolve_slot_blockers(track_ids, now_s: float) -> list[dict]: - """The live claims covering ``track_ids``, for a skip record. +def _record_resolve_slot(track_ids, n_nodes: int, now_s: float) -> None: + """Record that ``track_ids`` are covered by a PUBLISHED solve at n_nodes. - Read-only, and taken after the refusal rather than during it: the check - itself must stay one atomic test-and-claim, and a skip is rare enough - (relative to the queue drain rate) that a second lock acquisition on that - path costs nothing. Any claim that moves between the two is a claim the - diagnosis would have wanted to name anyway. + Called from the publish path alone, with the post-trim survivors + (``result["source_track_ids"]``). Nothing else may call it: a claim is a + statement that this aircraft is on the map, and a rejected solve puts + nothing there. """ + if _SOLVER_RESOLVE_INTERVAL_S <= 0 or not track_ids: + return + n_nodes = int(n_nodes or 0) cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S - out: list[dict] = [] with _RECENT_SOLVES_LOCK: for tid in track_ids: held = _RECENT_SOLVES.get(tid) - if held is not None and held[0] > cutoff: - out.append({"track_id": tid, "held_ts": round(held[0], 3), "held_n": held[1]}) - return out + # Keep the widest claim of the window: a narrow publish after a + # wide one must not lower the bar the next copy is tested against. + held_nodes = held[1] if held is not None and held[0] > cutoff else 0 + _RECENT_SOLVES[tid] = (now_s, max(n_nodes, held_nodes)) + _sweep_recent_solves(now_s) -def _record_resolve_skip(s_in, now_s: float, blocking: list[dict] | None = None) -> None: +def _record_resolve_skip(s_in, now_s: float, blocking: list[dict]) -> None: """Count and remember one resolve-slot refusal. The counter alone could not answer the question the suppression rule @@ -1089,7 +1116,7 @@ def _record_resolve_skip(s_in, now_s: float, blocking: list[dict] | None = None) "lane": "dark" if dark else "adsb", "track_ids": track_ids, "n_nodes": int(s.get("n_nodes") or 0), - "blocking": _resolve_slot_blockers(track_ids, now_s) if blocking is None else blocking, + "blocking": blocking, "guess_lat": round(float(ig["lat"]), 6) if ig.get("lat") else None, "guess_lon": round(float(ig["lon"]), 6) if ig.get("lon") else None, } @@ -1850,8 +1877,9 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus # and a copy that queued before its twin was solved can only be recognised # once it reaches a worker. _now_s = time.time() - if not _claim_resolve_slot(s_in, _now_s): - _record_resolve_skip(s_in, _now_s) + _covered, _blocking = _resolve_slot_covered(s_in, _now_s) + if _covered: + _record_resolve_skip(s_in, _now_s, _blocking) return None n_nodes = s_in.get("n_nodes", 0) if isinstance(s_in, dict) else 0 consensus_meta: dict | None = None @@ -2417,6 +2445,12 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus archive_record = dict(result) archive_record["solve_ts_ms"] = int(time.time() * 1000) state.track_archive_buffer.append(archive_record) + # The re-solve claim, taken here and nowhere else: this aircraft is now + # on the map at this width, which is the only thing that makes a + # duplicate not worth solving. Survivors only — source_track_ids is + # rebuilt from the post-trim node set. Outside _MN_TRACKS_LOCK on + # purpose, so _RECENT_SOLVES_LOCK is never nested inside it. + _record_resolve_slot(result.get("source_track_ids"), result.get("n_nodes"), time.time()) _record_solve_history( "published", s_in, diff --git a/backend/tests/test_mlat_history.py b/backend/tests/test_mlat_history.py index e8e91bbe..a406d1b4 100644 --- a/backend/tests/test_mlat_history.py +++ b/backend/tests/test_mlat_history.py @@ -1031,9 +1031,10 @@ def _client(self): def _skip(self, track_ids=("a1", "b1"), n_nodes=3, **s_in): now = time.time() s = dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids), **s_in) - solver_mod._claim_resolve_slot(s, now) - assert solver_mod._claim_resolve_slot(dict(s), now) is False - solver_mod._record_resolve_skip(dict(s), now) + solver_mod._record_resolve_slot(list(track_ids), n_nodes, now) + covered, blocking = solver_mod._resolve_slot_covered(dict(s), now) + assert covered is True + solver_mod._record_resolve_skip(dict(s), now, blocking) def test_skip_records_the_blocking_claim(self): self._skip() diff --git a/backend/tests/test_solver_trimming.py b/backend/tests/test_solver_trimming.py index 93504a5a..0f529848 100644 --- a/backend/tests/test_solver_trimming.py +++ b/backend/tests/test_solver_trimming.py @@ -827,3 +827,76 @@ def solve_fn(_s_in, _cfgs): assert state.fov_shadow_agree == 1 assert state.fov_shadow_would_pass == 0 assert state.fov_shadow_would_reject == 0 + + +class TestTrimmedTracksAreNotClaimed(_TrimmingTestBase): + """A trimmed node's tracks must not take a re-solve claim. + + The claim says "this aircraft is on the map at this width". A node + dropped for a bad residual contributed nothing to the published position + and its track was probably a different aircraft's — claiming it would + suppress that aircraft's own candidate on the strength of a measurement + this solve threw away. + """ + + _FULL = ["n1", "n2", "n3", "n4", "bad"] + _TRIM = ["n1", "n2", "n3", "n4"] + + def test_the_dropped_nodes_track_is_left_unclaimed(self): + table = { + frozenset(self._FULL): _stub_result( + self._FULL, + rms_delay=8.0, + per_node={"n1": 0.5, "n2": 0.5, "n3": 0.5, "n4": 0.5, "bad": 12.0}, + ), + frozenset(self._TRIM): _stub_result( + self._TRIM, + rms_delay=0.8, + per_node={"n1": 0.3, "n2": 0.3, "n3": 0.3, "n4": 0.3}, + ), + } + s_in = _s_in( + self._FULL, + track_ids=["t1", "t2", "t3", "t4", "tbad"], + track_ids_by_node={ + "n1": ["t1"], + "n2": ["t2"], + "n3": ["t3"], + "n4": ["t4"], + "bad": ["tbad"], + }, + ) + result = self._run(s_in, _stub_solve_fn(table)) + assert result is not None and result["success"] + assert result["source_track_ids"] == ["t1", "t2", "t3", "t4"] + assert set(solver_mod._RECENT_SOLVES) == {"t1", "t2", "t3", "t4"} + + def test_a_candidate_built_on_the_dropped_track_still_runs(self): + """The other half of the same claim: whoever "tbad" really belongs to + keeps its slot.""" + table = { + frozenset(self._FULL): _stub_result( + self._FULL, + rms_delay=8.0, + per_node={"n1": 0.5, "n2": 0.5, "n3": 0.5, "n4": 0.5, "bad": 12.0}, + ), + frozenset(self._TRIM): _stub_result( + self._TRIM, + rms_delay=0.8, + per_node={"n1": 0.3, "n2": 0.3, "n3": 0.3, "n4": 0.3}, + ), + } + s_in = _s_in( + self._FULL, + track_ids=["t1", "t2", "t3", "t4", "tbad"], + track_ids_by_node={ + "n1": ["t1"], + "n2": ["t2"], + "n3": ["t3"], + "n4": ["t4"], + "bad": ["tbad"], + }, + ) + self._run(s_in, _stub_solve_fn(table)) + neighbour = {"n_nodes": 2, "track_ids": ["tbad", "tother"]} + assert solver_mod._resolve_slot_covered(neighbour, time.time())[0] is False diff --git a/backend/tests/test_solver_worker.py b/backend/tests/test_solver_worker.py index a4a6c8e9..6e3a2035 100644 --- a/backend/tests/test_solver_worker.py +++ b/backend/tests/test_solver_worker.py @@ -335,56 +335,153 @@ class TestResolveSuppression: emits its own candidate for it inside one association window. Solving all of them starves aircraft that have no solve at all — the queue ages out behind work whose result is superseded the moment it lands. + + The claim that suppresses a duplicate is taken on PUBLICATION + (_record_resolve_slot), not on admission: the rule is "this aircraft is + already on the map at this width", and only a publish puts it there. + _resolve_slot_covered is the pure test run before the solve. """ def _s_in(self, track_ids, n_nodes=2): return dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids)) - def test_a_second_copy_of_the_same_tracks_is_skipped(self): + def _covered(self, track_ids, n_nodes=2, now=None): + return solver_mod._resolve_slot_covered(self._s_in(track_ids, n_nodes), now or time.time())[0] + + def _publish(self, track_ids, n_nodes=2, now=None): + solver_mod._record_resolve_slot(list(track_ids), n_nodes, now or time.time()) + + def test_a_second_copy_of_a_published_candidate_is_skipped(self): + now = time.time() + assert self._covered(["a1", "b1"], now=now) is False + self._publish(["a1", "b1"], now=now) + assert self._covered(["a1", "b1"], now=now) is True + + def test_the_check_alone_claims_nothing(self): + """The whole point of the split: a candidate that is admitted and then + rejected by the gate stack must leave no trace.""" now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is False + assert self._covered(["a1", "b1"], now=now) is False + assert self._covered(["a1", "b1"], now=now) is False - def test_a_candidate_carrying_an_unsolved_track_runs(self): + def test_a_candidate_carrying_an_unpublished_track_runs(self): """An aircraft entering coverage must never be suppressed.""" now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b2"]), now) is True + self._publish(["a1", "b1"], now=now) + assert self._covered(["a1", "b2"], now=now) is False def test_a_wider_view_of_the_same_tracks_runs(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=2), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True + self._publish(["a1", "b1"], n_nodes=2, now=now) + assert self._covered(["a1", "b1"], n_nodes=5, now=now) is False def test_a_narrower_view_after_a_wider_one_is_skipped(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=2), now) is False + self._publish(["a1", "b1"], n_nodes=5, now=now) + assert self._covered(["a1", "b1"], n_nodes=2, now=now) is True - def test_a_narrow_admission_does_not_lower_the_bar(self): + def test_a_narrow_publish_does_not_lower_the_bar(self): """The window holds the widest claim, not the most recent one.""" now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=5), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b2"], n_nodes=2), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"], n_nodes=3), now) is False + self._publish(["a1", "b1"], n_nodes=5, now=now) + self._publish(["a1", "b2"], n_nodes=2, now=now) + assert self._covered(["a1", "b1"], n_nodes=3, now=now) is True def test_claims_expire(self): now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True + self._publish(["a1", "b1"], now=now) later = now + solver_mod._SOLVER_RESOLVE_INTERVAL_S + 1.0 - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), later) is True + assert self._covered(["a1", "b1"], now=later) is False def test_an_input_without_track_provenance_always_runs(self): """Detection-level inputs carry no track ids — nothing to match on.""" now = time.time() - assert solver_mod._claim_resolve_slot({"n_nodes": 2}, now) is True - assert solver_mod._claim_resolve_slot({"n_nodes": 2}, now) is True + assert solver_mod._resolve_slot_covered({"n_nodes": 2}, now)[0] is False + solver_mod._record_resolve_slot(None, 2, now) + assert solver_mod._resolve_slot_covered({"n_nodes": 2}, now)[0] is False def test_zero_interval_disables_suppression(self, monkeypatch): monkeypatch.setattr(solver_mod, "_SOLVER_RESOLVE_INTERVAL_S", 0.0) now = time.time() - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True - assert solver_mod._claim_resolve_slot(self._s_in(["a1", "b1"]), now) is True + self._publish(["a1", "b1"], now=now) + assert self._covered(["a1", "b1"], now=now) is False + + def test_the_check_names_every_blocking_claim(self): + now = time.time() + self._publish(["a1", "b1"], n_nodes=4, now=now) + covered, blocking = solver_mod._resolve_slot_covered(self._s_in(["a1", "b1"], n_nodes=3), now) + assert covered is True + assert {b["track_id"]: b["held_n"] for b in blocking} == {"a1": 4, "b1": 4} + + def test_an_admitted_candidate_reports_no_blockers(self): + covered, blocking = solver_mod._resolve_slot_covered(self._s_in(["a1", "b1"]), time.time()) + assert (covered, blocking) == (False, []) + + def _solve_fn(self, calls, rms_delay=0.5, lat=37.5, lon=-122.1): + def fn(s_in, cfgs): + calls.append(s_in) + return { + "success": True, + "lat": lat, + "lon": lon, + "alt_m": 9000.0, + "rms_delay": rms_delay, + "rms_doppler": 5.0, + "timestamp_ms": int(time.time() * 1000), + "contributing_node_ids": ["n1", "n2"], + "n_nodes": s_in.get("n_nodes", 2), + } + + return fn + + def test_a_rejected_candidate_does_not_block_an_identical_twin(self, monkeypatch): + """The bug this split exists to fix. A candidate the gate stack sank + put nothing on the map, so the next copy of the same aircraft is its + first real chance — and used to be blacked out for the full 12 s.""" + _reset_state() + monkeypatch.setattr(state, "node_analytics", _StubAnalytics()) + calls: list = [] + s_in = self._s_in(["a1", "b1"]) + + # rms_delay past the gate: solves, then rejected, publishes nothing. + solver_mod._process_solver_item((dict(s_in), {}, time.time()), self._solve_fn(calls, rms_delay=10.0)) + assert state.solver_fail_rms_delay == 1 + assert not state.multinode_tracks + + solver_mod._process_solver_item((dict(s_in), {}, time.time()), self._solve_fn(calls)) + assert len(calls) == 2, "the twin must not be suppressed by a reject" + assert state.multinode_tracks + assert state.solver_resolve_skips == 0 + + def test_a_subset_for_another_aircraft_survives_a_rejected_superset(self, monkeypatch): + """Tracker track ids are shared across the candidates of DIFFERENT + aircraft, so a contaminated superset that the gates sank used to take + every clean subset behind it down with it — including its neighbour's + only candidate.""" + _reset_state() + monkeypatch.setattr(state, "node_analytics", _StubAnalytics()) + calls: list = [] + superset = self._s_in(["a1", "b1", "c1"], n_nodes=3) + solver_mod._process_solver_item((superset, {}, time.time()), self._solve_fn(calls, rms_delay=10.0)) + assert not state.multinode_tracks + + # The neighbour: fewer nodes, sharing one contaminated track id. + subset = self._s_in(["a1", "b1"], n_nodes=2) + solver_mod._process_solver_item((subset, {}, time.time()), self._solve_fn(calls)) + assert len(calls) == 2 + assert state.multinode_tracks + + def test_only_the_published_width_is_claimed(self, monkeypatch): + """A publish claims at the width it published, so a later narrower + copy is suppressed and a wider one still runs.""" + _reset_state() + monkeypatch.setattr(state, "node_analytics", _StubAnalytics()) + calls: list = [] + solver_mod._process_solver_item((self._s_in(["a1", "b1"], n_nodes=3), {}, time.time()), self._solve_fn(calls)) + assert state.multinode_tracks + now = time.time() + assert self._covered(["a1", "b1"], n_nodes=2, now=now) is True + assert self._covered(["a1", "b1"], n_nodes=4, now=now) is False def test_a_skipped_item_never_reaches_the_solver(self, monkeypatch): _reset_state() @@ -408,6 +505,8 @@ def solve_fn(s_in, cfgs): solver_mod._process_solver_item((s_in, {}, time.time()), solve_fn) assert len(solve_calls) == 1 assert state.solver_successes == 1 + # The first item PUBLISHED, which is what makes the second redundant. + assert state.multinode_tracks assert solver_mod._process_solver_item((dict(s_in), {}, time.time()), solve_fn) is None assert len(solve_calls) == 1, "the duplicate must not be solved" @@ -425,9 +524,10 @@ def test_a_skip_is_recorded_with_the_claim_that_blocked_it(self): state.solver_resolve_skips_recent.clear() now = time.time() s_in = dict(self._s_in(["a1", "b1"], n_nodes=4), initial_guess={"lat": 35.0, "lon": -82.0}) - assert solver_mod._claim_resolve_slot(dict(s_in), now) is True - assert solver_mod._claim_resolve_slot(dict(s_in), now) is False - solver_mod._record_resolve_skip(dict(s_in), now) + solver_mod._record_resolve_slot(["a1", "b1"], 4, now) + covered, blocking = solver_mod._resolve_slot_covered(dict(s_in), now) + assert covered is True + solver_mod._record_resolve_skip(dict(s_in), now, blocking) assert state.solver_resolve_skips == 1 assert state.solver_resolve_skips_dark == 1 @@ -444,7 +544,7 @@ def test_a_tagged_candidate_is_counted_but_not_as_dark(self): state.solver_resolve_skips_recent.clear() now = time.time() s_in = dict(self._s_in(["a1"], n_nodes=3), adsb_hex="abc123") - solver_mod._record_resolve_skip(s_in, now) + solver_mod._record_resolve_skip(s_in, now, []) assert state.solver_resolve_skips == 1 assert state.solver_resolve_skips_dark == 0 assert state.solver_resolve_skips_recent[0]["lane"] == "adsb" @@ -455,7 +555,7 @@ def test_skips_never_enter_the_solve_history(self): _reset_state() state.mlat_solve_history.clear() s_in = self._s_in(["a1", "b1"]) - solver_mod._record_resolve_skip(s_in, time.time()) + solver_mod._record_resolve_skip(s_in, time.time(), []) assert not state.mlat_solve_history assert not state.mlat_solve_history_known diff --git a/docs/solverflow.md b/docs/solverflow.md index c80c4673..d22fc152 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -382,13 +382,30 @@ The centerpiece: every candidate from either lane, once dequeued from any gate stops the chain, bumps a counter, and (from 6.5 onward) writes a named record to solve history. +**6.2 claims on publication, not on admission.** The suppression rule is +"this aircraft is already on the map at this width, at every track it is +built from" — so `_resolve_slot_covered` only *reads* the claims, and +`_record_resolve_slot` takes them from the publish path, with the +**post-trim survivors** (`result["source_track_ids"]`, rebuilt from the +surviving `track_ids_by_node`). Claiming on admission instead meant a +candidate that never reached the map still blacked out every later candidate +sharing any of its track ids for the full 12 s — including *other aircraft's*, +since tracker track ids are shared across the association candidates of +different aircraft (the same finding behind `_supersession_match`'s spatial +guard; see Caveats). Live that ran at ~1 537 skips per 646 dark attempts per +30 min: more candidates suppressed than solved, by a factor of two. The price +of the split is that the check no longer claims under the same lock, so two +workers can now both solve duplicates that arrived together; that costs one +extra solve and is resolved downstream by keying and supersession, which +handle exactly this case already. + ```mermaid flowchart TD deq["Dequeue (s_in, node_cfgs, enqueued_at)"] deq --> g61{"6.1 Staleness
age_s > _SOLVER_MAX_QUEUE_AGE_S 45.0s?"} g61 -->|"yes"| f61["solver_stale_drops
(no history record)"]:::inert - g61 -->|"no"| g62{"6.2 Re-solve suppression
_claim_resolve_slot False?"} - g62 -->|"yes"| f62["solver_resolve_skips"]:::inert + g61 -->|"no"| g62{"6.2 Re-solve suppression
_resolve_slot_covered (pure)?"} + g62 -->|"yes"| f62["solver_resolve_skips (+_dark)
+ skip record with blockers"]:::inert g62 -->|"no"| g63["6.3 Solve dispatch:
no guess -> bare solve_fn;
n>=3 -> consensus? then
_solve_best_altitude (sweep);
n=2 -> _solve_best_altitude_n2
(single altitude)"] g63 -->|"exception"| f63["solver_failures +
solver_fail_exception,
result=None"]:::inert g63 --> g64{"6.4 Trim & resolve (recovery):
guess AND n>=4 AND
rms_delay > 3.0us?"} @@ -464,7 +481,7 @@ flowchart TD | Constant | Value | Defined in | |---|---|---| | `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py` | -| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py` (`_SOLVER_RESOLVE_INTERVAL_S`) | +| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py` (`_SOLVER_RESOLVE_INTERVAL_S`, `_resolve_slot_covered`, `_record_resolve_slot`) | | `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py` | | `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py` (`_SOLVER_RMS_DELAY_MAX_US`) | | `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py` | @@ -509,7 +526,8 @@ flowchart TD popped --> store["state.multinode_tracks[key] = result"] blocked --> store store --> archive["track-archive buffer append"] - archive --> histpub["_record_solve_history: published"] + archive --> claimslot["_record_resolve_slot:
claim the POST-TRIM survivors
for _SOLVER_RESOLVE_INTERVAL_S"] + claimslot --> histpub["_record_solve_history: published"] histpub --> feed["build_combined_aircraft_json
(1 Hz flush)"] feed --> gN2{"n=2 display gate:
solve_count < MN_N2_MIN_SOLVES 2?"} @@ -573,7 +591,7 @@ are all the DARK lane; `lane_split` gives the per-lane record counts and | Block | Says | Watch for | |---|---|---| | `contamination` | Of the dark records that matched ground truth, how many carried a node that could not see the aircraft (`foreign_node_ids` on the record; verdict is the associator's own `_point_in_beam`, the same gate known-lane claiming uses) | `pct` is the live version of the offline ~60 % the cluster-splitting work exists to move. Records with no GT match, or no registered geometry for any contributing node, are **out of the denominator** — abstention, not innocence | -| `resolve_skips` | Candidates the re-solve suppression refused in this window, from `state.solver_resolve_skips_recent`, with the claims that blocked each one | `attempts_ratio` is all-lane skips over DARK attempts (live baseline ~2.4). The deque holds 500 entries against ~50 skips/min, so read `window_effective_minutes` before reading `total` as a window count | +| `resolve_skips` | Candidates the re-solve suppression refused in this window, from `state.solver_resolve_skips_recent`, with the claims that blocked each one | `attempts_ratio` is all-lane skips over DARK attempts. It read ~2.4 while 6.2 claimed on admission; with the claim on publication it should sit at or below 0.5. The deque holds 500 entries against a live rate of tens per minute, so read `window_effective_minutes` before reading `total` as a window count | | `counters.resolve_skips_dark` | Dark share of the since-boot skip counter | — | | `counters.node_frames_rate_limited` | Frames `NODE_FRAME_MIN_INTERVAL_S` refused before the tracker saw them (Gate B in §2) | Not the same event as `/api/admin/metrics`' `frames_dropped`, which is `frame_queue` saturation and normally reads zero | From 964029ec5ece8a513839c4a588dc1a890bdc9cac Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 07:09:23 +0000 Subject: [PATCH 07/15] Score contamination at the publish point as well as the candidate point The candidate-level rate has a denominator association itself moves. Splitting one contaminated cluster into two clean sub-clusters plus the false pairing it was hiding emits three inputs where there was one, and the false pairing on its own scores as 100% contaminated -- so the rate rises while what reaches the map gets cleaner. Measured on the 50-node scatter scene: 20.3% -> 25.7% of candidates, with the ghost rate by solve falling 3.2% -> 2.4% over the same change. So the same score is now also taken at the publish point, over solves that cleared every gate and bound to a real aircraft. That is the population the live audit sampled (45% of published dark solves carried a foreign node), and its denominator is fixed by the aircraft in the sky rather than by how many hypotheses association chose to emit. Co-Authored-By: Claude Fable 5.1 --- backend/scripts/association_bench.py | 66 +++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py index 9e5792db..eac7048a 100644 --- a/backend/scripts/association_bench.py +++ b/backend/scripts/association_bench.py @@ -322,7 +322,7 @@ def _index_detection_truth(det_truth: dict, geo, node_id: str, frame: dict, airc det_truth[key] = oid if (prev is None or prev == oid) else _TRUTH_AMBIGUOUS -def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) -> None: +def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) -> int | None: """Count the nodes in one solver input that are not looking at its aircraft. The input's own aircraft is the plurality of its measurements' true @@ -335,16 +335,23 @@ def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) A node is foreign when its measurement belongs to a different aircraft, or to no aircraft at all (clutter that survived the tracker's M-of-N and the delay grid). Ambiguous attributions are counted in neither. + + Returns the foreign-node count, so the caller can score the same input + again at the publish point (see the PUBLISHED counters on Result: the + candidate-level rate has a denominator the association layer itself moves, + and a change that emits more, cleaner candidates reads as a regression on + it while being an improvement on what actually reaches the map). None + when nothing could be attributed at all. """ oids = [ det_truth.get((m["node_id"], float(m["delay_us"]), float(m["doppler_hz"]))) for m in s_in.get("measurements") or [] ] if not oids: - return + return None counts = Counter(o for o in oids if o is not None and o != _TRUTH_AMBIGUOUS) if not counts: - return + return None top_n = max(counts.values()) contenders = sorted(o for o, c in counts.items() if c == top_n) if len(contenders) > 1: @@ -366,6 +373,7 @@ def _score_contamination(res: Result, s_in: dict, det_truth: dict, truth: list) res.foreign_nodes += foreign if foreign: res.inputs_contaminated += 1 + return foreign def _strip_adsb(frame: dict) -> dict: @@ -667,6 +675,16 @@ class Result: inputs_contaminated: int = 0 foreign_nodes: int = 0 input_nodes: int = 0 + # The same score restricted to inputs that cleared every gate and bound + # to a real aircraft — what actually reached the map. Reported next to + # the candidate rate because the two answer different questions: the + # candidate rate's denominator is the number of candidates association + # chooses to emit, so splitting one contaminated cluster into several + # clean ones plus the false pairing it was hiding *raises* it while + # lowering this one. + published_inputs: int = 0 + published_contaminated: int = 0 + published_foreign_nodes: int = 0 # Stone-Soup GOSPA/SIAP scalars for this one run (--ss-metrics), or None # when it was off, stonesoup wasn't available, or the recorder had @@ -775,6 +793,9 @@ def keys_per_object(self): "inputs_contaminated", "foreign_nodes", "input_nodes", + "published_inputs", + "published_contaminated", + "published_foreign_nodes", "cluster_splits", ) _EXTEND_FIELDS = ( @@ -835,6 +856,14 @@ def contaminated_inputs_pct(self): def foreign_nodes_per_input(self): return self.foreign_nodes / self.inputs_scored if self.inputs_scored else 0.0 + @property + def published_contaminated_pct(self): + return 100.0 * self.published_contaminated / self.published_inputs if self.published_inputs else 0.0 + + @property + def published_foreign_per_solve(self): + return self.published_foreign_nodes / self.published_inputs if self.published_inputs else 0.0 + def build_scene( seed: int, @@ -1165,11 +1194,13 @@ def _geo_key(nid): res.cluster_sizes[(_k, len(s_in.get("track_ids") or []))] += 1 if s_in.get("n_nodes", 0) < 2: continue + _foreign = None if mode == "track": # Scored here, ahead of the solve and every gate below: # this measures what association emitted, which is the - # thing the cluster-merge rework changes. - _score_contamination(res, s_in, det_truth, truth) + # thing the cluster-merge rework changes. Re-scored at + # the publish point further down, on the same number. + _foreign = _score_contamination(res, s_in, det_truth, truth) try: _t0 = time.perf_counter() out = solve_fn(s_in, node_cfgs) @@ -1273,6 +1304,17 @@ def _geo_key(nid): (_keys_real if d <= MATCH_KM else _keys_ghost).add(_key) if d <= MATCH_KM: res.matched += 1 + if _foreign is not None: + # Same input, scored again now that every gate has + # accepted it and it has bound to a real aircraft: + # this is the population the live audit sampled (45% + # of published dark solves carried a foreign node), + # and unlike the candidate rate its denominator is + # not something association can inflate. + res.published_inputs += 1 + res.published_foreign_nodes += _foreign + if _foreign: + res.published_contaminated += 1 res.errors_km.append(d) res.n_nodes_matched[nn] += 1 # Broken out because the whole dual-site hypothesis is @@ -1506,6 +1548,12 @@ def report(label: str, r: Result, truth_max_kt: float | None = None): f"foreign nodes/input {r.foreign_nodes_per_input:.2f}" f" ({r.foreign_nodes}/{r.input_nodes} nodes)" ) + if r.published_inputs: + print( + f" CONTAMINATION (published): {r.published_contaminated}/{r.published_inputs} matched solves" + f" -> {r.published_contaminated_pct:5.1f}% " + f"foreign nodes/solve {r.published_foreign_per_solve:.2f}" + ) if r.cluster_splits: print(f" cluster splits (same-node track conflict): {r.cluster_splits}") if r.gate_gated: @@ -1846,7 +1894,7 @@ def main(): solve_fn = _ESTIMATORS[estimator_name] rates, solve_rates, reals, fakes, speed_errs = [], [], [], [], [] n2_rates = [] - contam_rates, foreign_rates, med_errs = [], [], [] + contam_rates, foreign_rates, med_errs, pub_contam_rates = [], [], [], [] agg = Result() last = None for k in range(args.repeat): @@ -1896,6 +1944,7 @@ def main(): speed_errs.append(statistics.median(last.speed_err_ms)) contam_rates.append(last.contaminated_inputs_pct) foreign_rates.append(last.foreign_nodes_per_input) + pub_contam_rates.append(last.published_contaminated_pct) med_errs.append(statistics.median(last.errors_km) if last.errors_km else float("nan")) label = f"assoc_interval={interval:g}s" if chi2_max is not None: @@ -1930,6 +1979,11 @@ def main(): f"{', '.join(f'{x:.2f}' for x in foreign_rates)}" f" mean {statistics.mean(foreign_rates):.2f}" ) + print( + f" published contaminated per seed: " + f"{', '.join(f'{x:.0f}%' for x in pub_contam_rates)}" + f" mean {statistics.mean(pub_contam_rates):.1f}%" + ) print( f" median matched error per seed: " f"{', '.join(f'{x:.2f}' for x in med_errs)} km" From ed1a023b229071997c233d989766478d13080434 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 07:12:20 +0000 Subject: [PATCH 08/15] Report superseded and cluster_splits as live association counters /api/radar/association/status filed track_pairs_superseded under "track_pairs_inline_only", documented as permanently zero in production because the only exclusivity stage ran on a chi2 that cv_fit=None never computes. The deferred path now has an exclusivity stage of its own, so the counter moves on a live fleet and that grouping would be a lie. Moved to the live block alongside the new cluster_splits. Co-Authored-By: Claude Fable 5.1 --- backend/routes/analytics.py | 14 ++++++++++--- backend/tests/test_analytics_routes.py | 29 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/backend/routes/analytics.py b/backend/routes/analytics.py index 07d83a9c..8cb8c836 100644 --- a/backend/routes/analytics.py +++ b/backend/routes/analytics.py @@ -131,20 +131,28 @@ async def association_status(): # Track-pairing outcomes since boot. gated is everything past the # coarse delay grid; unfitted counts the pairings handed to the solver # worker (which runs the fit and the n=2 gate); deferred counts rounds - # a budget cut short. Those three are the live production surface. + # a budget cut short; superseded counts pairings dropped because a + # better-ranked one claiming the same track implied a velocity theirs + # contradicts; cluster_splits counts position clusters that held two + # tracks of one node and were emitted as one solver input each. All + # five are the live production surface. "track_pairs": { "gated": getattr(_a, "track_pairs_gated", 0), "unfitted": getattr(_a, "track_pairs_unfitted", 0), "deferred": getattr(_a, "track_pairs_deferred", 0), + "superseded": getattr(_a, "track_pairs_superseded", 0), + "cluster_splits": getattr(_a, "cluster_splits", 0), }, # Inline-fit counters — permanently zero in production BY DESIGN # (state.py builds the associator with cv_fit=None; only the offline - # bench's inline mode exercises stage-2 selection). Split out so + # bench's inline mode exercises the chi2 threshold). Split out so # nobody reads a structural zero as "no rejections happening". + # superseded used to live here too, and no longer can: the deferred + # path now has an exclusivity stage of its own, so the counter moves + # in production. "track_pairs_inline_only": { "accepted": getattr(_a, "track_pairs_accepted", 0), "rejected": getattr(_a, "track_pairs_rejected", 0), - "superseded": getattr(_a, "track_pairs_superseded", 0), }, # Top-down claiming (ASSOC_CLAIM_MODE) since boot. rounds/matched/ # conflicts/anchored_inputs are all live in shadow too — _claim_round diff --git a/backend/tests/test_analytics_routes.py b/backend/tests/test_analytics_routes.py index 22a821e0..1778cdf8 100644 --- a/backend/tests/test_analytics_routes.py +++ b/backend/tests/test_analytics_routes.py @@ -129,6 +129,35 @@ def test_status_returns_expected_fields(self, client): assert "overlap_zones" in body assert "overlaps" in body + def test_track_pairs_block_reports_the_live_counters(self, client): + """superseded and cluster_splits belong to the LIVE block, not the + inline-only one. + + They used to be structurally zero in production because the only + exclusivity stage ran on a chi2 nothing computes with cv_fit=None. + The deferred path now prunes on implied-velocity conflict and splits + clusters that hold two tracks of one node, so both counters move on a + live fleet and reading them as "inline only" would be wrong. + """ + _a = state.node_associator + _a.track_pairs_superseded += 5 + _a.cluster_splits += 3 + try: + body = client.get("/api/radar/association/status").json() + assert body["track_pairs"].keys() == { + "gated", + "unfitted", + "deferred", + "superseded", + "cluster_splits", + } + assert body["track_pairs"]["superseded"] == 5 + assert body["track_pairs"]["cluster_splits"] == 3 + assert body["track_pairs_inline_only"].keys() == {"accepted", "rejected"} + finally: + _a.track_pairs_superseded -= 5 + _a.cluster_splits -= 3 + def test_status_includes_claiming_block(self, client): """Top-down claiming (ASSOC_CLAIM_MODE) since boot — off by default in tests, so this pins the shape rather than any particular mode.""" From ca748c39e46c7b65af429b1ed3851c7b34704a6a Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 07:46:33 +0000 Subject: [PATCH 09/15] Bump retina-analytics: bound the cluster diameter Picks up offworldlabs/retina-analytics@7e5414a, which extends the cluster partition from same-node conflicts only to every group, bounding each emitted solver input at the merge distance. Foreign nodes per published solve 0.80 -> 0.32 on the 15-node ring bench scene. Co-Authored-By: Claude Fable 5.1 --- libs/retina-analytics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/retina-analytics b/libs/retina-analytics index 4ff48fca..7e5414ac 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 4ff48fcaa3abdaad1d42eb576f8573342addad4e +Subproject commit 7e5414ac1671bb66c160a425363b1d19c889eea4 From 3702fec563e601ab56a4d8f077e98c8205da7bf4 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 08:22:57 +0000 Subject: [PATCH 10/15] Exclude stale coasting tracks and align measurement epochs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent corrections to the same failure: legitimately in-cone nodes being trimmed out of dark solves because their measurement did not describe the moment the solve claims to. Stale-track exclusion (TRACK_MAX_STALE_S, default 3.0 s). A COASTING track survives N_DELETE=10 frames past its last association, and at the fleet's 0.74-1 Hz per-node cadence that is up to ~13 s of dead reckoning. confirmed_track_views hands association the track's newest REAL sample and association hands the solver that sample as the node's current measurement, so an aircraft that has flown out of a node's beam keeps contributing seconds-old delay to n>=3 solves. Measured on the test droplet: 230 out-of-cone nodes reached published dark solves in 20 min, median 4 deg outside the beam edge (p90 22 deg) and 1.8 km beyond max range. The freshness test reads the newest entry from get_recent_detections, which returns only ASSOCIATED samples, so it is the last real detection's time rather than a coast count — and it is compared against the frame timestamp being processed, never wall clock, so replays and backfills are unaffected. Epoch alignment (SOLVER_EPOCH_ALIGN, default on). The solver's residual model evaluates every measurement against one target state; nodes sample at independent phases, so it is fitting a set that spans up to a frame interval. align_measurement_epochs dead-reckons each delay onto the newest sample's time along its own measured Doppler, using d(delay_us)/dt = -doppler_hz * 1e6 / fc_hz — the rate implied by the geolocator's own residual model, in which positive Doppler is a closing target whose delay is decreasing. All-or-nothing per input: a partially aligned set just relocates the error, so a missing t_s, doppler_hz or fc_hz skips the input and counts solver_epoch_align_skipped. Both flags exist to be turned off live rather than rolled back. The known lane reuses the same helper: it does NOT have one epoch either (_CLAIM_SPREAD_S admits claims up to 5 s apart, ~1.5 km of motion at 300 m/s), and it is the lane whose residual is the fleet's accuracy measurement. Pins retina-analytics feat/measurement-epochs (PR #27), which carries t_s on each solver-input measurement. Co-Authored-By: Claude Fable 5.1 --- backend/config/constants.py | 13 ++++ backend/core/state.py | 30 ++++++++- backend/routes/admin.py | 2 + backend/routes/test.py | 9 +++ backend/scripts/association_bench.py | 2 +- backend/services/frame_processor.py | 33 ++++++++-- backend/services/tasks/known_lane.py | 19 +++++- backend/services/tasks/solver.py | 96 ++++++++++++++++++++++++++++ libs/retina-analytics | 2 +- 9 files changed, 197 insertions(+), 9 deletions(-) diff --git a/backend/config/constants.py b/backend/config/constants.py index ea788abf..2d1cd19c 100644 --- a/backend/config/constants.py +++ b/backend/config/constants.py @@ -84,6 +84,19 @@ def as_num(v) -> float: N2_CONFIRM_MIN_EPOCHS = 4 # Floor on samples; span is the real gate N2_TRACK_HISTORY_MAX = 20 # Per-node track samples fed to the fit +# How old a track's newest REAL detection may be before the track stops being +# offered to association (see services/frame_processor.confirmed_track_views). +# A COASTING track is kept alive for N_DELETE=10 frames after its last +# association, and at the fleet's 0.74-1 Hz per-node cadence that is up to ~13 s +# of dead reckoning. confirmed_track_views hands association the track's last +# real sample, and association hands the solver that sample as if it were +# current — so an aircraft that has flown out of a node's beam keeps +# contributing a seconds-old delay to n>=3 solves. Measured on the test +# droplet: 230 out of-cone nodes survived into published dark solves in 20 min, +# median 4 deg outside the beam edge (p90 22 deg) and 1.8 km beyond max range, +# and they are the nodes the rms trim then throws away. 0 disables the filter. +TRACK_MAX_STALE_S = float(os.getenv("TRACK_MAX_STALE_S", "3.0")) + # A 2-node track needs this many solves before it renders a plane; 1 # disables the gate. One-shot n=2 solves were the dominant ghost source. MN_N2_MIN_SOLVES = int(os.getenv("MN_N2_MIN_SOLVES", "2")) diff --git a/backend/core/state.py b/backend/core/state.py index 634aa212..f9e4b6f3 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -74,6 +74,18 @@ if KNOWN_LANE_MODE not in ("off", "shadow", "binding"): KNOWN_LANE_MODE = "shadow" +# Measurement epoch alignment (see services/tasks/solver.align_measurement_epochs). +# on/off rather than the off/shadow/active triple its neighbours use: there is +# nothing to shadow — the correction is a closed-form dead-reckoning of each +# delay along its own measured Doppler, so a dry run would produce the same +# number the acting run applies and observe nothing extra. Default "on", +# because leaving it off is the bug: nodes sample at independent phases and the +# solver treats their measurements as simultaneous, so a 250 m/s target charges +# up to ~1 us of delay error per second of skew (measured ~0.3 us rms at 2 s +# skew on the fleet) straight to the 3 us rms gate. The flag exists so the +# alignment can be turned off live without a rollback if it ever misbehaves. +SOLVER_EPOCH_ALIGN = os.getenv("SOLVER_EPOCH_ALIGN", "on").strip().lower() != "off" + node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE) @@ -526,6 +538,21 @@ def _adsb_for_seeding() -> dict[str, dict]: # fleet's trigger rate — constraints are then converging slower than the # coverage they follow, which no rebuild counter can show. coverage_rebuild_backlog: int = 0 + +# Confirmed tracks withheld from association because their newest REAL +# detection was older than TRACK_MAX_STALE_S at the frame being processed — +# see services/frame_processor.confirmed_track_views. These are aircraft that +# have left a node's beam and whose track is dead-reckoning toward deletion; +# their last real sample used to reach the solver as a current measurement. +tracks_stale_skipped: int = 0 + +# Solver inputs whose measurements could not be aligned to a common epoch +# because at least one lacked t_s, doppler_hz, or a node config with fc_hz — +# see services/tasks/solver.align_measurement_epochs. Counted only when +# SOLVER_EPOCH_ALIGN is on; a nonzero value against solver_successes says how +# much of the fleet is still emitting untimed measurements. +solver_epoch_align_skipped: int = 0 + solver_queue_drops: int = 0 # Queue items discarded unsolved because they aged past _SOLVER_MAX_QUEUE_AGE_S # waiting for a worker. Was only a DEBUG log, which staging does not emit — @@ -750,7 +777,7 @@ def _reset_for_tests() -> None: global known_claims_made, known_claim_contentions, known_claims_bound global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes - global coverage_rebuild_backlog + global coverage_rebuild_backlog, tracks_stale_skipped, solver_epoch_align_skipped global solver_queue_drops, solver_stale_drops, solver_resolve_skips global mn_superseded, mn_superseded_blocked, solver_trimmed global solver_consensus_selected, solver_consensus_filtered @@ -840,6 +867,7 @@ def _reset_for_tests() -> None: known_claims_world_rejects = 0 coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0 coverage_rebuild_backlog = 0 + tracks_stale_skipped = solver_epoch_align_skipped = 0 solver_stale_drops = 0 solver_resolve_skips = 0 mn_superseded = mn_superseded_blocked = 0 diff --git a/backend/routes/admin.py b/backend/routes/admin.py index 2d0eca6d..4d11fb0b 100644 --- a/backend/routes/admin.py +++ b/backend/routes/admin.py @@ -590,6 +590,8 @@ async def system_metrics(_user=Depends(require_admin)): "solver_queue_drops": state.solver_queue_drops, "solver_stale_drops": state.solver_stale_drops, "solver_resolve_skips": state.solver_resolve_skips, + "tracks_stale_skipped": state.tracks_stale_skipped, + "solver_epoch_align_skipped": state.solver_epoch_align_skipped, "mn_superseded": state.mn_superseded, "solver_trimmed": state.solver_trimmed, "solver_last_latency_s": round(state.solver_last_latency_s, 3), diff --git a/backend/routes/test.py b/backend/routes/test.py index e4018b36..afc57dc7 100644 --- a/backend/routes/test.py +++ b/backend/routes/test.py @@ -214,6 +214,13 @@ def _build_dashboard_data() -> bytes: # against stale_drops: skips are work correctly not done, # stale drops are work lost. "resolve_skips": state.solver_resolve_skips, + # Confirmed tracks withheld from association because their + # newest real detection had aged past TRACK_MAX_STALE_S, and + # solver inputs the epoch alignment could not correct because a + # measurement carried no sample time (see frame_processor's + # confirmed_track_views and solver's align_measurement_epochs). + "tracks_stale_skipped": state.tracks_stale_skipped, + "epoch_align_skipped": state.solver_epoch_align_skipped, # Multinode entries replaced because a later solve consumed # the same source tracks under a new key (fragmented re-solve). "mn_superseded": state.mn_superseded, @@ -1192,6 +1199,8 @@ def _solver_window_stats(minutes: float) -> dict: "solver_trimmed": state.solver_trimmed, "stale_drops": state.solver_stale_drops, "resolve_skips": state.solver_resolve_skips, + "tracks_stale_skipped": state.tracks_stale_skipped, + "epoch_align_skipped": state.solver_epoch_align_skipped, "queue_drops": state.solver_queue_drops, "worker_errors": state.solver_worker_errors, "vel_untrusted_published": state.solver_vel_untrusted_published, diff --git a/backend/scripts/association_bench.py b/backend/scripts/association_bench.py index e08999bb..acc7196a 100644 --- a/backend/scripts/association_bench.py +++ b/backend/scripts/association_bench.py @@ -978,7 +978,7 @@ def _geo_key(nid): # frame is what association pairs against — but only let a node # *trigger* a round on its own cadence. if mode == "track": - assoc._pending_tracks[nid] = confirmed_track_views(trackers[nid], history_n) + assoc._pending_tracks[nid] = confirmed_track_views(trackers[nid], history_n, ts_ms) else: assoc._pending_frames[nid] = frame if (t - last_assoc.get(nid, -1e9)) < assoc_interval: diff --git a/backend/services/frame_processor.py b/backend/services/frame_processor.py index 0fb2bbd5..9490e8ca 100644 --- a/backend/services/frame_processor.py +++ b/backend/services/frame_processor.py @@ -18,6 +18,7 @@ ARCHIVE_BATCH_MAX, ARCHIVE_FLUSH_INTERVAL_S, N2_TRACK_HISTORY_MAX, + TRACK_MAX_STALE_S, ) from core import state from pipeline.passive_radar import PassiveRadarPipeline @@ -254,7 +255,11 @@ def _view_adsb_hex(track, hist) -> str | None: return hexn -def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> list[dict]: +def confirmed_track_views( + tracker, + history_n: int = N2_TRACK_HISTORY_MAX, + now_ts_ms: int | None = None, +) -> list[dict]: """A tracker's confirmed tracks, in the shape submit_tracks takes. TENTATIVE tracks are excluded, the same filter the arc builder applies: they @@ -264,9 +269,26 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis reason arcs keep it — at 22 fps a single missed frame flips ACTIVE → COASTING and the next flips it back. + But COASTING is kept only while its newest REAL detection is fresh. What + travels downstream is ``history[-1]``, and association hands that sample to + the solver as the node's current measurement — so a track coasting toward + its N_DELETE deletion point contributes a delay from wherever the aircraft + was several seconds ago. That is the out-of-cone node the rms trim then + has to discard (see TRACK_MAX_STALE_S). The staleness test reads + ``hist[-1]["timestamp"]`` rather than the track's coast count because + get_recent_detections returns only ASSOCIATED samples — mark_missed appends + None to ``history["measurements"]`` and the reverse scan skips those — so + that timestamp IS the last real detection's, exactly the honest signal, + while n_missed only counts frames the node happened to process. Compared + against *now_ts_ms*, the frame timestamp being processed, never wall clock: + the fleet replays and backfills, and a filter keyed on wall clock would + silently empty every view in those runs. Skipped when the caller supplies + no frame time, or when TRACK_MAX_STALE_S is 0. + Shared with scripts/association_bench.py (which carried a near-verbatim copy) so the bench feeds association exactly what production does. """ + max_stale_ms = TRACK_MAX_STALE_S * 1000.0 if now_ts_ms is not None else 0.0 views = [] for tr in tracker.tracks: if tr.state_status == TrackState.TENTATIVE: @@ -274,6 +296,9 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis hist = tr.get_recent_detections(history_n) if len(hist) < 2: continue + if max_stale_ms > 0 and (now_ts_ms - hist[-1]["timestamp"]) > max_stale_ms: + state.bump_counter("tracks_stale_skipped") + continue views.append( { "track_id": tr.id or f"tmp-{id(tr)}", @@ -292,8 +317,8 @@ def confirmed_track_views(tracker, history_n: int = N2_TRACK_HISTORY_MAX) -> lis return views -def _node_track_views(pipeline: PassiveRadarPipeline) -> list[dict]: - return confirmed_track_views(pipeline.tracker) +def _node_track_views(pipeline: PassiveRadarPipeline, now_ts_ms: int | None = None) -> list[dict]: + return confirmed_track_views(pipeline.tracker, now_ts_ms=now_ts_ms) def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarPipeline): @@ -397,7 +422,7 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP # Track-level association. The detection-level path it replaced now lives # in retina_analytics.detection_association, reachable only from the # offline bench, which keeps it as the A/B baseline. - _track_views = _node_track_views(pipeline) + _track_views = _node_track_views(pipeline, _ts_ms_assoc or None) # Feed the per-node distinct-track counters — total_tracks / # geolocated_tracks were exported (and read by the admin API) but never # written anywhere. diff --git a/backend/services/tasks/known_lane.py b/backend/services/tasks/known_lane.py index 6ffe29ad..8d335bbd 100644 --- a/backend/services/tasks/known_lane.py +++ b/backend/services/tasks/known_lane.py @@ -245,6 +245,12 @@ def _build_solver_input(hexn: str, claims: dict[str, dict]) -> dict | None: "delay_us": float(c["delay_us"]), "doppler_hz": float(c["doppler_hz"]), "snr": _num(c.get("snr")), + # This lane does NOT have one epoch: _CLAIM_SPREAD_S admits + # claims up to 5 s apart, which at 300 m/s is ~1.5 km of target + # motion charged straight to the residual this lane exists to + # measure. Carrying each claim's own capture time lets + # _attempt reuse the regular lane's epoch alignment. + "t_s": int(c["ts_ms"]) / 1000.0, } for nid, c in sorted(claims.items()) ], @@ -362,6 +368,15 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non record's displacement_km and the accuracy error are the same number. """ state.bump_counter("known_lane_attempts") + # Same correction, same flag, same helper as the regular lane — see + # solver.align_measurement_epochs. Applied here rather than in + # _build_solver_input because the alignment needs the node configs, and + # because the accuracy classification below compares the solve against an + # initial guess already dead-reckoned to the newest claim's epoch, which is + # exactly the t0 the helper aligns onto. + epoch_meta: dict = {"epoch_aligned": False} + if state.SOLVER_EPOCH_ALIGN: + s_in, epoch_meta = solver_mod.align_measurement_epochs(s_in, node_cfgs) try: # Single solve at the pinned ADS-B altitude — no layer sweep. The # sweep exists to DISCOVER an unknown altitude; here identity already @@ -379,7 +394,7 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non "known_no_converge", s_in, result if isinstance(result, dict) else None, - extra={"known_lane": True, "label": "no_converge", "published": False}, + extra={"known_lane": True, "label": "no_converge", "published": False, **epoch_meta}, ) return @@ -419,7 +434,7 @@ def _attempt(hexn: str, s_in: dict, node_cfgs: dict, solve_fn, mode: str) -> Non raw_lat=raw_lat, raw_lon=raw_lon, displacement_km=err_km, - extra={"known_lane": True, "label": label, "published": published}, + extra={"known_lane": True, "label": label, "published": published, **epoch_meta}, ) diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 7c6b947c..4c1f56ea 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -279,6 +279,91 @@ def _dark_displacement_cap_km() -> float: # It lived here while the frame path had its own, looser, unstated one. +# ── Measurement epoch alignment ────────────────────────────────────────────── +# The solver's residual model evaluates every measurement against ONE target +# state: the measurement set is assumed simultaneous. It is not. Each node +# samples on its own free-running cadence (~0.74-1 Hz on the fleet), so the +# delays in one solver input were captured at times spread over up to a frame +# interval, and association hands over each track's newest sample regardless of +# when that was. A 250 m/s target moves ~250 m per second of skew, which shows +# up as up to ~1 us of bistatic delay error per second — charged in full to the +# 3 us rms_delay gate, where it is indistinguishable from a contaminated node +# and drives the trim to throw away legitimately in-cone nodes. +# +# The correction is closed-form and needs nothing the measurement does not +# already carry. Writing d_tx / d_rx for the TX->target and target->RX ranges, +# the bistatic delay is (d_tx + d_rx - baseline)/c and the bistatic Doppler is +# (fc/c)(v_tx + v_rx), where v_tx / v_rx are the target's velocity components +# along the unit vectors pointing FROM the target TOWARD the TX and the RX +# (retina_geolocator.multinode_solver._residual_function; the simulator's +# _bistatic_delay / _bistatic_doppler in retina_simulation.world use the +# identical convention). Moving toward a site shortens that leg, so +# d(d_tx)/dt = -v_tx and d(d_rx)/dt = -v_rx, and therefore +# +# d(delay_us)/dt = -(v_tx + v_rx) / C_KM_US +# = -doppler_hz * (C_KM_S / fc_hz) / C_KM_US +# = -doppler_hz * 1e6 / fc_hz +# +# i.e. positive Doppler is a closing target and its delay is DECREASING. The +# unit test test_epoch_alignment.py checks the sign against a target flown +# through the simulator's own geometry helpers at two times, rather than +# against this derivation. +_DELAY_RATE_HZ_TO_US_PER_S = 1e6 + + +def align_measurement_epochs(s_in: dict, node_cfgs: dict) -> tuple[dict, dict]: + """Dead-reckon every measurement's delay onto the newest one's epoch. + + Pure: returns a new solver input (shallow copy, fresh measurement dicts) + and a metadata dict for the history record; *s_in* is never mutated, so a + caller can drop the result and keep the untouched input. + + Alignment is all-or-nothing per input. A partially aligned set is worse + than an unaligned one — the residual model has no way to know which + measurements share an epoch, so mixing corrected and uncorrected delays + just moves the error onto a different node. Any measurement missing t_s + or doppler_hz, or whose node has no config to read fc_hz from, therefore + skips the whole input and counts solver_epoch_align_skipped. + + Returns (s_in, meta) where meta carries epoch_aligned and, when it ran, + epoch_skew_s — the widest gap the correction closed. + """ + meas = s_in.get("measurements") or [] + if len(meas) < 2: + return s_in, {"epoch_aligned": False} + + rates = [] + for m in meas: + t_s = m.get("t_s") + doppler = m.get("doppler_hz") + cfg = node_cfgs.get(m.get("node_id")) or {} + # Same fallback chain the geolocator uses when it builds its NodeSetup, + # so a node whose config spells the carrier "FC" aligns on exactly the + # frequency the solve will predict against. + fc_hz = cfg.get("fc_hz", cfg.get("FC")) + if t_s is None or doppler is None or not fc_hz: + state.bump_counter("solver_epoch_align_skipped") + return s_in, {"epoch_aligned": False} + rates.append((float(t_s), -float(doppler) * _DELAY_RATE_HZ_TO_US_PER_S / float(fc_hz))) + + # The newest sample, not the input's timestamp_ms: t0 has to be a time some + # measurement was actually taken, or every delay is extrapolated and the + # freshest node — the one that needed no correction — acquires an error. + t0 = max(t for t, _ in rates) + skew_s = t0 - min(t for t, _ in rates) + + aligned = dict(s_in) + aligned["measurements"] = [ + {**m, "delay_us": float(m["delay_us"]) + rate * (t0 - t_s)} for m, (t_s, rate) in zip(meas, rates) + ] + if "timestamp_ms" in aligned: + # The set now describes t0, so everything downstream that ages this + # solve (multinode expiry, the dead-reckoning gates, the history + # record's measurement_ts_ms) should date it from t0 too. + aligned["timestamp_ms"] = int(round(t0 * 1000.0)) + return aligned, {"epoch_aligned": True, "epoch_skew_s": round(skew_s, 3)} + + def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[float], metric: str) -> dict | None: """Try each altitude layer; return the result with lowest value of `metric`. @@ -1728,6 +1813,13 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus state.bump_counter("solver_resolve_skips") return None n_nodes = s_in.get("n_nodes", 0) if isinstance(s_in, dict) else 0 + # Before anything reads a delay: the nodes did not sample simultaneously, + # and every gate below (rms_delay first among them) assumes they did. Runs + # ahead of consensus and the altitude sweep so both judge the same aligned + # numbers the published solve is fitted to. + epoch_meta: dict = {"epoch_aligned": False} + if state.SOLVER_EPOCH_ALIGN and isinstance(s_in, dict): + s_in, epoch_meta = align_measurement_epochs(s_in, node_cfgs) consensus_meta: dict | None = None try: if "initial_guess" not in s_in: @@ -1771,6 +1863,10 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus _extra: dict | None = dict(trim_meta) if trim_meta else {} if consensus_meta is not None: _extra["consensus_meta"] = consensus_meta + # Always stamped, aligned or not: "this solve was not aligned" is the + # fact /api/test/mlat-history needs to separate a residual the + # correction could not have helped from one it was applied to. + _extra.update(epoch_meta) _extra = _extra or None rms_delay = result.get("rms_delay", 0) or 0 diff --git a/libs/retina-analytics b/libs/retina-analytics index 14504176..d9dc9e00 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 145041767422723d89d98ec003f843347ddbb880 +Subproject commit d9dc9e00f97d3788f5319109e0832cb270ed2fec From cb38610b1eae394ee8472491fc2e7759d6b48ddc Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 08:33:35 +0000 Subject: [PATCH 11/15] solver: one free-altitude start by default, SOLVER_FREE_ALT_STARTS for more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The free mode shipped with three start altitudes on the reasoning that freeing z removes the ladder's quantisation but not the LM's locality, so several starts are still what keeps a solve off the wrong side of a bistatic ellipse. On this fleet's geometry there was almost nothing for them to keep it off. Over a 20-minute window on test, 1019 free-mode solves: the three starts' rms_delay differed by more than 0.1 us in 13 of them (1.3%), and the nearest-layer start — the one the sweep would have pinned at — was more than 0.5 us worse than the best start in 2. So the extra two starts bought ~0.2% of solves a rounding error at three times the solver CPU, and solver CPU is now the constraint: ~1.7 attempts/s against a 2.0 s average latency on two pool workers. The count becomes SOLVER_FREE_ALT_STARTS, default 1, read per call beside the mode flag and clamped into [1, len(layers)] by _free_alt_starts against the ladder that module owns. One start is the layer nearest the association guess, or the guess altitude itself when the input carries a non-layer one (ADS-B) — the same splice the sweep does, so the one exact altitude a candidate has is still what it starts from. Above 1 the window is unchanged, so 3 reproduces what was measured; the knob stays because locality is a property of the geometry, not of this fleet, and a deployment whose nodes sit nearer an ellipse should not need a code change to buy the starts back. alt_starts_km / alt_start_rms_us keep being recorded — a one-element list rather than three — so the live comparison channel is unchanged. Co-Authored-By: Claude Fable 5.1 --- backend/.env.example | 9 +++ backend/core/state.py | 21 +++++- backend/services/tasks/solver.py | 34 ++++++--- backend/tests/test_solver_alt_mode.py | 101 +++++++++++++++++++++++--- docs/solverflow.md | 28 +++++-- 5 files changed, 162 insertions(+), 31 deletions(-) diff --git a/backend/.env.example b/backend/.env.example index 488ecbdc..51a1c782 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -138,10 +138,19 @@ MENDER_PAT= # multi-start helper, which solves altitude as a sixth unknown. Both modes # stamp altitude_mode on the solve-history record, so /api/test/mlat-history # can compare them across a deploy of each. See docs/solverflow.md. +# +# SOLVER_FREE_ALT_STARTS is how many start altitudes free mode gives that +# helper, clamped to the number of layers. 1 (the default) starts at the layer +# nearest the association guess, or at the guess altitude itself when it comes +# from ADS-B; more is a window around it, at one LM run each. Measured on test +# over 1019 free solves, three starts changed rms_delay by more than 0.1 us in +# 13 of them, so the extra runs are off by default and worth turning on only +# where the geometry sends a single start to the wrong side of an ellipse. # SOLVER_WORKERS=2 # SOLVER_RESOLVE_INTERVAL_S=12 # SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0 # SOLVER_ALT_MODE=sweep +# SOLVER_FREE_ALT_STARTS=1 # Detection mirror. Production only. Every accepted v1 detection frame is # forwarded to another environment's /api/radar/detections/bulk, batched once a diff --git a/backend/core/state.py b/backend/core/state.py index f1239459..962c9006 100644 --- a/backend/core/state.py +++ b/backend/core/state.py @@ -84,7 +84,8 @@ # systematically up to 1 km wrong and that error lands in the # residual the reject gate reads. # free — one pool call to retina_geolocator's multi-start helper, which -# solves altitude as a sixth unknown from three start layers. +# solves altitude as a sixth unknown, started from +# SOLVER_FREE_ALT_STARTS of those layers. # Not off/shadow/active: there is no shadow here, because the two modes # produce the same shape of result and the history record carries # altitude_mode either way — running both would double the solver's cost to @@ -94,6 +95,24 @@ if SOLVER_ALT_MODE not in ("sweep", "free"): SOLVER_ALT_MODE = "sweep" +# How many start altitudes the free mode hands that helper. Read here beside +# the mode it qualifies; _free_alt_starts in services/tasks/solver.py clamps it +# into [1, len(layers)] against the ladder that module owns. 1 starts at the +# layer nearest the association guess — where the sweep would have pinned; +# more is a window around it. +# +# The default is 1 because three starts did not pay for themselves: over 1019 +# free-mode solves on test, the three starts' rms_delay differed by more than +# 0.1 us in 13 of them, and the nearest-layer start was more than 0.5 us worse +# than the best start in 2. That is ~0.2% of solves helped for 3x the solver +# CPU, and the pool — not the altitude ladder — is what this deployment is +# short of (~1.7 attempts/s against a 2.0 s average latency on two workers). +# The knob stays because the reason for several starts is the LM's locality, +# which is a property of the geometry rather than of this fleet: nodes lying +# nearer a bistatic ellipse than these can send a single start to the wrong +# side of it, and finding that out should not need a code change. +SOLVER_FREE_ALT_STARTS = max(1, int(os.getenv("SOLVER_FREE_ALT_STARTS", "1"))) + node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE) diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py index 41bb53ec..df890941 100644 --- a/backend/services/tasks/solver.py +++ b/backend/services/tasks/solver.py @@ -338,12 +338,6 @@ def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[floa return best_result -# How many start altitudes the free mode hands the multi-start helper: the -# layer nearest the guess and its two neighbours. Freeing z removes the -# ladder's quantisation but not the LM's locality, so the starts are still -# what stops a solve settling on the wrong side of a bistatic ellipse — but -# three of them, in one pool call, rather than the sweep's six. -_FREE_ALT_N_STARTS = 3 # Fewest measurements the free mode is used at. Below this altitude is not # observable and retina_geolocator pins it anyway; the sweep is left in place # so the n=2 path keeps its documented behaviour exactly. @@ -351,17 +345,33 @@ def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[floa def _free_alt_starts(ig_alt_km, layers: list[float]) -> list[float]: - """The layer nearest ``ig_alt_km`` and its neighbours — _FREE_ALT_N_STARTS - of them, clamped to the ends of the ladder so the count never shrinks - there (the top and bottom layers are where a wrong start is least + """The start altitudes the free mode hands the multi-start helper. + + state.SOLVER_FREE_ALT_STARTS of them, clamped into [1, len(layers)] — read + per call, like the mode flag, so a test and a config reload both see what + they set. One start is the layer nearest ``ig_alt_km``, which is the + altitude spliced into ``layers`` when the input carries a non-layer one of + its own (ADS-B), exactly as the sweep treats it. Several are a window + centred on that layer, clamped to the ends of the ladder so the count never + shrinks there (the top and bottom layers are where a wrong start is least recoverable, not most). + + Freeing z removes the ladder's quantisation but not the LM's locality, and + the extra starts are what would stop a solve settling on the wrong side of + a bistatic ellipse. On this fleet's geometry they had almost nothing to + stop: over 1019 free-mode solves on test, three starts' rms_delay differed + by more than 0.1 µs in 13 of them, and the nearest-layer start was more + than 0.5 µs worse than the best in 2 — so the default is one start and the + other two are bought explicitly, by a deployment whose geometry shows it + needs them. See core/state.py for the numbers and the trade. """ if not layers: return [] + n = max(1, min(int(state.SOLVER_FREE_ALT_STARTS), len(layers))) alt = float(ig_alt_km) if ig_alt_km is not None else 7.0 nearest = min(range(len(layers)), key=lambda i: abs(layers[i] - alt)) - lo = max(0, min(nearest - 1, len(layers) - _FREE_ALT_N_STARTS)) - return layers[lo : lo + _FREE_ALT_N_STARTS] + lo = max(0, min(nearest - (n - 1) // 2, len(layers) - n)) + return layers[lo : lo + n] def _solve_best_altitude( @@ -399,7 +409,7 @@ def _solve_best_altitude( n_meas = len({m.get("node_id") for m in (s_in.get("measurements") or [])}) if state.SOLVER_ALT_MODE == "free" and n_meas >= _FREE_ALT_MIN_NODES: # No fall back to the sweep when this returns None: a helper that got - # no solve out of three starts is reporting the same thing the sweep + # no solve out of its starts is reporting the same thing the sweep # reports when every layer fails, and sweeping anyway would cost the # six round trips this mode exists to avoid on exactly the candidates # that are least likely to repay them. diff --git a/backend/tests/test_solver_alt_mode.py b/backend/tests/test_solver_alt_mode.py index 8a374dbc..2c9eec02 100644 --- a/backend/tests/test_solver_alt_mode.py +++ b/backend/tests/test_solver_alt_mode.py @@ -4,7 +4,8 @@ lowest rms_delay — six process-pool round trips, and an altitude quantised to a ladder 2 km wide, which puts up to 1 km of error into the residual the reject gate reads. free makes ONE call to the geolocator's multi-start -helper, which solves altitude as a sixth unknown from three start layers. +helper, which solves altitude as a sixth unknown from SOLVER_FREE_ALT_STARTS +start layers — one by default, the layer nearest the association guess. These tests are about the routing, not the physics: the geolocator's own suite (tests/test_free_altitude.py there) measures what the free solve @@ -78,7 +79,32 @@ def teardown_method(self): class TestFreeAltStarts: - """The three starts handed to the multi-start helper.""" + """The starts handed to the multi-start helper: SOLVER_FREE_ALT_STARTS of + them, one by default.""" + + @pytest.mark.parametrize( + "alt_km,expected", + [ + (9.0, [9.0]), + (7.0, [7.0]), + (8.2, [9.0]), + (1.5, [1.5]), + # Off the ends of the ladder: still the nearest layer, not nothing. + (0.4, [1.5]), + (40.0, [11.0]), + ], + ) + def test_one_start_at_the_nearest_layer_by_default(self, alt_km, expected): + assert state.SOLVER_FREE_ALT_STARTS == 1 + assert solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) == expected + + def test_an_adsb_altitude_in_the_ladder_is_the_start(self): + """_solve_best_altitude splices a non-layer altitude (ADS-B) into the + layers, and the starts are taken over that spliced list — so the single + default start is that exact altitude, which is what the sweep would + have pinned too.""" + layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4])) + assert solver_mod._free_alt_starts(8.4, layers) == [8.4] @pytest.mark.parametrize( "alt_km,expected", @@ -94,18 +120,31 @@ class TestFreeAltStarts: (40.0, [7.0, 9.0, 11.0]), ], ) - def test_window_around_the_nearest_layer(self, alt_km, expected): + def test_three_starts_are_the_window_around_the_nearest_layer(self, alt_km, expected, monkeypatch): + """The pre-default behaviour, still reachable by configuration.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3) starts = solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) assert starts == expected - assert len(starts) == solver_mod._FREE_ALT_N_STARTS + assert len(starts) == 3 - def test_an_adsb_altitude_in_the_ladder_is_a_start(self): - """_solve_best_altitude splices an ADS-B altitude into the layers, and - the window is taken over that spliced list — otherwise the one exact - altitude available would never be started from.""" + def test_three_starts_window_the_spliced_adsb_altitude(self, monkeypatch): + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3) layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4])) assert solver_mod._free_alt_starts(8.4, layers) == [7.0, 8.4, 9.0] + @pytest.mark.parametrize("configured", [0, -3]) + def test_fewer_than_one_start_still_starts_somewhere(self, configured, monkeypatch): + """A count below one would leave the LM no start at all, so it clamps + rather than raises: a mis-set env degrades to a working solve.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", configured) + assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == [9.0] + + def test_more_starts_than_layers_is_every_layer(self, monkeypatch): + """The other clamp: a count past the end of the ladder would slice + short of it, quietly dropping starts that were asked for.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 99) + assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == solver_mod._SOLVER_ALT_LAYERS_KM + def test_no_layers_gives_no_starts(self): assert solver_mod._free_alt_starts(9.0, []) == [] @@ -135,10 +174,10 @@ def teardown_method(self): state.SOLVER_ALT_MODE = self._saved_mode super().teardown_method() - def test_one_multistart_call_with_three_starts(self): + def test_one_multistart_call_with_one_start(self): nodes = ["n1", "n2", "n3"] solve = _Recorder(lambda n, s, *r: pytest.fail("sweep ran in free mode")) - multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[1.2, 0.4, 0.9])) + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4])) result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart) @@ -146,7 +185,29 @@ def test_one_multistart_call_with_three_starts(self): assert solve.calls == [] assert len(multistart.calls) == 1 (_, _, rest) = multistart.calls[0] - assert rest == ([7.0, 9.0, 11.0],) + assert rest == ([9.0],) + + def test_an_adsb_guess_altitude_is_the_start(self): + """A non-layer initial_guess altitude is spliced into the ladder and + becomes the start itself — the free-mode analogue of the sweep's extra + layer, and the one exact altitude the candidate has.""" + nodes = ["n1", "n2", "n3"] + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free")) + + solver_mod._solve_best_altitude(_s_in(nodes, alt_km=8.437), {}, lambda s, c: None, multistart) + + assert multistart.calls[0][2] == ([8.437],) + + def test_the_start_count_is_configurable(self, monkeypatch): + """SOLVER_FREE_ALT_STARTS buys back the neighbour window for a geometry + whose single start lands on the wrong side of an ellipse.""" + monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3) + multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free")) + + solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, lambda s, c: None, multistart) + + assert len(multistart.calls) == 1 + assert multistart.calls[0][2] == ([7.0, 9.0, 11.0],) def test_n2_keeps_the_sweep(self): """Altitude is unobservable at n=2 — the free path is not entered even @@ -188,6 +249,24 @@ def test_history_carries_the_mode_and_the_per_start_residuals(self): assert rec["alt_starts_km"] == [5.0, 7.0, 9.0] assert rec["alt_start_rms_us"] == [1.234, None, 0.432] + def test_a_single_start_still_records_its_residual(self): + """The comparison channel does not depend on there being several + starts: one start records a one-element list, not a bare number or + nothing at all.""" + nodes = ["n1", "n2", "n3"] + multistart = _Recorder( + lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4321], alt_starts_km=[9.0]) + ) + solver_mod._process_solver_item( + (_s_in(nodes), {}, time.time()), + lambda s, c: pytest.fail("sweep ran in free mode"), + multistart_fn=multistart, + ) + + rec = state.mlat_solve_history[-1] + assert rec["alt_starts_km"] == [9.0] + assert rec["alt_start_rms_us"] == [0.432] + def test_z_saturation_reaches_the_history(self): nodes = ["n1", "n2", "n3"] multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", z_saturated=True)) diff --git a/docs/solverflow.md b/docs/solverflow.md index 50101e85..3908611b 100644 --- a/docs/solverflow.md +++ b/docs/solverflow.md @@ -81,7 +81,8 @@ own. Everything that reaches a solve passes through one gate stack | `FRAME_WORKERS` | 4 (compose sets 6) | `main.py:164`, `docker-compose.yml:54` | | `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py:31,67` | | `KNOWN_LANE_MODE` default | `binding` | `core/state.py:72-74` | -| `SOLVER_ALT_MODE` default | `sweep` | `core/state.py:77-95` | +| `SOLVER_ALT_MODE` default | `sweep` | `core/state.py:77-96` | +| `SOLVER_FREE_ALT_STARTS` default | 1 | `core/state.py:98-114` | --- @@ -456,7 +457,7 @@ flowchart TD m7 --> alt{"n_nodes >= 3?"} alt -->|"yes"| mode{"SOLVER_ALT_MODE"} mode -->|"sweep (default)"| sweep["_solve_best_altitude:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"] - mode -->|"free"| freealt["_solve_best_altitude:
ONE pool call to
solve_multinode_multistart,
3 start layers, z solved"] + mode -->|"free"| freealt["_solve_best_altitude:
ONE pool call to
solve_multinode_multistart,
SOLVER_FREE_ALT_STARTS start
layers (1 by default), z solved"] alt -->|"no, n=2"| single["_solve_best_altitude_n2:
one LM call at the
association altitude"] classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3 @@ -475,13 +476,26 @@ trimming (6.4) drops measurements that were never the problem. `free` instead calls `solve_multinode_multistart`, which runs the LM with altitude as a sixth unknown (state `[x, y, z, vx, vy, vz]`, z bounded -0.05–20 km, the `vz` bound unchanged) from three start layers — the one nearest -the association guess and its two neighbours — and keeps the lowest `rms_delay`. -Freeing z removes the ladder's quantisation but not the LM's locality, which is -what the several starts are still for. It is also cheaper: **one** process-pool +0.05–20 km, the `vz` bound unchanged) from `SOLVER_FREE_ALT_STARTS` start +layers, keeping the lowest `rms_delay`. It is also cheaper: **one** process-pool round trip per candidate instead of six, each of which pickles the node configs the input needs. +`SOLVER_FREE_ALT_STARTS` defaults to **1** — the layer nearest the association +guess, or the guess altitude itself when that came from ADS-B and was spliced +into the ladder (the same splice the sweep does). Freeing z removes the +ladder's quantisation but not the LM's locality, and extra starts are what +would stop a solve settling on the wrong side of a bistatic ellipse; on this +fleet's geometry they had almost nothing to stop. Over a 20-minute window of +1019 free-mode solves on test, the three starts' `rms_delay` differed by more +than 0.1 us in **13** of them, and the nearest-layer start was more than 0.5 us +worse than the best start in **2** — ~0.2% of solves helped, at three times the +solver CPU, while the pool is the binding constraint (~1.7 attempts/s against a +2.0 s average latency on two workers). Set it above 1 for a geometry where that +locality does bite; `_free_alt_starts` clamps it into `[1, len(layers)]` and +values above 1 give the same neighbour window as before, so `3` restores the +original behaviour exactly. + At n=2 the mode is inert — four residuals cannot support six unknowns, so the geolocator pins altitude regardless and `_solve_best_altitude_n2` is unchanged. Trimming re-solves through `_solve_best_altitude`, so a trim round inherits @@ -498,7 +512,7 @@ mode per environment and read the two lanes' `rms_delay` and `gt_error_km` off | Mode | Pool calls per n>=3 candidate | Altitude | |---|---|---| | `sweep` (default) | 6 (one per layer) | quantised to the nearest layer | -| `free` | 1 (three starts inside it) | solved, 0.05–20 km | +| `free` | 1 (`SOLVER_FREE_ALT_STARTS` starts inside it, 1 by default) | solved, 0.05–20 km | | Constant | Value | File:line | |---|---|---| From fcccd03fecce25f12857d6f14ba99fe0fef33440 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 08:47:55 +0000 Subject: [PATCH 12/15] Test stale-track exclusion and measurement epoch alignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sign of the Doppler-to-delay-rate conversion is the part of this that cannot be checked by reading it, so it is checked against the simulator's own _bistatic_delay / _bistatic_doppler: a target is flown in a straight line, sampled at two times, and the correction applied to the older sample must land on the newer sample's true delay. Three velocities (inbound, outbound, crossing-with-climb), plus a direct assertion that the correction and the true delay change share a sign, so a failure says "the sign is wrong" rather than "the error did not shrink enough". The staleness tests express the coast as a gap between the newest ASSOCIATED sample and the frame time, which is exactly what the tracker produces — get_recent_detections skips the None measurements mark_missed appends — and pin that TENTATIVE exclusion is unchanged and does not feed the stale counter, so the counter keeps meaning "an aircraft left this node's cone". Co-Authored-By: Claude Fable 5.1 --- backend/tests/test_epoch_alignment.py | 274 ++++++++++++++++++++++++++ backend/tests/test_frame_processor.py | 93 +++++++++ backend/tests/test_solver_stats.py | 4 + 3 files changed, 371 insertions(+) create mode 100644 backend/tests/test_epoch_alignment.py diff --git a/backend/tests/test_epoch_alignment.py b/backend/tests/test_epoch_alignment.py new file mode 100644 index 00000000..a805dc22 --- /dev/null +++ b/backend/tests/test_epoch_alignment.py @@ -0,0 +1,274 @@ +"""Measurement epoch alignment (SOLVER_EPOCH_ALIGN) — solver.align_measurement_epochs. + +The solver's residual model evaluates every measurement against ONE target +state, so a solver input is implicitly a claim that its measurements were +simultaneous. Nodes sample on independent free-running cadences, so the claim +is false by up to a frame interval, and the resulting delay error is charged to +the rms_delay gate. These tests pin the correction, and — more importantly — +pin its SIGN against the simulator's own geometry rather than against the +derivation the helper's comment gives, since a sign error there would silently +double the very error the correction exists to remove. +""" + +import pytest +from retina_simulation.world import _bistatic_delay, _bistatic_doppler + +from core import state +from services.tasks import solver as solver_mod +from services.tasks.solver import align_measurement_epochs + +_FC_HZ = 183e6 + +# One node's ENU geometry, km. Only fc_hz is read by the helper; the rest is +# here because the simulator's delay/Doppler helpers need a real bistatic +# triangle to produce numbers whose sign means anything. +_TX_ENU = (-20.0, 5.0, 0.05) +_RX_ENU = (0.0, 0.0, 0.3) + +_NODE_CFGS = { + "node-a": {"fc_hz": _FC_HZ}, + "node-b": {"fc_hz": _FC_HZ}, + "node-c": {"FC": _FC_HZ}, # the alternate spelling the geolocator accepts +} + + +def _s_in(measurements, **over): + base = { + "initial_guess": {"lat": 34.85, "lon": -82.4, "alt_km": 9.0}, + "measurements": measurements, + "n_nodes": len({m["node_id"] for m in measurements}), + "timestamp_ms": 1_700_000_000_000, + } + base.update(over) + return base + + +def _m(node_id, delay_us, doppler_hz, t_s, snr=15.0): + return { + "node_id": node_id, + "delay_us": delay_us, + "doppler_hz": doppler_hz, + "snr": snr, + "t_s": t_s, + } + + +@pytest.fixture(autouse=True) +def _zero_counter(): + state.solver_epoch_align_skipped = 0 + yield + + +class TestPureHelper: + def test_newest_measurement_is_the_epoch_and_is_untouched(self): + """t0 is the newest SAMPLE time, not the input's timestamp_ms: the + freshest node needed no correction and must not acquire one.""" + s_in = _s_in( + [ + _m("node-a", 40.0, 100.0, 1000.0), + _m("node-b", 50.0, -80.0, 1001.5), + _m("node-c", 60.0, 0.0, 1002.0), + ] + ) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + by_id = {m["node_id"]: m for m in out["measurements"]} + assert by_id["node-c"]["delay_us"] == 60.0 + assert meta["epoch_aligned"] is True + assert meta["epoch_skew_s"] == pytest.approx(2.0) + + def test_each_delay_moves_by_its_own_doppler_rate(self): + """d(delay_us)/dt = -doppler_hz * 1e6 / fc_hz, applied over that + measurement's own gap to t0.""" + s_in = _s_in( + [ + _m("node-a", 40.0, 100.0, 1000.0), + _m("node-b", 50.0, -80.0, 1001.5), + _m("node-c", 60.0, 0.0, 1002.0), + ] + ) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + by_id = {m["node_id"]: m for m in out["measurements"]} + assert by_id["node-a"]["delay_us"] == pytest.approx(40.0 + (-100.0 * 1e6 / _FC_HZ) * 2.0) + assert by_id["node-b"]["delay_us"] == pytest.approx(50.0 + (80.0 * 1e6 / _FC_HZ) * 0.5) + + def test_zero_doppler_measurement_is_unchanged(self): + """A tangential target's bistatic range is stationary, so no amount of + skew moves its delay — the rate is the only thing that can.""" + s_in = _s_in([_m("node-a", 40.0, 0.0, 1000.0), _m("node-b", 50.0, 20.0, 1004.0)]) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + assert out["measurements"][0]["delay_us"] == 40.0 + + def test_input_is_not_mutated(self): + """Pure: a caller must be able to drop the result and keep the + original, which is exactly what the flag-off path does.""" + meas = [_m("node-a", 40.0, 100.0, 1000.0), _m("node-b", 50.0, -80.0, 1002.0)] + s_in = _s_in(meas) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + assert s_in["measurements"][0]["delay_us"] == 40.0 + assert s_in["measurements"] is not out["measurements"] + assert s_in["timestamp_ms"] == 1_700_000_000_000 + + def test_timestamp_ms_is_restamped_to_the_epoch(self): + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), _m("node-b", 50.0, -80.0, 1002.25)]) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + assert out["timestamp_ms"] == 1_002_250 + + def test_fc_spelled_FC_is_accepted(self): + """Same fallback chain the geolocator uses to build its NodeSetup, so + a node aligns on exactly the carrier its solve predicts against.""" + s_in = _s_in([_m("node-c", 40.0, 100.0, 1000.0), _m("node-b", 50.0, 0.0, 1001.0)]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert meta["epoch_aligned"] is True + assert out["measurements"][0]["delay_us"] == pytest.approx(40.0 - 100.0 * 1e6 / _FC_HZ) + + def test_single_measurement_input_is_a_no_op(self): + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0)]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert out is s_in + assert meta == {"epoch_aligned": False} + assert state.solver_epoch_align_skipped == 0 + + +class TestSkipPath: + @pytest.mark.parametrize( + "broken", + [ + {"t_s": None}, + {"doppler_hz": None}, + ], + ) + def test_missing_field_skips_the_whole_input(self, broken): + """All-or-nothing: a partially aligned set has no marker saying which + measurements share an epoch, so it just relocates the error.""" + good = _m("node-a", 40.0, 100.0, 1000.0) + bad = {**_m("node-b", 50.0, -80.0, 1002.0), **broken} + s_in = _s_in([good, bad]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert out is s_in + assert meta == {"epoch_aligned": False} + assert state.solver_epoch_align_skipped == 1 + + def test_missing_t_s_key_entirely_skips(self): + """The pre-upgrade measurement shape: no t_s key at all.""" + untimed = {"node_id": "node-b", "delay_us": 50.0, "doppler_hz": -80.0, "snr": 9.0} + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), untimed]) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert out is s_in + assert meta["epoch_aligned"] is False + assert state.solver_epoch_align_skipped == 1 + + def test_unknown_node_config_skips(self): + s_in = _s_in([_m("node-a", 40.0, 100.0, 1000.0), _m("node-zzz", 50.0, -80.0, 1002.0)]) + out, meta = align_measurement_epochs(s_in, {"node-a": {"fc_hz": _FC_HZ}}) + assert out is s_in + assert meta["epoch_aligned"] is False + assert state.solver_epoch_align_skipped == 1 + + +class TestSignAgainstSimulatorGeometry: + """The sign check, run against the simulator's own delay/Doppler model. + + A target is flown in a straight line and sampled at two times using + _bistatic_delay / _bistatic_doppler. The older sample plus the correction + must land on the newer sample's true delay — which is a statement about the + sign of the Doppler-to-delay-rate conversion that no amount of algebra in a + comment can substitute for. + """ + + _POS0 = (10.0, 15.0, 9.0) # km ENU + _DT_S = 2.0 + + @staticmethod + def _truth(vel_kms, dt_s): + pos0 = TestSignAgainstSimulatorGeometry._POS0 + pos1 = tuple(pos0[i] + vel_kms[i] * dt_s for i in range(3)) + return ( + _bistatic_delay(pos0, _TX_ENU, _RX_ENU), + _bistatic_delay(pos1, _TX_ENU, _RX_ENU), + _bistatic_doppler(pos0, vel_kms, _TX_ENU, _RX_ENU, _FC_HZ), + ) + + @pytest.mark.parametrize( + "vel_kms", + [ + (-0.20, -0.15, 0.0), # inbound: bistatic range shrinking + (0.20, 0.15, 0.0), # outbound: bistatic range growing + (0.05, -0.24, 0.01), # mostly crossing, with a climb + ], + ) + def test_alignment_moves_the_stale_delay_toward_the_truth(self, vel_kms): + delay0, delay1, doppler0 = self._truth(vel_kms, self._DT_S) + + # node-a sampled _DT_S seconds ago; node-b is the newest sample and + # therefore defines t0. node-b's own numbers are irrelevant to the + # assertion — it is only here to set the epoch. + s_in = _s_in( + [ + _m("node-a", delay0, doppler0, 1000.0), + _m("node-b", 77.0, 0.0, 1000.0 + self._DT_S), + ] + ) + out, meta = align_measurement_epochs(s_in, _NODE_CFGS) + assert meta["epoch_aligned"] is True + aligned = out["measurements"][0]["delay_us"] + + err_before = abs(delay0 - delay1) + err_after = abs(aligned - delay1) + # A wrong sign would double err_before rather than shrink it, so the + # margin here is the sign test. Over 2 s of straight-line flight the + # first-order term dominates; the residual is the trajectory's + # curvature in bistatic range, not a modelling disagreement. + assert err_after < err_before * 0.2 + assert err_before > 0.05 # the case would prove nothing otherwise + + def test_correction_and_truth_share_a_sign(self): + """Stated directly, so a failure says 'the sign is wrong' rather than + 'the error did not shrink enough'.""" + vel_kms = (-0.20, -0.15, 0.0) + delay0, delay1, doppler0 = self._truth(vel_kms, self._DT_S) + s_in = _s_in( + [ + _m("node-a", delay0, doppler0, 1000.0), + _m("node-b", 77.0, 0.0, 1000.0 + self._DT_S), + ] + ) + out, _ = align_measurement_epochs(s_in, _NODE_CFGS) + correction = out["measurements"][0]["delay_us"] - delay0 + assert correction * (delay1 - delay0) > 0 + + +class TestProcessSolverItemWiring: + """The flag, and that the aligned numbers are what the solve actually sees.""" + + @staticmethod + def _item(): + s_in = _s_in( + [ + _m("node-a", 40.0, 300.0, 1000.0), + _m("node-b", 50.0, 0.0, 1004.0), + ] + ) + return (s_in, _NODE_CFGS, None) + + def test_flag_off_leaves_the_input_untouched(self, monkeypatch): + monkeypatch.setattr(state, "SOLVER_EPOCH_ALIGN", False) + seen = {} + + def _solve(s_in, node_cfgs): + seen["delays"] = [m["delay_us"] for m in s_in["measurements"]] + return None + + solver_mod._process_solver_item(self._item(), _solve) + assert seen["delays"] == [40.0, 50.0] + + def test_flag_on_hands_the_solver_aligned_delays(self, monkeypatch): + monkeypatch.setattr(state, "SOLVER_EPOCH_ALIGN", True) + seen = {} + + def _solve(s_in, node_cfgs): + seen["delays"] = [m["delay_us"] for m in s_in["measurements"]] + return None + + solver_mod._process_solver_item(self._item(), _solve) + assert seen["delays"][0] == pytest.approx(40.0 + (-300.0 * 1e6 / _FC_HZ) * 4.0) + assert seen["delays"][1] == 50.0 diff --git a/backend/tests/test_frame_processor.py b/backend/tests/test_frame_processor.py index c5d31e92..56a1409a 100644 --- a/backend/tests/test_frame_processor.py +++ b/backend/tests/test_frame_processor.py @@ -6,15 +6,19 @@ import queue import time +import types import pytest +from retina_tracker.track import TrackState from config.constants import GT_DISPLAY_STALE_S from core import state from pipeline.passive_radar import DEFAULT_NODE_CONFIG, PassiveRadarPipeline +from services import frame_processor from services.frame_processor import ( append_track_history, build_combined_aircraft_json, + confirmed_track_views, dedup_aircraft, flush_all_archive_buffers, get_node_configs, @@ -277,6 +281,95 @@ def test_claiming_anchored_inputs_reach_the_solver_queue(self, monkeypatch): # ── Multinode result conversion ────────────────────────────────────────────── +class TestConfirmedTrackViewsStaleness: + """TRACK_MAX_STALE_S: a coasting track stops being offered to association + once its newest REAL detection has aged out. + + The freshness signal is the newest entry get_recent_detections returns, + which by construction is an ASSOCIATED sample (mark_missed appends None to + history["measurements"] and the reverse scan skips those) — so these fakes + hand back only real detections, exactly as the tracker does, and the coast + is expressed as a gap between that newest sample and the frame time. + """ + + @staticmethod + def _track(newest_ts_ms: int, status=TrackState.COASTING, track_id="trk-stale"): + hist = [ + {"timestamp": newest_ts_ms - 1000, "delay": 40.0, "doppler": 5.0, "snr": 12.0, "adsb": None}, + {"timestamp": newest_ts_ms, "delay": 41.0, "doppler": 5.0, "snr": 12.0, "adsb": None}, + ] + return types.SimpleNamespace( + id=track_id, + state_status=status, + adsb_hex=None, + get_recent_detections=lambda n: hist[-n:], + ) + + def _tracker(self, *tracks): + return types.SimpleNamespace(tracks=list(tracks)) + + def test_five_second_old_coasting_track_is_excluded_at_three(self, monkeypatch): + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms - 5000)) + assert confirmed_track_views(tracker, now_ts_ms=now_ms) == [] + assert state.tracks_stale_skipped == 1 + + def test_same_track_is_included_at_ten(self, monkeypatch): + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 10.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms - 5000)) + views = confirmed_track_views(tracker, now_ts_ms=now_ms) + assert [v["track_id"] for v in views] == ["trk-stale"] + assert state.tracks_stale_skipped == 0 + + def test_zero_disables_the_filter(self, monkeypatch): + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 0.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms - 600_000)) + assert len(confirmed_track_views(tracker, now_ts_ms=now_ms)) == 1 + assert state.tracks_stale_skipped == 0 + + def test_no_frame_time_disables_the_filter(self, monkeypatch): + """The bench and the ADS-B-seeding tests call without a frame time; + wall clock is not a substitute, so those callers stay unfiltered.""" + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + tracker = self._tracker(self._track(0)) + assert len(confirmed_track_views(tracker)) == 1 + assert state.tracks_stale_skipped == 0 + + def test_fresh_track_survives_beside_a_stale_one(self, monkeypatch): + """Staleness is per track, not per tracker — the node keeps + contributing whatever it can still actually see.""" + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker( + self._track(now_ms - 5000, track_id="gone"), + self._track(now_ms - 500, track_id="here"), + ) + views = confirmed_track_views(tracker, now_ts_ms=now_ms) + assert [v["track_id"] for v in views] == ["here"] + assert state.tracks_stale_skipped == 1 + + def test_tentative_is_still_excluded_regardless_of_freshness(self, monkeypatch): + """The TENTATIVE filter is unchanged and independent: a brand-new + TENTATIVE track's newest detection is as fresh as it gets, and it must + still not reach association.""" + monkeypatch.setattr(frame_processor, "TRACK_MAX_STALE_S", 3.0) + state.tracks_stale_skipped = 0 + now_ms = 1_000_000 + tracker = self._tracker(self._track(now_ms, status=TrackState.TENTATIVE)) + assert confirmed_track_views(tracker, now_ts_ms=now_ms) == [] + # Skipped as TENTATIVE, not as stale — the counter must stay clean so + # it only ever means "an aircraft left this node's cone". + assert state.tracks_stale_skipped == 0 + + class TestMultinodeToAircraft: def test_basic_conversion(self): r = { diff --git a/backend/tests/test_solver_stats.py b/backend/tests/test_solver_stats.py index 67bc599c..35c80e99 100644 --- a/backend/tests/test_solver_stats.py +++ b/backend/tests/test_solver_stats.py @@ -257,6 +257,8 @@ def test_consensus_and_counters_reflect_state(self): state.solver_consensus_fallback = 9 state.solver_consensus_shadow = 10 state.solver_vel_untrusted_published = 11 + state.tracks_stale_skipped = 13 + state.solver_epoch_align_skipped = 14 out = _solver_window_stats(10.0) assert out["counters"] == { "successes": 5, @@ -265,6 +267,8 @@ def test_consensus_and_counters_reflect_state(self): "solver_trimmed": 3, "stale_drops": 4, "resolve_skips": 12, + "tracks_stale_skipped": 13, + "epoch_align_skipped": 14, "queue_drops": 6, "worker_errors": 0, "vel_untrusted_published": 11, From c66b55f2ca9b3e49ac4e5a09e3c10581ae60ff48 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 08:55:29 +0000 Subject: [PATCH 13/15] Bump retina-analytics: merge distance down to one grid step Picks up offworldlabs/retina-analytics@2b83871. The 6.0 / 4.5 / 3.0 km sweep on the association bench is monotone with the real-track count flat at every point, so 3.0 is taken: published contamination 32.0% -> 25.1%, foreign nodes per published solve 0.76 -> 0.52. Co-Authored-By: Claude Fable 5.1 --- libs/retina-analytics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/retina-analytics b/libs/retina-analytics index 7e5414ac..2b83871e 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 7e5414ac1671bb66c160a425363b1d19c889eea4 +Subproject commit 2b83871e38cb17d9a2e5ceda3a55ceeaf1492f42 From 6f84343ee842f189fa8e1b9ac09de897846d4c08 Mon Sep 17 00:00:00 2001 From: Jehan Azad Date: Sat, 5 Sep 2026 09:13:00 +0000 Subject: [PATCH 14/15] Bump retina-analytics: lint fix Picks up offworldlabs/retina-analytics@67f1488 (ruff-format only, no behaviour change). Co-Authored-By: Claude Fable 5.1 --- libs/retina-analytics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/retina-analytics b/libs/retina-analytics index 2b83871e..67f1488d 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 2b83871e38cb17d9a2e5ceda3a55ceeaf1492f42 +Subproject commit 67f1488d5d06dd99dfa247cd7deb92bb5b59e01b From 1cdb64754d8ed74140b8cee009a559a877bec308 Mon Sep 17 00:00:00 2001 From: "Claude_Fable_5.1" Date: Sun, 6 Sep 2026 02:26:50 +0000 Subject: [PATCH 15/15] geolocator: point the submodule at main now that free-altitude (#24) is merged there Co-Authored-By: Claude Fable 5.1 --- libs/retina-geolocator | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/retina-geolocator b/libs/retina-geolocator index b87b4707..6979943a 160000 --- a/libs/retina-geolocator +++ b/libs/retina-geolocator @@ -1 +1 @@ -Subproject commit b87b4707cecb331fea2a8895f3e63f256183f4a9 +Subproject commit 6979943aa7ae3d05e61a9cb355490eafcc8b45b3