diff --git a/backend/.env.example b/backend/.env.example
index 20a107b4..51a1c782 100644
--- a/backend/.env.example
+++ b/backend/.env.example
@@ -129,9 +129,28 @@ MENDER_PAT=
# from its association guess before it is rejected; the default is 6 km = 2 x
# the 3 km association grid step, since a dark guess is a quantised grid point
# rather than an ADS-B fix. ADS-B-anchored solves keep the fixed 2 km cap.
+#
+# SOLVER_ALT_MODE is how an n>=3 solve gets its altitude. sweep (default)
+# solves once per fixed altitude layer and keeps the lowest rms_delay — six
+# process-pool round trips per candidate, and an altitude quantised to a
+# ladder whose 2 km spacing puts up to 1 km of error straight into the
+# residual the reject gate reads. free makes one pool call to the geolocator's
+# multi-start helper, which solves altitude as a sixth unknown. Both modes
+# stamp altitude_mode on the solve-history record, so /api/test/mlat-history
+# can compare them across a deploy of each. See docs/solverflow.md.
+#
+# SOLVER_FREE_ALT_STARTS is how many start altitudes free mode gives that
+# helper, clamped to the number of layers. 1 (the default) starts at the layer
+# nearest the association guess, or at the guess altitude itself when it comes
+# from ADS-B; more is a window around it, at one LM run each. Measured on test
+# over 1019 free solves, three starts changed rms_delay by more than 0.1 us in
+# 13 of them, so the extra runs are off by default and worth turning on only
+# where the geometry sends a single start to the wrong side of an ellipse.
# SOLVER_WORKERS=2
# SOLVER_RESOLVE_INTERVAL_S=12
# SOLVER_MAX_DISPLACEMENT_KM_DARK=6.0
+# SOLVER_ALT_MODE=sweep
+# SOLVER_FREE_ALT_STARTS=1
# Detection mirror. Production only. Every accepted v1 detection frame is
# forwarded to another environment's /api/radar/detections/bulk, batched once a
diff --git a/backend/core/state.py b/backend/core/state.py
index 634aa212..64c0093e 100644
--- a/backend/core/state.py
+++ b/backend/core/state.py
@@ -74,6 +74,45 @@
if KNOWN_LANE_MODE not in ("off", "shadow", "binding"):
KNOWN_LANE_MODE = "shadow"
+# How the n>=3 solve gets its altitude (see services/tasks/solver.py's
+# _solve_best_altitude). sweep/free, read here rather than in that module so
+# it sits with its sibling mode flags and a test can monkeypatch it without
+# reimporting the solver.
+# sweep (default) — solve once per fixed altitude layer, keep the lowest
+# rms_delay. Six pool round trips per candidate, and an altitude
+# quantised to the ladder: layers are 2 km apart, so the pin is
+# systematically up to 1 km wrong and that error lands in the
+# residual the reject gate reads.
+# free — one pool call to retina_geolocator's multi-start helper, which
+# solves altitude as a sixth unknown, started from
+# SOLVER_FREE_ALT_STARTS of those layers.
+# Not off/shadow/active: there is no shadow here, because the two modes
+# produce the same shape of result and the history record carries
+# altitude_mode either way — running both would double the solver's cost to
+# learn what one deploy of each already says. An unrecognised value falls
+# back to "sweep", the same degrade-to-inert rule the sibling flags use.
+SOLVER_ALT_MODE = os.getenv("SOLVER_ALT_MODE", "sweep").lower()
+if SOLVER_ALT_MODE not in ("sweep", "free"):
+ SOLVER_ALT_MODE = "sweep"
+
+# How many start altitudes the free mode hands that helper. Read here beside
+# the mode it qualifies; _free_alt_starts in services/tasks/solver.py clamps it
+# into [1, len(layers)] against the ladder that module owns. 1 starts at the
+# layer nearest the association guess — where the sweep would have pinned;
+# more is a window around it.
+#
+# The default is 1 because three starts did not pay for themselves: over 1019
+# free-mode solves on test, the three starts' rms_delay differed by more than
+# 0.1 us in 13 of them, and the nearest-layer start was more than 0.5 us worse
+# than the best start in 2. That is ~0.2% of solves helped for 3x the solver
+# CPU, and the pool — not the altitude ladder — is what this deployment is
+# short of (~1.7 attempts/s against a 2.0 s average latency on two workers).
+# The knob stays because the reason for several starts is the LM's locality,
+# which is a property of the geometry rather than of this fleet: nodes lying
+# nearer a bistatic ellipse than these can send a single start to the wrong
+# side of it, and finding that out should not need a code change.
+SOLVER_FREE_ALT_STARTS = max(1, int(os.getenv("SOLVER_FREE_ALT_STARTS", "1")))
+
node_analytics = NodeAnalyticsManager(storage_dir=COVERAGE_STORAGE_DIR, fov_mode=FOV_MODE)
@@ -475,6 +514,15 @@ def _adsb_for_seeding() -> dict[str, dict]:
# Monotonic counter for dropped frames (useful for monitoring)
frames_dropped: int = 0
+# Frames the per-node rate limiter refused before they ever reached
+# frame_queue (tcp_handler's NODE_FRAME_MIN_INTERVAL_S gate). A different
+# event from frames_dropped, which is queue saturation: this one is the
+# pipeline deliberately sampling a node down to ~1 Hz, and a node streaming at
+# 22 fps therefore reports a large number here while dropping nothing. It was
+# uncounted, so "how much of a node's evidence does the tracker actually see"
+# had no answer at all — the frames_dropped that IS published
+# (/api/admin/metrics) says zero throughout.
+node_frames_rate_limited: int = 0
frames_processed: int = 0
solver_successes: int = 0
solver_failures: int = 0
@@ -543,6 +591,25 @@ def _adsb_for_seeding() -> dict[str, dict]:
# solver_stale_drops that means work was lost.
solver_resolve_skips: int = 0
+# The dark-lane share of the counter above, split out because the two lanes
+# read completely differently: an ADS-B-anchored duplicate that is skipped
+# costs nothing (the transponder keeps the track alive anyway), while a
+# skipped dark candidate may be the only chance that aircraft had of reaching
+# the map this window. Lane is decided by solver._is_dark_solver_input, the
+# same predicate routes.test._record_lane falls back to for a record that
+# never got a key — and a skip never gets one.
+solver_resolve_skips_dark: int = 0
+
+# The last few hundred resolve-slot skips, with the claims that blocked them.
+# Deliberately NOT the solve-history deque: a skip is not a solve outcome, and
+# writing one record per skip into mlat_solve_history would evict the real
+# records at roughly twice their rate (live: ~1 537 skips per 646 dark
+# attempts per 30 min). Small and separate, read by
+# /api/test/solver-stats' resolve_skips block and dumped by
+# /api/test/mlat-history?kind=resolve_skips. ~250 B/entry.
+SOLVER_RESOLVE_SKIPS_RECENT_MAX = 500
+solver_resolve_skips_recent: deque = deque(maxlen=SOLVER_RESOLVE_SKIPS_RECENT_MAX)
+
# Multinode entries removed because a later solve shared a source single-node
# track with them AND the spatial/identical-inputs guard in solver.py's
# _supersession_match agreed they are the same aircraft — the age-scaled
@@ -746,12 +813,14 @@ def _reset_for_tests() -> None:
global latest_mlat_accuracy_bytes, latest_mlat_verification_bytes
global latest_storage_bytes, simulation_config
global frames_dropped, frames_processed, solver_successes, solver_failures
+ global node_frames_rate_limited
global adsb_seed_frames_autotagged, adsb_capture_ts_fallback
global known_claims_made, known_claim_contentions, known_claims_bound
global known_claims_errors, known_claims_visibility_rejects, known_claims_world_rejects
global n2_unconfirmed, coverage_rebuilds, coverage_rebuild_nodes
global coverage_rebuild_backlog
global solver_queue_drops, solver_stale_drops, solver_resolve_skips
+ global solver_resolve_skips_dark
global mn_superseded, mn_superseded_blocked, solver_trimmed
global solver_consensus_selected, solver_consensus_filtered
global solver_consensus_fallback, solver_consensus_shadow
@@ -801,6 +870,7 @@ def _reset_for_tests() -> None:
track_archive_buffer.clear()
mlat_solve_history.clear()
mlat_solve_history_known.clear()
+ solver_resolve_skips_recent.clear()
accuracy_samples.clear()
mlat_samples.clear()
for q in (frame_queue, solver_queue):
@@ -832,7 +902,7 @@ def _reset_for_tests() -> None:
simulation_config = dict(_SIMULATION_CONFIG_DEFAULTS)
with counters_lock:
- frames_dropped = frames_processed = 0
+ frames_dropped = frames_processed = node_frames_rate_limited = 0
solver_successes = solver_failures = n2_unconfirmed = 0
adsb_seed_frames_autotagged = adsb_capture_ts_fallback = 0
known_claims_made = known_claim_contentions = known_claims_bound = 0
@@ -841,7 +911,7 @@ def _reset_for_tests() -> None:
coverage_rebuilds = coverage_rebuild_nodes = solver_queue_drops = 0
coverage_rebuild_backlog = 0
solver_stale_drops = 0
- solver_resolve_skips = 0
+ solver_resolve_skips = solver_resolve_skips_dark = 0
mn_superseded = mn_superseded_blocked = 0
solver_trimmed = 0
solver_consensus_selected = solver_consensus_filtered = 0
diff --git a/backend/routes/analytics.py b/backend/routes/analytics.py
index 07d83a9c..23225ccf 100644
--- a/backend/routes/analytics.py
+++ b/backend/routes/analytics.py
@@ -125,6 +125,14 @@ async def association_status():
return {
"registered_nodes": len(_a.node_geometries),
"overlap_zones": len(_a.overlap_zones),
+ # Node pairs that got no grid because the two nodes are in different
+ # worlds (node_world above). Read next to overlap_zones: on a fleet
+ # of 50 synthetic nodes over the same city as 8 receivers it is the
+ # 400 sim/real pairs whose grids could only ever have paired a
+ # simulated echo with a real one. Counted per pair considered, so it
+ # keeps rising as nodes re-register — zero means the fleet is single-
+ # world (or untagged), not that the gate is off.
+ "assoc_world_skipped_pairs": getattr(_a, "assoc_world_skipped_pairs", 0),
# Confirmed single-node tracks each node last submitted; these are what
# pairings are drawn from.
"pending_tracks": {nid: len(tracks) for nid, tracks in list(_a._pending_tracks.items())},
diff --git a/backend/routes/test.py b/backend/routes/test.py
index e4018b36..5909ff9d 100644
--- a/backend/routes/test.py
+++ b/backend/routes/test.py
@@ -752,11 +752,40 @@ def _record_lane(rec: dict) -> str:
return "adsb" if hexn and is_transponder_hex(hexn) else "dark"
+_LANES = ("dark", "known", "adsb")
+
+
+def _cap_per_lane(records: list[dict], limit: int) -> list[dict]:
+ """Keep the ``limit`` newest records OF EACH LANE, newest first.
+
+ ``records`` must already be newest-first. A single flat ``[:limit]``
+ made the cap a race between lanes rather than a retention rule, exactly
+ as the shared deque did before PR #289 split it: the known lane writes
+ ~16x the dark lane's volume, so a flat 1 000-record answer to a 30 min
+ request held only the newest ~6 min of dark records and the rest of the
+ window read as a quiet period. Capping per lane means known-lane volume
+ can never evict a dark record from a response.
+ """
+ kept: list[dict] = []
+ counts: dict[str, int] = {}
+ for r in records:
+ lane = _record_lane(r)
+ n = counts.get(lane, 0)
+ if n >= limit:
+ continue
+ counts[lane] = n + 1
+ kept.append(r)
+ return kept
+
+
@router.get("/api/test/mlat-history")
async def mlat_history(
hex: str | None = None,
all: int = 0,
minutes: float = 30.0,
+ lane: str = "all",
+ limit: int = 1000,
+ kind: str = "solves",
):
"""Per-solve MLAT history from the last ~30 minutes.
@@ -770,22 +799,73 @@ async def mlat_history(
merged here, so both lanes answer either query exactly as they did when
they shared a deque.
+ ``?lane=dark|known|adsb`` narrows the answer to one lane (default
+ ``all``, classified by ``_record_lane``); ``?limit=`` caps the record
+ list (default 1 000, max 5 000) and is applied PER LANE, so a known-lane
+ burst can never push dark records out of an ``all`` response — see
+ _cap_per_lane.
+
+ ``?kind=resolve_skips`` dumps a different store entirely: the solver's
+ recent resolve-slot refusals (state.solver_resolve_skips_recent), each
+ with the claims that blocked it. Those are not solve outcomes and
+ deliberately do not live in the solve-history deques.
+
``window_effective_minutes`` is how much of the requested window the
stores actually hold — below ``window_minutes`` the answer is truncated.
"""
+ if lane not in ("all", *_LANES):
+ return Response(
+ content=orjson.dumps({"error": f"lane must be one of all,{','.join(_LANES)}"}),
+ media_type="application/json",
+ status_code=400,
+ )
+ if kind not in ("solves", "resolve_skips"):
+ return Response(
+ content=orjson.dumps({"error": "kind must be solves or resolve_skips"}),
+ media_type="application/json",
+ status_code=400,
+ )
minutes = max(0.0, min(minutes, 35.0))
+ limit = max(1, min(int(limit), 5000))
cutoff_ms = int((time.time() - minutes * 60.0) * 1000)
+
+ if kind == "resolve_skips":
+ skips = [
+ s
+ for s in list(state.solver_resolve_skips_recent)
+ if s["ts_ms"] >= cutoff_ms and (lane == "all" or s["lane"] == lane)
+ ]
+ skips.reverse() # newest first
+ payload = {
+ "kind": "resolve_skips",
+ "window_minutes": minutes,
+ "lane": lane,
+ "lane_counts": {ln: sum(1 for s in skips if s["lane"] == ln) for ln in _LANES},
+ "n_records": len(skips),
+ "records": skips[:limit],
+ }
+ return Response(content=orjson.dumps(payload), media_type="application/json")
+
merged = _merged_solve_history()
effective_minutes = _window_effective_minutes(merged, minutes)
records = [r for r in merged if r["ts_ms"] >= cutoff_ms]
records.reverse() # newest first
+ if lane != "all":
+ records = [r for r in records if _record_lane(r) == lane]
+ lane_counts = dict.fromkeys(_LANES, 0)
+ for r in records:
+ lane_counts[_record_lane(r)] += 1
if all:
payload = {
"window_minutes": minutes,
"window_effective_minutes": effective_minutes,
+ "lane": lane,
+ # Pre-cap, so a truncated `records` can be read against what the
+ # window actually held.
+ "lane_counts": lane_counts,
"n_records": len(records),
- "records": records[:1000],
+ "records": _cap_per_lane(records, limit),
}
return Response(content=orjson.dumps(payload), media_type="application/json")
@@ -813,6 +893,8 @@ async def mlat_history(
"hex": norm,
"window_minutes": minutes,
"window_effective_minutes": effective_minutes,
+ "lane": lane,
+ "lane_counts": lane_counts,
"n_solves": len(solves),
"solves": solves[:500],
"rejects_nearby": {
@@ -914,6 +996,46 @@ def _solver_window_stats(minutes: float) -> dict:
reason = outcome[len("rejected_") :] if outcome.startswith("rejected_") else outcome
by_reason[reason] = by_reason.get(reason, 0) + 1
+ # ── cluster contamination (dark, windowed) ──────────────────────────────
+ # Of the dark records this window that matched ground truth, how many
+ # carried a node that could not see the aircraft they were matched to —
+ # the live version of the offline number Phase 2 exists to move (~60 %).
+ # Records without the stamp are records nothing could be asked about (no
+ # GT match, or no registered geometry for any contributing node) and stay
+ # out of the denominator rather than counting as clean; see
+ # solver._stamp_foreign_nodes.
+ judged = [r for r in records if r.get("foreign_node_ids") is not None]
+ contaminated = [r for r in judged if r.get("contaminated")]
+ n_judged = len(judged)
+ contamination = {
+ "records_with_gt": n_judged,
+ "contaminated": len(contaminated),
+ "pct": round(100.0 * len(contaminated) / n_judged, 1) if n_judged else None,
+ "foreign_nodes_per_record": (
+ round(sum(len(r["foreign_node_ids"]) for r in judged) / n_judged, 2) if n_judged else None
+ ),
+ }
+
+ # ── resolve-slot skips (windowed, from the skip deque) ──────────────────
+ # The counter in "counters" below is since-boot; these are the skips that
+ # happened inside this window, so they can be read against the attempts in
+ # the same window. attempts_ratio is (all-lane skips / DARK attempts) —
+ # the shape the acceptance target for the claim-on-publish fix is quoted
+ # in (live baseline ~1 537 / 646 = 2.4), not a per-lane rate. The dark
+ # numerator is published beside it for anyone who wants one.
+ all_skips = list(state.solver_resolve_skips_recent)
+ skips = [s for s in all_skips if s["ts_ms"] >= cutoff_ms]
+ resolve_skips = {
+ "total": len(skips),
+ "dark": sum(1 for s in skips if s["lane"] == "dark"),
+ "attempts_ratio": round(len(skips) / attempts, 3) if attempts else None,
+ # The skip deque is 500 entries against a live rate of ~50/min, so a
+ # long window IS truncated here even when the solve-history stores
+ # cover it. Same honesty rule as window_effective_minutes above: read
+ # it before reading total as a window count.
+ "window_effective_minutes": _window_effective_minutes(all_skips, minutes),
+ }
+
pos_errors.sort()
n_err = len(pos_errors)
median_err = pos_errors[n_err // 2] if n_err else None
@@ -1066,6 +1188,9 @@ def _solver_window_stats(minutes: float) -> dict:
"published": {"total": n2 + n3plus, "n2": n2, "n3plus": n3plus},
"rejects": {"total": reject_total, "by_reason": by_reason},
"position_error_km": {"median": median_err, "p90": p90_err, "n": n_err},
+ # Both windowed and both DARK-lane, like the funnel above them.
+ "contamination": contamination,
+ "resolve_skips": resolve_skips,
"ghosts": {
# Scoped to dark tracks: precision_pct's denominator is
# dark_tracks, and it is None (not 100.0) when there are none.
@@ -1192,7 +1317,15 @@ def _solver_window_stats(minutes: float) -> dict:
"solver_trimmed": state.solver_trimmed,
"stale_drops": state.solver_stale_drops,
"resolve_skips": state.solver_resolve_skips,
+ # Dark share of the line above. The windowed version, with the
+ # blocking claims, is the "resolve_skips" block further up.
+ "resolve_skips_dark": state.solver_resolve_skips_dark,
"queue_drops": state.solver_queue_drops,
+ # Frames the per-node rate limiter refused before the tracker ever
+ # saw them (tcp_handler's NODE_FRAME_MIN_INTERVAL_S). Not the
+ # same event as /api/admin/metrics' frames_dropped, which is
+ # frame_queue saturation and normally reads zero.
+ "node_frames_rate_limited": state.node_frames_rate_limited,
"worker_errors": state.solver_worker_errors,
"vel_untrusted_published": state.solver_vel_untrusted_published,
},
diff --git a/backend/services/frame_processor.py b/backend/services/frame_processor.py
index 0fb2bbd5..fe7b086d 100644
--- a/backend/services/frame_processor.py
+++ b/backend/services/frame_processor.py
@@ -156,6 +156,27 @@ def get_node_configs() -> dict[str, dict]:
return configs
+def configs_for_solver_input(node_cfgs: dict[str, dict], s_in: dict) -> dict[str, dict]:
+ """The subset of ``node_cfgs`` a solver input can actually reach.
+
+ The solver runs in a *spawn* process pool, so everything queued with an
+ input is pickled and shipped to a child on every call — and the fleet is
+ 58 nodes while a candidate carries 2-8 measurements. Sending the whole
+ set meant ~50 configs per solve that no code path could look at.
+
+ Nothing downstream needs the rest. The solver builds NodeSetups from the
+ measurements only; trimming and consensus both narrow that set further
+ (_filter_s_in_to_nodes) and never widen it; the beam gate iterates the
+ result's contributing_node_ids, which are measurement node ids by
+ construction; and cv_epochs is built from the same matched nodes as the
+ measurements in all three input shapes association emits. The known lane
+ is unaffected — it fetches its own configs (known_lane.run_known_lane_pass)
+ rather than reusing what was queued here.
+ """
+ wanted = {m.get("node_id") for m in (s_in.get("measurements") or ())}
+ return {nid: cfg for nid, cfg in node_cfgs.items() if nid in wanted}
+
+
# ── Per-node pipeline factory ─────────────────────────────────────────────────
@@ -427,7 +448,7 @@ def process_one_frame(node_id: str, frame: dict, default_pipeline: PassiveRadarP
if s_in["n_nodes"] < 2:
continue
try:
- state.solver_queue.put_nowait((s_in, node_cfgs, time.time()))
+ state.solver_queue.put_nowait((s_in, configs_for_solver_input(node_cfgs, s_in), time.time()))
except Exception:
state.bump_counter("solver_queue_drops")
if state.solver_queue_drops % 100 == 1:
diff --git a/backend/services/tasks/solver.py b/backend/services/tasks/solver.py
index 7c6b947c..a3a80205 100644
--- a/backend/services/tasks/solver.py
+++ b/backend/services/tasks/solver.py
@@ -11,6 +11,8 @@
from collections import deque
from concurrent.futures.process import BrokenProcessPool
+from retina_analytics.association import _point_in_beam
+
from config.constants import (
ARC_ONLY_ANOMALY_ALLOWLIST,
ASSOC_GRID_STEP_KM,
@@ -100,6 +102,24 @@ def _pool_call(fn, *args):
return fn(*args)
+def _pool_solve_multistart(s_in, node_cfgs, alt_starts_km):
+ """solve_multinode_multistart via the process pool (inline when none).
+
+ Defined here rather than beside _pool_solve_multinode at the foot of this
+ module for the reason _pool_select_consensus is: it is a default argument
+ value, resolved when the ``def`` executes, so it has to be bound before
+ _solve_best_altitude's signature is reached.
+
+ A module-level function taking only picklable arguments, because the pool
+ is a *spawn* pool — a child imports retina_geolocator and nothing of the
+ backend, so what crosses is this function's qualified name plus the input
+ dicts.
+ """
+ from retina_geolocator.multinode_solver import solve_multinode_multistart
+
+ return _pool_call(solve_multinode_multistart, s_in, node_cfgs, alt_starts_km, True)
+
+
# Altitude layers (km) tried when n_nodes ≥ 3. For an overdetermined system
# (3+ delay equations, 2 unknowns after altitude pinning) only the correct
# altitude layer yields rms_delay ≈ 0; wrong layers give rms > 0, so picking
@@ -320,17 +340,82 @@ def _sweep_altitudes(s_in: dict, node_cfgs: dict, solve_fn, layers_km: list[floa
return best_result
-def _solve_best_altitude(s_in: dict, node_cfgs: dict, solve_fn) -> dict | None:
- """Altitude sweep for n≥3: pick by minimum rms_delay.
+# Fewest measurements the free mode is used at. Below this altitude is not
+# observable and retina_geolocator pins it anyway; the sweep is left in place
+# so the n=2 path keeps its documented behaviour exactly.
+_FREE_ALT_MIN_NODES = 3
+
+
+def _free_alt_starts(ig_alt_km, layers: list[float]) -> list[float]:
+ """The start altitudes the free mode hands the multi-start helper.
+
+ state.SOLVER_FREE_ALT_STARTS of them, clamped into [1, len(layers)] — read
+ per call, like the mode flag, so a test and a config reload both see what
+ they set. One start is the layer nearest ``ig_alt_km``, which is the
+ altitude spliced into ``layers`` when the input carries a non-layer one of
+ its own (ADS-B), exactly as the sweep treats it. Several are a window
+ centred on that layer, clamped to the ends of the ladder so the count never
+ shrinks there (the top and bottom layers are where a wrong start is least
+ recoverable, not most).
+
+ Freeing z removes the ladder's quantisation but not the LM's locality, and
+ the extra starts are what would stop a solve settling on the wrong side of
+ a bistatic ellipse. On this fleet's geometry they had almost nothing to
+ stop: over 1019 free-mode solves on test, three starts' rms_delay differed
+ by more than 0.1 µs in 13 of them, and the nearest-layer start was more
+ than 0.5 µs worse than the best in 2 — so the default is one start and the
+ other two are bought explicitly, by a deployment whose geometry shows it
+ needs them. See core/state.py for the numbers and the trade.
+ """
+ if not layers:
+ return []
+ n = max(1, min(int(state.SOLVER_FREE_ALT_STARTS), len(layers)))
+ alt = float(ig_alt_km) if ig_alt_km is not None else 7.0
+ nearest = min(range(len(layers)), key=lambda i: abs(layers[i] - alt))
+ lo = max(0, min(nearest - (n - 1) // 2, len(layers) - n))
+ return layers[lo : lo + n]
+
- If the initial_guess already carries an ADS-B altitude (not one of the fixed
- grid layers), include it in the sweep so the correct exact altitude is tried.
+def _solve_best_altitude(
+ s_in: dict,
+ node_cfgs: dict,
+ solve_fn,
+ multistart_fn=_pool_solve_multistart,
+) -> dict | None:
+ """Altitude for n≥3, by whichever rule state.SOLVER_ALT_MODE names.
+
+ sweep (default): solve once per layer, pick by minimum rms_delay. If the
+ initial_guess already carries an ADS-B altitude (not one of the fixed grid
+ layers), include it in the sweep so the correct exact altitude is tried.
+
+ free: one call to the multi-start helper, which solves altitude as a sixth
+ unknown from _free_alt_starts. The sweep cannot do better than half its
+ 2 km layer spacing, and on noise-free replay of this fleet's geometry that
+ quantisation alone left rms_delay at a 1.76 µs median against the 3.0 µs
+ reject gate — spending most of the gate's budget on an altitude the
+ measurements themselves determine, and provoking _trim_and_resolve to drop
+ nodes that were never the problem. Costs one pool round trip per
+ candidate instead of six.
+
+ The mode is read per call rather than captured at import, so a test (and a
+ live config reload) sees the value it set. Read here and not inside
+ _process_solver_item because _trim_and_resolve re-enters through this same
+ function: a trim must re-solve under the mode its first solve used, or the
+ residuals it is comparing are not the same quantity.
"""
ig_alt = s_in.get("initial_guess", {}).get("alt_km")
if ig_alt is not None and ig_alt not in _SOLVER_ALT_LAYERS_KM:
layers = sorted(set(_SOLVER_ALT_LAYERS_KM + [round(float(ig_alt), 3)]))
else:
layers = _SOLVER_ALT_LAYERS_KM
+ n_meas = len({m.get("node_id") for m in (s_in.get("measurements") or [])})
+ if state.SOLVER_ALT_MODE == "free" and n_meas >= _FREE_ALT_MIN_NODES:
+ # No fall back to the sweep when this returns None: a helper that got
+ # no solve out of its starts is reporting the same thing the sweep
+ # reports when every layer fails, and sweeping anyway would cost the
+ # six round trips this mode exists to avoid on exactly the candidates
+ # that are least likely to repay them.
+ return multistart_fn(s_in, node_cfgs, _free_alt_starts(ig_alt, layers))
return _sweep_altitudes(s_in, node_cfgs, solve_fn, layers, "rms_delay")
@@ -386,6 +471,7 @@ def _trim_and_resolve(
node_cfgs: dict,
solve_fn,
result: dict,
+ multistart_fn=_pool_solve_multistart,
) -> tuple[dict, dict, dict | None]:
"""Drop the worst-residual node(s) and re-solve, down to _TRIM_MIN_NODES.
@@ -395,6 +481,11 @@ def _trim_and_resolve(
so re-solving on the survivors after dropping the offending node recovers
a solve the blanket gate would otherwise discard outright.
+ Re-solves through _solve_best_altitude, so it inherits whichever altitude
+ mode is in force — the loop compares this round's rms against the previous
+ round's, and mixing a swept altitude with a free one would make that
+ comparison meaningless.
+
Returns (final_result, final_s_in, trim_meta). trim_meta is None only
when no round ever produced a successful re-solve — i.e. no trimming was
actually performed — never when trimming ran but rms stayed high (that
@@ -439,7 +530,7 @@ def _trim_and_resolve(
s_next = _filter_s_in_to_nodes(s_in, survivors)
try:
- new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn)
+ new_result = _solve_best_altitude(s_next, node_cfgs, solve_fn, multistart_fn)
except Exception:
logging.exception("Solver trim re-solve failed")
break
@@ -1036,6 +1127,64 @@ def _claim_resolve_slot(s_in, now_s: float) -> bool:
return True
+def _resolve_slot_blockers(track_ids, now_s: float) -> list[dict]:
+ """The live claims covering ``track_ids``, for a skip record.
+
+ Read-only, and taken after the refusal rather than during it: the check
+ itself must stay one atomic test-and-claim, and a skip is rare enough
+ (relative to the queue drain rate) that a second lock acquisition on that
+ path costs nothing. Any claim that moves between the two is a claim the
+ diagnosis would have wanted to name anyway.
+ """
+ cutoff = now_s - _SOLVER_RESOLVE_INTERVAL_S
+ out: list[dict] = []
+ with _RECENT_SOLVES_LOCK:
+ for tid in track_ids:
+ held = _RECENT_SOLVES.get(tid)
+ if held is not None and held[0] > cutoff:
+ out.append({"track_id": tid, "held_ts": round(held[0], 3), "held_n": held[1]})
+ return out
+
+
+def _record_resolve_skip(s_in, now_s: float, blocking: list[dict] | None = None) -> None:
+ """Count and remember one resolve-slot refusal.
+
+ The counter alone could not answer the question the suppression rule
+ raises — *whose* claim blocked this, and was it even the same aircraft.
+ Live on the test droplet the rule refuses ~1 537 candidates per 646 dark
+ attempts per 30 min, and nothing recorded which claim did it, so a skip
+ that suppressed a genuinely different aircraft (tracker track ids are
+ shared across candidates — see _supersession_match) was indistinguishable
+ from one that suppressed a duplicate. The deque carries the blocking
+ claims and the candidate's own guess position so the two can be told apart
+ after the fact.
+
+ Deliberately NOT a solve-history record: skips outrun real dark records
+ roughly two to one, and writing them into that deque would evict the
+ solves the same investigation needs (see state.solver_resolve_skips_recent).
+ """
+ s = s_in if isinstance(s_in, dict) else {}
+ track_ids = list(s.get("track_ids") or [])
+ dark = _is_dark_solver_input(s)
+ state.bump_counter("solver_resolve_skips")
+ if dark:
+ state.bump_counter("solver_resolve_skips_dark")
+ ig = s.get("initial_guess") or {}
+ state.solver_resolve_skips_recent.append(
+ {
+ "ts_ms": int(now_s * 1000),
+ # No key is minted for a candidate that never solves, so lane is
+ # the same fallback routes.test._record_lane uses for a reject.
+ "lane": "dark" if dark else "adsb",
+ "track_ids": track_ids,
+ "n_nodes": int(s.get("n_nodes") or 0),
+ "blocking": _resolve_slot_blockers(track_ids, now_s) if blocking is None else blocking,
+ "guess_lat": round(float(ig["lat"]), 6) if ig.get("lat") else None,
+ "guess_lon": round(float(ig["lon"]), 6) if ig.get("lon") else None,
+ }
+ )
+
+
# Which single-node track pair currently owns a published n=2 track, and how
# well it fitted. One track is one aircraft, so two pairings sharing a track
# are mutually exclusive; the better chi2 wins and the loser is withheld.
@@ -1384,6 +1533,59 @@ def _is_dark_solver_input(s_in) -> bool:
return not (hx and is_transponder_hex(hx))
+def _stamp_foreign_nodes(rec: dict) -> None:
+ """Stamp which of a dark record's own nodes could not see the aircraft.
+
+ Cluster contamination is the dark lane's largest known defect — a
+ candidate assembled by format_track_pairs_for_solver can carry a node
+ whose track belongs to a *different* aircraft, and the solver then fits a
+ geometry no single aircraft ever occupied. Offline the audit measured it
+ at ~60 % of dark candidates; this makes the same number live.
+
+ The test is the associator's own visibility predicate applied whole
+ (retina_analytics.association._point_in_beam against the registered
+ NodeGeometry), which is the same gate known-lane claiming uses — claiming
+ and the dark lane must mean the same thing by "this node can see there",
+ and a second bespoke rule here would let the two disagree. Two
+ consequences worth knowing: it is a ground-projected bearing/footprint
+ test with no altitude term, and under FOV_MODE=active it is the learned
+ FOV rather than the theoretical wedge. Both are exactly what the rest of
+ the pipeline believes about coverage, which is the point.
+
+ Position is the matched ground-truth point already stamped on the record
+ (gt_lat/gt_lon at the solve epoch), so this costs no extra trail lookup —
+ only one cone test per contributing node. Nodes trimmed out by
+ _trim_and_resolve are included: a node dropped for a bad residual is
+ precisely the contamination this measures, and leaving it out would hide
+ every case trimming already rescued.
+
+ A node with no registered geometry is not judged either way. When that
+ leaves nothing judgeable the record is left unstamped rather than stamped
+ clean, so contamination_pct never counts an abstention as innocence.
+ """
+ lat, lon = rec.get("gt_lat"), rec.get("gt_lon")
+ if lat is None or lon is None:
+ return
+ node_ids = list(rec.get("contributing_node_ids") or [])
+ node_ids += [nid for nid in (rec.get("trimmed_node_ids") or []) if nid not in node_ids]
+ if not node_ids:
+ return
+ geometries = state.node_associator.node_geometries
+ judged = 0
+ foreign: list[str] = []
+ for nid in node_ids:
+ geo = geometries.get(nid)
+ if geo is None:
+ continue
+ judged += 1
+ if not _point_in_beam(lat, lon, geo):
+ foreign.append(nid)
+ if not judged:
+ return
+ rec["foreign_node_ids"] = foreign
+ rec["contaminated"] = bool(foreign)
+
+
def _record_dark_accuracy_sample(rec: dict) -> None:
"""Offer one published DARK solve to the rolling accuracy store.
@@ -1460,7 +1662,14 @@ def _record_solve_history(
``extra`` merges caller-supplied fields (trim metadata, beam-rejection
diagnostics) into the record. Applied before the GT stamp so it can
- never clobber gt_hex/gt_error_km/gt_lat/gt_lon.
+ never clobber gt_hex/gt_error_km/gt_lat/gt_lon — and so the trimmed node
+ ids it carries are in hand for the contamination stamp below.
+
+ ``foreign_node_ids``/``contaminated`` are stamped on DARK records that
+ matched ground truth: which of this candidate's own nodes could not see
+ the aircraft it was matched to (see _stamp_foreign_nodes). Absent on
+ every other record, which is what /api/test/solver-stats' contamination
+ block counts as "not judged" rather than as clean.
"""
r = result if isinstance(result, dict) else {}
s = s_in if isinstance(s_in, dict) else {}
@@ -1574,6 +1783,11 @@ def _record_solve_history(
rec["vel_err_ms"] = round(math.hypot(ve - gt_ve, vn - gt_vn), 1)
else:
rec["vel_err_ms"] = None
+ # Live cluster-contamination metric, dark lane only and only where ground
+ # truth actually matched — without a truth position there is nothing to
+ # ask "could this node see it?" about. See _stamp_foreign_nodes.
+ if _dark and rec.get("gt_hex"):
+ _stamp_foreign_nodes(rec)
if rec["outcome"] == "published" and _dark and rec.get("gt_error_km") is not None:
_record_dark_accuracy_sample(rec)
# Route by lane: the known lane's per-hex-per-pass volume would otherwise
@@ -1693,7 +1907,12 @@ def fov_gate_verdict(fov, n_nodes: int, brg: float, dist_km: float, range_rule_p
return range_rule_pass or fov_pass
-def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus) -> dict | None:
+def _process_solver_item(
+ item: tuple,
+ solve_fn,
+ select_fn=_pool_select_consensus,
+ multistart_fn=_pool_solve_multistart,
+) -> dict | None:
"""Process a single solver queue entry. Returns the solver result (or None).
Extracted from the worker loop so the success/failure/latency bookkeeping
@@ -1704,6 +1923,10 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
initial_guess and _CONSENSUS_MODE != "off" — n=2 (mirror-disambiguation
is the displacement/beam gates' job, not consensus's) and detection-level
inputs (no initial_guess to pin an altitude with) never call it.
+
+ multistart_fn is the free-altitude solve (_pool_solve_multistart by
+ default; tests substitute a stub), reached only when
+ state.SOLVER_ALT_MODE is "free" — see _solve_best_altitude.
"""
s_in, node_cfgs = item[0], item[1]
enqueued_at: float | None = item[2] if len(item) > 2 else None
@@ -1724,8 +1947,9 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
# here rather than at enqueue: the frame path must not carry solver state,
# and a copy that queued before its twin was solved can only be recognised
# once it reaches a worker.
- if not _claim_resolve_slot(s_in, time.time()):
- state.bump_counter("solver_resolve_skips")
+ _now_s = time.time()
+ if not _claim_resolve_slot(s_in, _now_s):
+ _record_resolve_skip(s_in, _now_s)
return None
n_nodes = s_in.get("n_nodes", 0) if isinstance(s_in, dict) else 0
consensus_meta: dict | None = None
@@ -1736,7 +1960,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
if _CONSENSUS_MODE != "off":
s_in, consensus_meta = _consensus_select(s_in, node_cfgs, select_fn)
n_nodes = s_in.get("n_nodes", n_nodes)
- result = _solve_best_altitude(s_in, node_cfgs, solve_fn)
+ result = _solve_best_altitude(s_in, node_cfgs, solve_fn, multistart_fn)
else:
result = _solve_best_altitude_n2(s_in, node_cfgs, solve_fn)
except Exception:
@@ -1762,7 +1986,7 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
and (result.get("rms_delay") or 0) > _SOLVER_RMS_DELAY_MAX_US
and result.get("per_node_delay_res_us")
):
- result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result)
+ result, s_in, trim_meta = _trim_and_resolve(s_in, node_cfgs, solve_fn, result, multistart_fn)
n_nodes = result.get("n_nodes", n_nodes)
# Built once and threaded through every history record below
@@ -1771,6 +1995,20 @@ def _process_solver_item(item: tuple, solve_fn, select_fn=_pool_select_consensus
_extra: dict | None = dict(trim_meta) if trim_meta else {}
if consensus_meta is not None:
_extra["consensus_meta"] = consensus_meta
+ # How this solve got its altitude, and — in free mode — what each
+ # start altitude fitted to. Stamped on every record, published or
+ # rejected, and in BOTH modes (the sweep's solves report
+ # altitude_mode "pinned"), because the only way to judge SOLVER_ALT_MODE
+ # live is to compare the two lanes' rms_delay and gt_error_km over the
+ # same history buffer. The per-start list is what says whether the
+ # three starts were worth keeping or one would have done.
+ if result.get("altitude_mode"):
+ _extra["altitude_mode"] = result["altitude_mode"]
+ if result.get("rms_by_start") is not None:
+ _extra["alt_starts_km"] = result.get("alt_starts_km")
+ _extra["alt_start_rms_us"] = [None if v is None else round(float(v), 3) for v in result["rms_by_start"]]
+ if result.get("z_saturated"):
+ _extra["z_saturated"] = True
_extra = _extra or None
rms_delay = result.get("rms_delay", 0) or 0
diff --git a/backend/services/tcp_handler.py b/backend/services/tcp_handler.py
index bbe82197..6b21be77 100644
--- a/backend/services/tcp_handler.py
+++ b/backend/services/tcp_handler.py
@@ -529,6 +529,12 @@ def _enqueue_detection(msg: dict, node_id: str | None):
if node_id:
last = _per_node_last_enqueue.get(node_id, 0.0)
if (now_m - last) < _NODE_MIN_INTERVAL_S:
+ # Counted, not silent: this is the only place a node's detections
+ # are discarded on purpose, and until now nothing said how many.
+ # state.frames_dropped is the queue-saturation counter and reads
+ # zero throughout, so "the tracker sees every frame this node
+ # sent" looked true from every published metric.
+ state.bump_counter("node_frames_rate_limited")
return # position already updated; skip expensive queue work
_per_node_last_enqueue[node_id] = now_m
diff --git a/backend/tests/test_adsb_seed_backend.py b/backend/tests/test_adsb_seed_backend.py
index cba3bb36..77b3fc86 100644
--- a/backend/tests/test_adsb_seed_backend.py
+++ b/backend/tests/test_adsb_seed_backend.py
@@ -605,3 +605,32 @@ def test_associator_gets_the_state_world_resolver(self):
must consult the same resolver claiming and the auto-tag filter use,
or one consumer accepts what another rejects."""
assert state.node_associator.node_world_provider is state.node_world
+
+ def test_a_sim_and_a_real_node_over_one_footprint_get_no_overlap_zone(self):
+ """The same resolver, one level down: bottom-up pairing must not build
+ a grid across worlds either. Registering a synthetic node and a
+ hardware node on overlapping coverage used to leave a zone whose only
+ possible pairing was a simulated echo against a real one — which is how
+ real node ids reached the synthetic fleet's dark solves."""
+ _a = state.node_associator
+ try:
+ _a.register_node("synth-GVL-9001", dict(_NODE_CFG))
+ _a.register_node("hw-9001", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36))
+
+ assert _a.overlap_zones == {}
+ assert _a._neighbors.get("synth-GVL-9001", set()) == set()
+ assert _a.assoc_world_skipped_pairs == 1
+ finally:
+ state._reset_for_tests()
+
+ def test_two_synthetic_nodes_over_one_footprint_still_pair(self):
+ """The gate is the world difference, not the registration."""
+ _a = state.node_associator
+ try:
+ _a.register_node("synth-GVL-9001", dict(_NODE_CFG))
+ _a.register_node("synth-GVL-9002", dict(_NODE_CFG, rx_lat=34.86, rx_lon=-82.36))
+
+ assert _a.overlap_zones
+ assert _a.assoc_world_skipped_pairs == 0
+ finally:
+ state._reset_for_tests()
diff --git a/backend/tests/test_analytics_routes.py b/backend/tests/test_analytics_routes.py
index 22a821e0..73c691d4 100644
--- a/backend/tests/test_analytics_routes.py
+++ b/backend/tests/test_analytics_routes.py
@@ -129,6 +129,18 @@ def test_status_returns_expected_fields(self, client):
assert "overlap_zones" in body
assert "overlaps" in body
+ def test_status_reports_world_skipped_pairs(self, client):
+ """The world gate on overlap zones is otherwise invisible: a fleet
+ whose sim/real pairs are being refused looks exactly like a fleet whose
+ pairs never overlapped, and only this counter separates them."""
+ _a = state.node_associator
+ _a.assoc_world_skipped_pairs += 7
+ try:
+ body = client.get("/api/radar/association/status").json()
+ assert body["assoc_world_skipped_pairs"] == 7
+ finally:
+ state._reset_for_tests()
+
def test_status_includes_claiming_block(self, client):
"""Top-down claiming (ASSOC_CLAIM_MODE) since boot — off by default
in tests, so this pins the shape rather than any particular mode."""
diff --git a/backend/tests/test_mlat_history.py b/backend/tests/test_mlat_history.py
index e03d9da4..e8e91bbe 100644
--- a/backend/tests/test_mlat_history.py
+++ b/backend/tests/test_mlat_history.py
@@ -777,3 +777,291 @@ def test_known_lane_records_are_not_sampled_by_this_path(self):
extra={"known_lane": True, "label": "truth_match", "published": True},
)
assert not state.accuracy_samples
+
+
+def _register_geo(node_id, beam_azimuth_deg, rx_lat=LAT, rx_lon=LON, max_range_km=50.0):
+ """Register one node geometry with the associator, aimed as given.
+
+ The contamination stamp asks the associator's own visibility predicate,
+ so a test node has to exist there rather than in a config dict.
+ """
+ from retina_analytics.association import NodeGeometry
+
+ geo = NodeGeometry(
+ node_id=node_id,
+ rx_lat=rx_lat,
+ rx_lon=rx_lon,
+ rx_alt_km=0.0,
+ tx_lat=rx_lat + 0.5,
+ tx_lon=rx_lon + 0.5,
+ tx_alt_km=0.3,
+ beam_azimuth_deg=beam_azimuth_deg,
+ beam_width_deg=41.0,
+ max_range_km=max_range_km,
+ )
+ state.node_associator.node_geometries[node_id] = geo
+ return geo
+
+
+class TestForeignNodeStamp:
+ """A dark record matched to ground truth says which of its own nodes
+ could not have seen that aircraft.
+
+ Cluster contamination — a solver candidate assembled from tracks of two
+ different aircraft — is the dark lane's largest known defect, and until
+ now it was measurable only offline. The verdict is the associator's own
+ visibility predicate, the same one known-lane claiming gates on.
+ """
+
+ def setup_method(self):
+ state._reset_for_tests()
+ solver_mod._reset_for_tests()
+
+ def teardown_method(self):
+ solver_mod._reset_for_tests()
+
+ def _run(self, contributing=("n_in", "n_out"), **extra):
+ return solver_mod._process_solver_item(
+ (dict(_CONFIRMED_N2), {}, time.time()),
+ _solve_fn(contributing_node_ids=list(contributing), **extra),
+ )
+
+ def test_a_node_aimed_away_is_named_foreign(self):
+ # Ground truth sits due north of both nodes; n_in is aimed at it and
+ # n_out at the opposite bearing.
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=180.0)
+ self._run()
+ rec = state.mlat_solve_history[0]
+ assert rec["gt_hex"] == "abc123"
+ assert rec["foreign_node_ids"] == ["n_out"]
+ assert rec["contaminated"] is True
+
+ def test_all_nodes_in_cone_is_not_contaminated(self):
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=10.0)
+ self._run()
+ rec = state.mlat_solve_history[0]
+ assert rec["foreign_node_ids"] == []
+ assert rec["contaminated"] is False
+
+ def test_a_node_out_of_range_is_foreign(self):
+ """Range, not only bearing: the predicate applies whole."""
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=0.0, max_range_km=1.0)
+ self._run()
+ assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_out"]
+
+ def test_trimmed_nodes_are_judged_too(self):
+ """A node dropped by _trim_and_resolve is exactly the contamination
+ this measures — excluding it would hide every case trimming already
+ rescued."""
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_trimmed", beam_azimuth_deg=180.0)
+ solver_mod._record_solve_history(
+ "published",
+ dict(_CONFIRMED_N2),
+ {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2, "contributing_node_ids": ["n_in"]},
+ solve_key="mn-dark-1",
+ raw_lat=LAT,
+ raw_lon=LON,
+ extra={"trimmed_node_ids": ["n_trimmed"], "trim_rounds": 1},
+ )
+ assert state.mlat_solve_history[0]["foreign_node_ids"] == ["n_trimmed"]
+
+ def test_no_ground_truth_means_no_stamp(self):
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=180.0)
+ self._run()
+ rec = state.mlat_solve_history[0]
+ assert "foreign_node_ids" not in rec
+ assert "contaminated" not in rec
+
+ def test_unregistered_nodes_are_not_stamped_clean(self):
+ """Nothing judgeable is an abstention, not innocence."""
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ self._run()
+ rec = state.mlat_solve_history[0]
+ assert rec["gt_hex"] == "abc123"
+ assert "foreign_node_ids" not in rec
+
+ def test_an_adsb_record_is_not_stamped(self):
+ """Dark lane only — the tagged lane's identity is not in doubt."""
+ _put_gt(lat=LAT + 0.05, lon=LON)
+ _register_geo("n_in", beam_azimuth_deg=0.0)
+ _register_geo("n_out", beam_azimuth_deg=180.0)
+ s_in = dict(_CONFIRMED_N2, adsb_hex="abc123")
+ solver_mod._process_solver_item(
+ (s_in, {}, time.time()),
+ _solve_fn(contributing_node_ids=["n_in", "n_out"]),
+ )
+ assert "foreign_node_ids" not in state.mlat_solve_history[0]
+
+
+class TestLaneFilterAndPerLaneCap:
+ """?lane= and ?limit= on /api/test/mlat-history.
+
+ The flat records[:1000] cap made the response a race between lanes: the
+ known lane writes ~16x the dark lane's volume, so a 30 min request held
+ only the newest ~6 min of dark records and the rest of the window read as
+ a quiet period. The cap is now per lane.
+ """
+
+ def setup_method(self):
+ state._reset_for_tests()
+ solver_mod._reset_for_tests()
+
+ def teardown_method(self):
+ solver_mod._reset_for_tests()
+
+ def _client(self):
+ from main import app
+
+ return TestClient(app)
+
+ def _dark(self, n=1):
+ for _ in range(n):
+ solver_mod._record_solve_history(
+ "published",
+ {"n_nodes": 3},
+ {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3},
+ solve_key="mn-dark-1",
+ raw_lat=LAT,
+ raw_lon=LON,
+ )
+
+ def _known(self, n=1):
+ for _ in range(n):
+ solver_mod._record_solve_history(
+ "known_truth_match",
+ {"n_nodes": 2, "adsb_hex": "abc123", "initial_guess": {"lat": LAT, "lon": LON}},
+ {"success": True, "lat": LAT, "lon": LON, "n_nodes": 2},
+ extra={"known_lane": True, "label": "truth_match", "published": False},
+ )
+
+ def _adsb(self, n=1):
+ for _ in range(n):
+ solver_mod._record_solve_history(
+ "published",
+ {"n_nodes": 3, "adsb_hex": "abc123"},
+ {"success": True, "lat": LAT, "lon": LON, "n_nodes": 3},
+ solve_key="mn-adsb-abc123",
+ raw_lat=LAT,
+ raw_lon=LON,
+ )
+
+ def test_default_lane_is_all_and_counts_every_lane(self):
+ self._dark()
+ self._known()
+ self._adsb()
+ data = self._client().get("/api/test/mlat-history?all=1").json()
+ assert data["lane"] == "all"
+ assert data["lane_counts"] == {"dark": 1, "known": 1, "adsb": 1}
+ assert data["n_records"] == 3
+
+ def test_lane_dark_returns_only_dark_records(self):
+ self._dark(2)
+ self._known(3)
+ self._adsb(1)
+ data = self._client().get("/api/test/mlat-history?all=1&lane=dark").json()
+ assert data["n_records"] == 2
+ assert data["lane_counts"] == {"dark": 2, "known": 0, "adsb": 0}
+ assert all(r["solve_key"] == "mn-dark-1" for r in data["records"])
+
+ def test_lane_known_returns_only_known_records(self):
+ self._dark(2)
+ self._known(3)
+ data = self._client().get("/api/test/mlat-history?all=1&lane=known").json()
+ assert data["n_records"] == 3
+ assert all(r["known_lane"] for r in data["records"])
+
+ def test_unknown_lane_is_rejected(self):
+ assert self._client().get("/api/test/mlat-history?all=1&lane=bogus").status_code == 400
+
+ def test_known_volume_cannot_evict_dark_records_from_the_response(self):
+ """The bug the per-lane cap fixes, at 1/500 scale."""
+ self._dark(2)
+ self._known(20)
+ data = self._client().get("/api/test/mlat-history?all=1&limit=2").json()
+ # 2 dark + 2 known survive the cap; the flat cap would have returned
+ # the 2 newest records overall, both known.
+ lanes = [("known" if r.get("known_lane") else "dark") for r in data["records"]]
+ assert sorted(lanes) == ["dark", "dark", "known", "known"]
+ # n_records / lane_counts stay pre-cap so truncation is legible.
+ assert data["n_records"] == 22
+ assert data["lane_counts"] == {"dark": 2, "known": 20, "adsb": 0}
+
+ def test_limit_is_clamped_to_the_maximum(self):
+ self._dark(3)
+ data = self._client().get("/api/test/mlat-history?all=1&limit=99999").json()
+ assert len(data["records"]) == 3
+
+ def test_hex_lookup_reports_the_lane_block_too(self):
+ self._dark()
+ rec = state.mlat_solve_history[0]
+ data = self._client().get(f"/api/test/mlat-history?hex={rec['solver_hex']}").json()
+ assert data["lane"] == "all"
+ assert data["lane_counts"]["dark"] == 1
+
+
+class TestResolveSkipDump:
+ """?kind=resolve_skips dumps the solver's skip deque.
+
+ A skip is not a solve outcome and must not be written into the
+ solve-history deques: on the live fleet skips outrun dark records roughly
+ two to one and would evict exactly the records an investigation needs.
+ """
+
+ def setup_method(self):
+ state._reset_for_tests()
+ solver_mod._reset_for_tests()
+
+ def teardown_method(self):
+ solver_mod._reset_for_tests()
+
+ def _client(self):
+ from main import app
+
+ return TestClient(app)
+
+ def _skip(self, track_ids=("a1", "b1"), n_nodes=3, **s_in):
+ now = time.time()
+ s = dict(_CONFIRMED_N2, n_nodes=n_nodes, track_ids=list(track_ids), **s_in)
+ solver_mod._claim_resolve_slot(s, now)
+ assert solver_mod._claim_resolve_slot(dict(s), now) is False
+ solver_mod._record_resolve_skip(dict(s), now)
+
+ def test_skip_records_the_blocking_claim(self):
+ self._skip()
+ data = self._client().get("/api/test/mlat-history?kind=resolve_skips").json()
+ assert data["kind"] == "resolve_skips"
+ assert data["n_records"] == 1
+ rec = data["records"][0]
+ assert rec["lane"] == "dark"
+ assert rec["track_ids"] == ["a1", "b1"]
+ assert rec["n_nodes"] == 3
+ assert {b["track_id"] for b in rec["blocking"]} == {"a1", "b1"}
+ assert all(b["held_n"] == 3 for b in rec["blocking"])
+
+ def test_skips_do_not_land_in_the_solve_history(self):
+ self._skip()
+ assert not state.mlat_solve_history
+ assert not state.mlat_solve_history_known
+
+ def test_lane_filter_applies_to_skips(self):
+ self._skip(track_ids=("a1", "b1"))
+ self._skip(track_ids=("a2", "b2"), adsb_hex="abc123")
+ assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=dark").json()["n_records"] == 1
+ assert self._client().get("/api/test/mlat-history?kind=resolve_skips&lane=adsb").json()["n_records"] == 1
+ assert self._client().get("/api/test/mlat-history?kind=resolve_skips").json()["lane_counts"] == {
+ "dark": 1,
+ "known": 0,
+ "adsb": 1,
+ }
+
+ def test_unknown_kind_is_rejected(self):
+ assert self._client().get("/api/test/mlat-history?kind=bogus").status_code == 400
diff --git a/backend/tests/test_solver_alt_mode.py b/backend/tests/test_solver_alt_mode.py
new file mode 100644
index 00000000..2c9eec02
--- /dev/null
+++ b/backend/tests/test_solver_alt_mode.py
@@ -0,0 +1,344 @@
+"""SOLVER_ALT_MODE: how the n>=3 solve gets its altitude.
+
+sweep (the default) calls the LM once per fixed altitude layer and keeps the
+lowest rms_delay — six process-pool round trips, and an altitude quantised to
+a ladder 2 km wide, which puts up to 1 km of error into the residual the
+reject gate reads. free makes ONE call to the geolocator's multi-start
+helper, which solves altitude as a sixth unknown from SOLVER_FREE_ALT_STARTS
+start layers — one by default, the layer nearest the association guess.
+
+These tests are about the routing, not the physics: the geolocator's own
+suite (tests/test_free_altitude.py there) measures what the free solve
+actually fits. What matters here is that the default is byte-identical to
+the sweep, that free spends one call and not six, that trimming re-solves
+under the same mode, and that both modes leave enough on the history record
+to be compared live.
+"""
+
+import time
+
+import pytest
+
+from core import state
+from services import frame_processor
+from services.tasks import solver as solver_mod
+
+LAT, LON = 35.0, -82.0
+
+
+def _s_in(node_ids, alt_km=9.0, **overrides):
+ s_in = {
+ "initial_guess": {"lat": LAT, "lon": LON, "alt_km": alt_km},
+ "measurements": [{"node_id": nid, "delay_us": 10.0, "doppler_hz": 1.0, "snr": 15.0} for nid in node_ids],
+ "n_nodes": len(node_ids),
+ "timestamp_ms": int(time.time() * 1000),
+ }
+ s_in.update(overrides)
+ return s_in
+
+
+def _stub_result(node_ids, rms_delay=0.5, **overrides):
+ result = {
+ "success": True,
+ "lat": LAT,
+ "lon": LON,
+ "alt_m": 9000.0,
+ "timestamp_ms": int(time.time() * 1000),
+ "vel_east": 0.0,
+ "vel_north": 0.0,
+ "rms_delay": rms_delay,
+ "rms_doppler": 5.0,
+ "n_nodes": len(node_ids),
+ "n_measurements": len(node_ids),
+ "contributing_node_ids": list(node_ids),
+ }
+ result.update(overrides)
+ return result
+
+
+class _Recorder:
+ """A solve_fn / multistart_fn that records every call it is given."""
+
+ def __init__(self, result_for):
+ self.calls: list[tuple] = []
+ self._result_for = result_for
+
+ def __call__(self, s_in, node_cfgs, *rest):
+ self.calls.append((s_in, node_cfgs, rest))
+ nodes = tuple(m["node_id"] for m in s_in["measurements"])
+ return self._result_for(nodes, s_in, *rest)
+
+
+class _AltModeBase:
+ def setup_method(self):
+ state._reset_for_tests()
+ solver_mod._reset_for_tests()
+
+ def teardown_method(self):
+ solver_mod._reset_for_tests()
+
+
+class TestFreeAltStarts:
+ """The starts handed to the multi-start helper: SOLVER_FREE_ALT_STARTS of
+ them, one by default."""
+
+ @pytest.mark.parametrize(
+ "alt_km,expected",
+ [
+ (9.0, [9.0]),
+ (7.0, [7.0]),
+ (8.2, [9.0]),
+ (1.5, [1.5]),
+ # Off the ends of the ladder: still the nearest layer, not nothing.
+ (0.4, [1.5]),
+ (40.0, [11.0]),
+ ],
+ )
+ def test_one_start_at_the_nearest_layer_by_default(self, alt_km, expected):
+ assert state.SOLVER_FREE_ALT_STARTS == 1
+ assert solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM) == expected
+
+ def test_an_adsb_altitude_in_the_ladder_is_the_start(self):
+ """_solve_best_altitude splices a non-layer altitude (ADS-B) into the
+ layers, and the starts are taken over that spliced list — so the single
+ default start is that exact altitude, which is what the sweep would
+ have pinned too."""
+ layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4]))
+ assert solver_mod._free_alt_starts(8.4, layers) == [8.4]
+
+ @pytest.mark.parametrize(
+ "alt_km,expected",
+ [
+ (9.0, [7.0, 9.0, 11.0]),
+ (7.0, [5.0, 7.0, 9.0]),
+ (8.2, [7.0, 9.0, 11.0]),
+ # Clamped at the ends: the ladder's first and last layers still get
+ # three starts, not one or two.
+ (1.5, [1.5, 3.0, 5.0]),
+ (0.4, [1.5, 3.0, 5.0]),
+ (11.0, [7.0, 9.0, 11.0]),
+ (40.0, [7.0, 9.0, 11.0]),
+ ],
+ )
+ def test_three_starts_are_the_window_around_the_nearest_layer(self, alt_km, expected, monkeypatch):
+ """The pre-default behaviour, still reachable by configuration."""
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3)
+ starts = solver_mod._free_alt_starts(alt_km, solver_mod._SOLVER_ALT_LAYERS_KM)
+ assert starts == expected
+ assert len(starts) == 3
+
+ def test_three_starts_window_the_spliced_adsb_altitude(self, monkeypatch):
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3)
+ layers = sorted(set(solver_mod._SOLVER_ALT_LAYERS_KM + [8.4]))
+ assert solver_mod._free_alt_starts(8.4, layers) == [7.0, 8.4, 9.0]
+
+ @pytest.mark.parametrize("configured", [0, -3])
+ def test_fewer_than_one_start_still_starts_somewhere(self, configured, monkeypatch):
+ """A count below one would leave the LM no start at all, so it clamps
+ rather than raises: a mis-set env degrades to a working solve."""
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", configured)
+ assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == [9.0]
+
+ def test_more_starts_than_layers_is_every_layer(self, monkeypatch):
+ """The other clamp: a count past the end of the ladder would slice
+ short of it, quietly dropping starts that were asked for."""
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 99)
+ assert solver_mod._free_alt_starts(9.0, solver_mod._SOLVER_ALT_LAYERS_KM) == solver_mod._SOLVER_ALT_LAYERS_KM
+
+ def test_no_layers_gives_no_starts(self):
+ assert solver_mod._free_alt_starts(9.0, []) == []
+
+
+class TestSweepIsTheDefault(_AltModeBase):
+ def test_sweep_calls_the_lm_once_per_layer_and_never_the_multistart(self):
+ nodes = ["n1", "n2", "n3"]
+ solve = _Recorder(lambda n, s, *r: _stub_result(n))
+ multistart = _Recorder(lambda n, s, *r: pytest.fail("multistart called in sweep mode"))
+
+ result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart)
+
+ assert result is not None and result["success"]
+ assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM)
+ assert [c[0]["initial_guess"]["alt_km"] for c in solve.calls] == solver_mod._SOLVER_ALT_LAYERS_KM
+ assert multistart.calls == []
+ assert state.SOLVER_ALT_MODE == "sweep"
+
+
+class TestFreeMode(_AltModeBase):
+ def setup_method(self):
+ super().setup_method()
+ self._saved_mode = state.SOLVER_ALT_MODE
+ state.SOLVER_ALT_MODE = "free"
+
+ def teardown_method(self):
+ state.SOLVER_ALT_MODE = self._saved_mode
+ super().teardown_method()
+
+ def test_one_multistart_call_with_one_start(self):
+ nodes = ["n1", "n2", "n3"]
+ solve = _Recorder(lambda n, s, *r: pytest.fail("sweep ran in free mode"))
+ multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4]))
+
+ result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart)
+
+ assert result is not None and result["success"]
+ assert solve.calls == []
+ assert len(multistart.calls) == 1
+ (_, _, rest) = multistart.calls[0]
+ assert rest == ([9.0],)
+
+ def test_an_adsb_guess_altitude_is_the_start(self):
+ """A non-layer initial_guess altitude is spliced into the ladder and
+ becomes the start itself — the free-mode analogue of the sweep's extra
+ layer, and the one exact altitude the candidate has."""
+ nodes = ["n1", "n2", "n3"]
+ multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free"))
+
+ solver_mod._solve_best_altitude(_s_in(nodes, alt_km=8.437), {}, lambda s, c: None, multistart)
+
+ assert multistart.calls[0][2] == ([8.437],)
+
+ def test_the_start_count_is_configurable(self, monkeypatch):
+ """SOLVER_FREE_ALT_STARTS buys back the neighbour window for a geometry
+ whose single start lands on the wrong side of an ellipse."""
+ monkeypatch.setattr(state, "SOLVER_FREE_ALT_STARTS", 3)
+ multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free"))
+
+ solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, lambda s, c: None, multistart)
+
+ assert len(multistart.calls) == 1
+ assert multistart.calls[0][2] == ([7.0, 9.0, 11.0],)
+
+ def test_n2_keeps_the_sweep(self):
+ """Altitude is unobservable at n=2 — the free path is not entered even
+ with the mode on."""
+ nodes = ["n1", "n2"]
+ solve = _Recorder(lambda n, s, *r: _stub_result(n))
+ multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken at n=2"))
+
+ result = solver_mod._solve_best_altitude(_s_in(nodes), {}, solve, multistart)
+
+ assert result is not None
+ assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM)
+ assert multistart.calls == []
+
+ def test_a_failed_multistart_is_a_failed_solve(self):
+ """No silent fall back to the sweep: three starts producing nothing is
+ the same verdict as every layer producing nothing."""
+ solve = _Recorder(lambda n, s, *r: pytest.fail("swept after a failed multistart"))
+ multistart = _Recorder(lambda n, s, *r: None)
+
+ assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart) is None
+
+ def test_history_carries_the_mode_and_the_per_start_residuals(self):
+ nodes = ["n1", "n2", "n3"]
+ multistart = _Recorder(
+ lambda n, s, *r: _stub_result(
+ n, altitude_mode="free", rms_by_start=[1.2345, None, 0.4321], alt_starts_km=[5.0, 7.0, 9.0]
+ )
+ )
+ solver_mod._process_solver_item(
+ (_s_in(nodes), {}, time.time()),
+ lambda s, c: pytest.fail("sweep ran in free mode"),
+ multistart_fn=multistart,
+ )
+
+ rec = state.mlat_solve_history[-1]
+ assert rec["outcome"] == "published"
+ assert rec["altitude_mode"] == "free"
+ assert rec["alt_starts_km"] == [5.0, 7.0, 9.0]
+ assert rec["alt_start_rms_us"] == [1.234, None, 0.432]
+
+ def test_a_single_start_still_records_its_residual(self):
+ """The comparison channel does not depend on there being several
+ starts: one start records a one-element list, not a bare number or
+ nothing at all."""
+ nodes = ["n1", "n2", "n3"]
+ multistart = _Recorder(
+ lambda n, s, *r: _stub_result(n, altitude_mode="free", rms_by_start=[0.4321], alt_starts_km=[9.0])
+ )
+ solver_mod._process_solver_item(
+ (_s_in(nodes), {}, time.time()),
+ lambda s, c: pytest.fail("sweep ran in free mode"),
+ multistart_fn=multistart,
+ )
+
+ rec = state.mlat_solve_history[-1]
+ assert rec["alt_starts_km"] == [9.0]
+ assert rec["alt_start_rms_us"] == [0.432]
+
+ def test_z_saturation_reaches_the_history(self):
+ nodes = ["n1", "n2", "n3"]
+ multistart = _Recorder(lambda n, s, *r: _stub_result(n, altitude_mode="free", z_saturated=True))
+ solver_mod._process_solver_item((_s_in(nodes), {}, time.time()), lambda s, c: None, multistart_fn=multistart)
+ assert state.mlat_solve_history[-1]["z_saturated"] is True
+
+ def test_trimming_re_solves_through_the_multistart(self):
+ """A trim round must use the mode its first solve used, or the rms it
+ compares against the previous round is a different quantity."""
+ full = ["n1", "n2", "n3", "n4", "bad"]
+ trimmed = ["n1", "n2", "n3", "n4"]
+
+ def _result(nodes, s_in, *rest):
+ if "bad" in nodes:
+ return _stub_result(
+ nodes,
+ rms_delay=8.0,
+ altitude_mode="free",
+ per_node_delay_res_us={n: (12.0 if n == "bad" else 0.5) for n in nodes},
+ )
+ return _stub_result(
+ nodes,
+ rms_delay=0.8,
+ altitude_mode="free",
+ per_node_delay_res_us={n: 0.3 for n in nodes},
+ )
+
+ multistart = _Recorder(_result)
+ result = solver_mod._process_solver_item(
+ (_s_in(full), {}, time.time()),
+ lambda s, c: pytest.fail("sweep ran during a free-mode trim"),
+ multistart_fn=multistart,
+ )
+
+ assert result is not None and result["n_nodes"] == 4
+ assert sorted(m["node_id"] for m in multistart.calls[-1][0]["measurements"]) == trimmed
+ rec = state.mlat_solve_history[-1]
+ assert rec["outcome"] == "published"
+ assert rec["trimmed_node_ids"] == ["bad"]
+ assert rec["altitude_mode"] == "free"
+
+ def test_an_unrecognised_mode_would_sweep(self):
+ """The flag degrades to the inert mode, like its siblings — asserted on
+ the resolution rule rather than by re-importing core.state."""
+ state.SOLVER_ALT_MODE = "definitely-not-a-mode"
+ solve = _Recorder(lambda n, s, *r: _stub_result(n))
+ multistart = _Recorder(lambda n, s, *r: pytest.fail("free path taken for a bad mode"))
+ assert solver_mod._solve_best_altitude(_s_in(["n1", "n2", "n3"]), {}, solve, multistart)
+ assert len(solve.calls) == len(solver_mod._SOLVER_ALT_LAYERS_KM)
+
+
+class TestConfigsForSolverInput:
+ """Only the configs a candidate can reach are queued with it.
+
+ The pool is a spawn pool, so whatever is queued is pickled and shipped on
+ every solve — 58 fleet configs against a candidate's 2-8 measurements.
+ """
+
+ _FLEET = {f"n{i}": {"rx_lat": 35.0 + i, "rx_lon": -82.0} for i in range(8)}
+
+ def test_restricted_to_the_measurement_nodes(self):
+ s_in = _s_in(["n1", "n3", "n5"])
+ cfgs = frame_processor.configs_for_solver_input(self._FLEET, s_in)
+ assert sorted(cfgs) == ["n1", "n3", "n5"]
+ assert cfgs["n3"] is self._FLEET["n3"]
+
+ def test_unknown_measurement_nodes_are_simply_absent(self):
+ """A measurement from a node with no config is the case
+ solve_multinode already handles by skipping it — not an error here."""
+ cfgs = frame_processor.configs_for_solver_input(self._FLEET, _s_in(["n1", "ghost"]))
+ assert sorted(cfgs) == ["n1"]
+
+ def test_no_measurements_gives_nothing(self):
+ assert frame_processor.configs_for_solver_input(self._FLEET, {"measurements": []}) == {}
+ assert frame_processor.configs_for_solver_input(self._FLEET, {}) == {}
diff --git a/backend/tests/test_solver_stats.py b/backend/tests/test_solver_stats.py
index 67bc599c..806e8992 100644
--- a/backend/tests/test_solver_stats.py
+++ b/backend/tests/test_solver_stats.py
@@ -257,6 +257,8 @@ def test_consensus_and_counters_reflect_state(self):
state.solver_consensus_fallback = 9
state.solver_consensus_shadow = 10
state.solver_vel_untrusted_published = 11
+ state.solver_resolve_skips_dark = 9
+ state.node_frames_rate_limited = 13
out = _solver_window_stats(10.0)
assert out["counters"] == {
"successes": 5,
@@ -265,7 +267,9 @@ def test_consensus_and_counters_reflect_state(self):
"solver_trimmed": 3,
"stale_drops": 4,
"resolve_skips": 12,
+ "resolve_skips_dark": 9,
"queue_drops": 6,
+ "node_frames_rate_limited": 13,
"worker_errors": 0,
"vel_untrusted_published": 11,
}
@@ -788,3 +792,118 @@ def get(self, *a, **kw):
state.multinode_tracks["mn-dark-1"] = {"lat": 35.0, "lon": -82.0}
state.adsb_aircraft["real1"] = _MutatingFix({"lat": 35.009, "lon": -82.0, "last_seen_ms": now_ms})
assert _solver_window_stats(10.0)["ghosts"]["ghost_tracks"] == 0
+
+
+def _skip_rec(lane="dark", age_s=0.0, track_ids=("a1",), n_nodes=3):
+ return {
+ "ts_ms": int((time.time() - age_s) * 1000),
+ "lane": lane,
+ "track_ids": list(track_ids),
+ "n_nodes": n_nodes,
+ "blocking": [{"track_id": track_ids[0], "held_ts": time.time() - age_s, "held_n": n_nodes}],
+ "guess_lat": None,
+ "guess_lon": None,
+ }
+
+
+class TestResolveSkipBlock:
+ """Resolve-slot skips are windowed from their own deque, not from the
+ since-boot counter, so they can be read against the attempts in the same
+ window — the ratio the claim-on-publish fix is judged on."""
+
+ def setup_method(self):
+ state._reset_for_tests()
+
+ def test_totals_split_by_lane(self):
+ for _ in range(3):
+ state.solver_resolve_skips_recent.append(_skip_rec("dark"))
+ state.solver_resolve_skips_recent.append(_skip_rec("adsb"))
+ out = _solver_window_stats(10.0)["resolve_skips"]
+ assert out["total"] == 4
+ assert out["dark"] == 3
+
+ def test_window_excludes_old_skips(self):
+ state.solver_resolve_skips_recent.append(_skip_rec(age_s=20 * 60))
+ state.solver_resolve_skips_recent.append(_skip_rec(age_s=1))
+ assert _solver_window_stats(10.0)["resolve_skips"]["total"] == 1
+
+ def test_attempts_ratio_is_skips_over_dark_attempts(self):
+ for _ in range(4):
+ state.solver_resolve_skips_recent.append(_skip_rec())
+ state.mlat_solve_history.append(_rec("published"))
+ state.mlat_solve_history.append(_rec("rejected_beam"))
+ out = _solver_window_stats(10.0)
+ assert out["attempts"] == 2
+ assert out["resolve_skips"]["attempts_ratio"] == 2.0
+
+ def test_attempts_ratio_is_none_without_attempts(self):
+ state.solver_resolve_skips_recent.append(_skip_rec())
+ assert _solver_window_stats(10.0)["resolve_skips"]["attempts_ratio"] is None
+
+ def test_window_effective_minutes_exposes_a_truncated_deque(self):
+ """The deque is 500 entries against ~50 skips/min live, so a long
+ window IS truncated here even when the solve stores cover it."""
+ state.solver_resolve_skips_recent.append(_skip_rec(age_s=6 * 60))
+ out = _solver_window_stats(30.0)["resolve_skips"]
+ assert 5.9 <= out["window_effective_minutes"] <= 6.1
+
+ def test_a_skip_is_not_an_attempt_or_a_reject(self):
+ """Skips must not leak into the funnel — they never reached a solve."""
+ for _ in range(5):
+ state.solver_resolve_skips_recent.append(_skip_rec())
+ out = _solver_window_stats(10.0)
+ assert out["attempts"] == 0
+ assert out["rejects"]["total"] == 0
+
+
+def _gt_rec(foreign=(), **kw):
+ """A dark record carrying the contamination stamp."""
+ rec = _rec("published", **kw)
+ rec["gt_hex"] = "abc123"
+ rec["foreign_node_ids"] = list(foreign)
+ rec["contaminated"] = bool(foreign)
+ return rec
+
+
+class TestContaminationBlock:
+ """Live cluster contamination: of the dark records that matched ground
+ truth, how many carried a node that could not see the aircraft."""
+
+ def setup_method(self):
+ state._reset_for_tests()
+
+ def test_pct_and_mean_over_judged_records(self):
+ state.mlat_solve_history.append(_gt_rec(foreign=["n1"]))
+ state.mlat_solve_history.append(_gt_rec(foreign=["n1", "n2"]))
+ state.mlat_solve_history.append(_gt_rec(foreign=[]))
+ state.mlat_solve_history.append(_gt_rec(foreign=[]))
+ out = _solver_window_stats(10.0)["contamination"]
+ assert out["records_with_gt"] == 4
+ assert out["contaminated"] == 2
+ assert out["pct"] == 50.0
+ assert out["foreign_nodes_per_record"] == 0.75
+
+ def test_unstamped_records_are_out_of_the_denominator(self):
+ """No GT match, or no judgeable node geometry, is an abstention — not
+ a clean record."""
+ state.mlat_solve_history.append(_gt_rec(foreign=["n1"]))
+ state.mlat_solve_history.append(_rec("published"))
+ out = _solver_window_stats(10.0)["contamination"]
+ assert out["records_with_gt"] == 1
+ assert out["pct"] == 100.0
+
+ def test_empty_window_abstains_rather_than_reporting_zero(self):
+ out = _solver_window_stats(10.0)["contamination"]
+ assert out == {
+ "records_with_gt": 0,
+ "contaminated": 0,
+ "pct": None,
+ "foreign_nodes_per_record": None,
+ }
+
+ def test_known_lane_records_are_not_counted(self):
+ """Dark lane only, like every other block in the funnel."""
+ rec = _gt_rec(foreign=["n1"])
+ rec["known_lane"] = True
+ state.mlat_solve_history_known.append(rec)
+ assert _solver_window_stats(10.0)["contamination"]["records_with_gt"] == 0
diff --git a/backend/tests/test_solver_worker.py b/backend/tests/test_solver_worker.py
index 6f5ea7b3..a4a6c8e9 100644
--- a/backend/tests/test_solver_worker.py
+++ b/backend/tests/test_solver_worker.py
@@ -38,6 +38,7 @@ def _reset_state():
state.n2_unconfirmed = 0
state.solver_stale_drops = 0
state.solver_resolve_skips = 0
+ state.solver_resolve_skips_dark = 0
state.multinode_tracks.clear()
state.task_last_success.clear()
@@ -415,6 +416,49 @@ def solve_fn(s_in, cfgs):
assert state.solver_failures == 0
assert state.solver_stale_drops == 0
+ def test_a_skip_is_recorded_with_the_claim_that_blocked_it(self):
+ """The counter alone cannot say WHOSE claim suppressed a candidate,
+ and tracker track ids are shared between different aircraft — so a
+ skip that suppressed a duplicate and one that suppressed a neighbour
+ looked identical. The deque carries the blocking claims."""
+ _reset_state()
+ state.solver_resolve_skips_recent.clear()
+ now = time.time()
+ s_in = dict(self._s_in(["a1", "b1"], n_nodes=4), initial_guess={"lat": 35.0, "lon": -82.0})
+ assert solver_mod._claim_resolve_slot(dict(s_in), now) is True
+ assert solver_mod._claim_resolve_slot(dict(s_in), now) is False
+ solver_mod._record_resolve_skip(dict(s_in), now)
+
+ assert state.solver_resolve_skips == 1
+ assert state.solver_resolve_skips_dark == 1
+ assert len(state.solver_resolve_skips_recent) == 1
+ rec = state.solver_resolve_skips_recent[0]
+ assert rec["lane"] == "dark"
+ assert rec["track_ids"] == ["a1", "b1"]
+ assert rec["n_nodes"] == 4
+ assert rec["guess_lat"] == 35.0
+ assert {b["track_id"]: b["held_n"] for b in rec["blocking"]} == {"a1": 4, "b1": 4}
+
+ def test_a_tagged_candidate_is_counted_but_not_as_dark(self):
+ _reset_state()
+ state.solver_resolve_skips_recent.clear()
+ now = time.time()
+ s_in = dict(self._s_in(["a1"], n_nodes=3), adsb_hex="abc123")
+ solver_mod._record_resolve_skip(s_in, now)
+ assert state.solver_resolve_skips == 1
+ assert state.solver_resolve_skips_dark == 0
+ assert state.solver_resolve_skips_recent[0]["lane"] == "adsb"
+
+ def test_skips_never_enter_the_solve_history(self):
+ """One skip per solve-history record would evict the solves the same
+ investigation needs — live, skips outrun dark records two to one."""
+ _reset_state()
+ state.mlat_solve_history.clear()
+ s_in = self._s_in(["a1", "b1"])
+ solver_mod._record_resolve_skip(s_in, time.time())
+ assert not state.mlat_solve_history
+ assert not state.mlat_solve_history_known
+
class TestSolveBestAltitude:
"""Altitude-sweep helpers: n_nodes >= 3 uses a layer sweep, n_nodes = 2 uses initial_guess directly."""
diff --git a/docs/solverflow.md b/docs/solverflow.md
index c626e5bf..980e577a 100644
--- a/docs/solverflow.md
+++ b/docs/solverflow.md
@@ -11,10 +11,12 @@ beyond what publication needs, see [`pipeline.md`](pipeline.md) (its own §3 is
stale on the known lane and pool fallback — this doc is the current source for
those two topics).
-File:line references are repo-relative to `backend/`, except the `libs/*`
-paths, which are already fully qualified (those are separate submodule repos
-vendored under `libs/`). All references were checked against `main` at
-`0a1d30f`.
+References name a **file and a symbol**, never a line number: paths are
+repo-relative to `backend/`, except the `libs/*` ones, which are already fully
+qualified (those are separate submodule repos vendored under `libs/`). Line
+numbers were what this document used to carry, and they were stale within two
+weeks of being written — every one of them had drifted by the time anyone
+followed it. A symbol survives an edit above it, so grep for the name.
## Legend
@@ -74,13 +76,15 @@ lane rides the solver loop's idle cycles rather than owning workers of its
own. Everything that reaches a solve passes through one gate stack
(`_process_solver_item`) before publication.
-| Constant | Value | File:line |
+| Constant | Value | Defined in |
|---|---|---|
-| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py:358-359` |
-| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py:365-366` |
-| `FRAME_WORKERS` | 4 (compose sets 6) | `main.py:164`, `docker-compose.yml:54` |
-| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py:31,67` |
-| `KNOWN_LANE_MODE` default | `binding` | `core/state.py:72-74` |
+| `frame_queue` size (`FRAME_QUEUE_SIZE`) | 10000 | `core/state.py` |
+| `solver_queue` size (`SOLVER_QUEUE_SIZE`) | 200 | `core/state.py` |
+| `FRAME_WORKERS` | 4 (compose sets 6) | `core/state.py` (`FRAME_WORKERS`), `docker-compose.yml` |
+| `SOLVER_WORKERS` | 2 daemon threads + same-size process pool | `services/tasks/solver.py` (`_N_SOLVER_WORKERS`, `_make_solver_pool`) |
+| `KNOWN_LANE_MODE` default | `binding` | `core/state.py` (`KNOWN_LANE_MODE`) |
+| `SOLVER_ALT_MODE` default | `sweep` | `core/state.py` (`SOLVER_ALT_MODE`) |
+| `SOLVER_FREE_ALT_STARTS` default | 1 | `core/state.py` (`SOLVER_FREE_ALT_STARTS`) |
---
@@ -89,11 +93,11 @@ own. Everything that reaches a solve passes through one gate stack
```mermaid
flowchart TD
subgraph producers["Five producers"]
- p1["TCP (primary)
tcp_handler.py:326"]
- p2["blah2 bridge
blah2_bridge.py:289"]
- p3["v1 node HTTP API
node_stream.py:250"]
- p4["Legacy HTTP radar routes
routes/radar.py:151,202"]
- p5["Startup priming
node_pipeline.py:139"]
+ p1["TCP (primary)
tcp_handler._enqueue_detection"]
+ p2["blah2 bridge
blah2_bridge.blah2_bridge_task"]
+ p3["v1 node HTTP API
node_stream._file_frame"]
+ p4["Legacy HTTP radar routes
radar.ingest_detections(_bulk)"]
+ p5["Startup priming
node_pipeline.prime_pipeline"]
end
p1 --> gA{"Gate A: timestamp present?"}
@@ -131,25 +135,25 @@ flowchart TD
classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3
```
-The ordering inside `process_one_frame` (`services/frame_processor.py:294`) is
+The ordering inside `process_one_frame` (`services/frame_processor.py`) is
load-bearing, not incidental: claiming (2.3) runs **before** ADS-B seeding
(2.4) so a node-supplied `adsb` field is still distinguishable from a claim,
and both run **before** the tracker (2.5) so that, in `binding` mode, a
claimed detection never reaches the dark-lane tracker or association at all
-— see the ordering comment at `services/frame_processor.py:327-337`.
+— see the ordering comment at the head of `process_one_frame`'s claiming step.
Frame-level gates (A/B/C on TCP, plus the connected-node check on the v1 API)
sit ahead of everything else; nothing downstream sees a frame that failed
one of them.
-| Constant | Value | File:line |
+| Constant | Value | Defined in |
|---|---|---|
-| Gate A: timestamp required | — | `tcp_handler.py:513-516` |
-| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node | `tcp_handler.py:495,527-532` |
-| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler.py:536-552` |
-| `process_one_frame` entry | — | `services/frame_processor.py:294` |
-| Ordering rationale (claim → seed → tracker) | — | `services/frame_processor.py:327-337` |
-| Gate 2.10: `n_nodes < 2` skip | — | `services/frame_processor.py:409-426` |
-| blah2 poll interval | 1.0 s | `config/constants.py:266` |
+| Gate A: timestamp required | — | `tcp_handler._enqueue_detection` |
+| Gate B: `NODE_FRAME_MIN_INTERVAL_S` | 1.0 s/node, counted as `node_frames_rate_limited` | `tcp_handler` (`_NODE_MIN_INTERVAL_S`, `_enqueue_detection`) |
+| Gate C: QueueFull | `frames_dropped` counter | `tcp_handler._enqueue_detection` |
+| `process_one_frame` entry | — | `services/frame_processor.py` |
+| Ordering rationale (claim → seed → tracker) | — | `frame_processor.process_one_frame` |
+| Gate 2.10: `n_nodes < 2` skip | — | `frame_processor.process_one_frame` |
+| blah2 poll interval | 1.0 s | `config/constants.py` (`BLAH2_POLL_INTERVAL_S`) |
---
@@ -205,17 +209,17 @@ also reject — a differential property test in `test_known_claiming.py`
failure increments the same `known_claims_visibility_rejects` counter as a
gate failure: same event, same meaning, just caught cheaper.
-**Mode semantics** (`KNOWN_LANE_MODE`, read once at `core/state.py:72-74`,
+**Mode semantics** (`KNOWN_LANE_MODE`, read once in `core/state.py`,
default `binding`; an unrecognized value falls back to `shadow`, not to the
default — a typo should degrade to the inert mode, not the acting one):
| Mode | Claiming | Frame the dark lane sees | Known-lane solver | Publication |
|---|---|---|---|---|
-| `off` | never runs | untouched | returns 0 immediately (`known_lane.py:391-392`); worker never even calls it (`solver.py:1961`) | none |
+| `off` | never runs | untouched | returns 0 immediately (`known_lane.run_known_lane_pass`); worker never even calls it (`solver._run_solver_worker`) | none |
| `shadow` | runs, records claims + residuals + counters | untouched | runs: solves, classifies, records accuracy samples | never |
-| `binding` | runs | `strip_claimed_detections` removes claimed indices (`frame_processor.py:347`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish |
+| `binding` | runs | `strip_claimed_detections` removes claimed indices (called from `frame_processor.process_one_frame`) | runs | `truth_match` results publish into `state.multinode_tracks` as `mn-adsb-`; ghosts never publish |
-`strip_claimed_detections` (`services/known_claiming.py:343`) returns a copy
+`strip_claimed_detections` (`services/known_claiming.py`) returns a copy
with claimed indices removed from `delay`/`doppler`/`snr`/`adsb`; the
original frame still feeds the archive and ADS-B extraction (steps 2.11-2.12)
unchanged.
@@ -224,7 +228,7 @@ unchanged.
```mermaid
flowchart TD
- arm["Solver worker loop arms known_lane
at thread start (solver.py:1961)"]
+ arm["Solver worker loop arms known_lane
at thread start (solver._run_solver_worker)"]
arm --> drain["After every queue-drain iteration,
call maybe_run_pass"]
drain --> gm{"mode == off?"}
gm -->|"yes"| ret1["return"]:::inert
@@ -261,7 +265,7 @@ flowchart TD
gpub -->|"no"| noop["accuracy sample only,
no feed entry"]:::inert
```
-The docstring at `services/tasks/known_lane.py:19-27` calls this the "free
+The module docstring of `services/tasks/known_lane.py` calls this the "free
solve invariant": the ADS-B fix seeds the initial guess and pins altitude,
nothing else — no regularization pulls the solve toward the truth position,
so the residual (`err_km`) is a genuine measurement of radar accuracy, not a
@@ -269,21 +273,21 @@ circular check. One more intentional-by-omission detail: known-lane
measurements carry `snr = 0.0` (claim records have no `snr` key), which the
LM's SNR weighting maps to a uniform weight of 1.0.
-| Constant | Value | File:line |
+| Constant | Value | Defined in |
|---|---|---|
-| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `ADSB_SEED_MAX_DR_AGE_S`, `association.py:106`) |
-| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `ADSB_SEED_*`, `association.py:98,99`) |
-| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py:79` |
-| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py:205` |
-| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py:89`, applied in `known_claiming.py:_dark_global_projections` |
-| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py:73,77` |
-| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py:80` |
-| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py:85,86` |
-| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py:274-275` |
-| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py:105` |
-| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py:91,99` |
-| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py:110` |
-| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py:205` |
+| `KNOWN_CLAIM_MAX_FIX_AGE_S` | 45.0 s | `known_claiming.py` (= `association.ADSB_SEED_MAX_DR_AGE_S`) |
+| Path 2 gates: `KNOWN_CLAIM_DELAY_GATE_US` / `KNOWN_CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz, age-scaled | `known_claiming.py` (= `association.ADSB_SEED_DELAY_GATE_US` / `_DOPPLER_GATE_HZ`) |
+| Prescreen slack `_SCREEN_MARGIN` | 1.02 | `known_claiming.py` |
+| Prescreen speed bound `_V_MAX_MS` | 340.0 m/s | `association.py` |
+| `CLAIM_MAX_GLOBAL_TRACKS` (contention reference cap, newest-first) | 200 | `association.py`, applied in `known_claiming._dark_global_projections` |
+| Contention gates: `CLAIM_DELAY_GATE_US` / `CLAIM_DOPPLER_GATE_HZ` | 10.0 us / 25.0 Hz | `libs/retina-analytics/.../association.py` |
+| `CLAIM_MAX_DR_AGE_S` (contention DR window) | 30.0 s | `association.py` |
+| `CLAIM_ELIGIBLE_MIN_N_NODES` / `MIN_SOLVE_COUNT` | 3 / 2 | `association.py` |
+| `KNOWN_CLAIMS_PER_HEX_MAX` | 64 | `core/state.py` |
+| `_PASS_MIN_INTERVAL_S` | 2.0 s | `services/tasks/known_lane.py` |
+| `_CLAIM_MAX_AGE_S` / `_CLAIM_SPREAD_S` | 45.0 s / 5.0 s | `known_lane.py` |
+| `_ATTEMPT_TTL_S` | 600 s | `known_lane.py` |
+| `_MAX_DISPLACEMENT_KM` (truth_match cutoff) | 2.0 km | `services/tasks/solver.py` |
---
@@ -291,7 +295,7 @@ LM's SNR weighting maps to a uniform weight of 1.0.
```mermaid
flowchart TD
- frame["pipeline.process_frame
passive_radar.py:672"]
+ frame["PassiveRadarPipeline.process_frame
pipeline/passive_radar.py"]
frame --> tracker["retina_tracker
Kalman + GNN"]
tracker --> geo["_run_geolocation per track
with new data"]
@@ -345,7 +349,7 @@ flowchart TD
classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3
```
-`compute_overlap_zone` (`libs/retina-analytics/.../association.py:578`)
+`compute_overlap_zone` (`libs/retina-analytics/.../association.py`)
underlies both the confirmed-track association round and the overlap-grid
cache: it fast-prunes non-overlapping node pairs by receiver separation,
grids the shared coverage at `ASSOC_GRID_STEP_KM` on six altitude layers that
@@ -353,22 +357,22 @@ must match the solver's `_SOLVER_ALT_LAYERS_KM`, and requires each grid
column to fall in **both** beams (`_point_in_beam`, FOV-aware only when
`FOV_MODE=active`).
-| Constant | Value | File:line |
+| Constant | Value | Defined in |
|---|---|---|
-| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py:194` |
-| Single-node min detections | 3 | `passive_radar.py:356-361` |
-| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py:59` |
-| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py:220` |
-| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py:22` |
-| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py:23` |
-| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py:31`, `association.py:1043` |
-| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.py:883` |
-| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.py:884` |
-| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py:166` |
-| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py:57-58` |
-| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.py:2160` |
-| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py:21` |
-| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py:111` |
+| `GEO_INTERVAL_S` (single-node geo rate limit) | 10.0 s | `config/constants.py` (applied as `_GEO_INTERVAL_S` in `_run_geolocation`) |
+| Single-node min detections | 3 | `pipeline/passive_radar.py` (`_geolocate_track_event`, `min_det`) |
+| `N2_TRACK_HISTORY_MAX` (track view window) | 20 | `config/constants.py` |
+| `ADSB_VIEW_TAG_FRESH_N` | 3 | `frame_processor.py` |
+| `ASSOC_MIN_INTERVAL_S` | 30.0 s | `config/constants.py` |
+| `ASSOC_MAX_NEIGHBORS` | 50/round | `config/constants.py` |
+| `ASSOC_MAX_PAIRS_PER_ROUND` / `_MAX_FITS_PER_ROUND` | 64 / 8 | `config/constants.py`, `association.py` |
+| `delay_gate_us` (bottom-up coarse gate) | 5.0 us | `association.compute_overlap_zone` (default arg) |
+| `doppler_gate_hz` (bottom-up) | 30.0 Hz, **inert** — delay-only grid gate | `association.compute_overlap_zone` (default arg) |
+| velocity seed cap `_V_MAX_MS` | 340 m/s | `association.py` |
+| `N2_CONFIRM_MIN_EPOCHS` / `MIN_SPAN_S` | 4 / 12.0 s | `config/constants.py` |
+| `_MERGE_DIST_KM` (clustering) | 6.0 km | `association.InterNodeAssociator.format_track_pairs_for_solver` (local) |
+| `ASSOC_GRID_STEP_KM` | 3.0 km | `config/constants.py` |
+| `_SOLVER_ALT_LAYERS_KM` | [1.5, 3, 5, 7, 9, 11] km | `services/tasks/solver.py` |
---
@@ -376,7 +380,7 @@ column to fall in **both** beams (`_point_in_beam`, FOV-aware only when
The centerpiece: every candidate from either lane, once dequeued from
`solver_queue`, runs through `_process_solver_item`
-(`services/tasks/solver.py:1344`) as a strict, ordered chain. A failure at
+(`services/tasks/solver.py`) as a strict, ordered chain. A failure at
any gate stops the chain, bumps a counter, and (from 6.5 onward) writes a
named record to solve history.
@@ -430,14 +434,14 @@ flowchart TD
```
`SOLVER_CONSENSUS_MODE` is `off` in production (see the mode-flag table in
-[`architecture.md:94-110`](architecture.md#feature-gates)), so in practice
+[`architecture.md`](architecture.md#feature-gates)), so in practice
this sub-branch never reaches `active` outside staging.
### The LM itself
-`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py:518`,
+`solve_multinode` — `libs/retina-geolocator/retina_geolocator/multinode_solver.py`,
invoked through the process pool via `_pool_solve_multinode`
-(`services/tasks/solver.py:1915`).
+(`services/tasks/solver.py`).
```mermaid
flowchart TD
@@ -453,25 +457,78 @@ flowchart TD
m6 -->|"no"| m7["vz_saturated if vz on bound;
rms recomputed unweighted;
cov_en_km2 from s^2(J^T J)^-1"]
m7 --> alt{"n_nodes >= 3?"}
- alt -->|"yes"| sweep["_solve_best_altitude wrapper:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"]
+ alt -->|"yes"| mode{"SOLVER_ALT_MODE"}
+ mode -->|"sweep (default)"| sweep["_solve_best_altitude:
calls the LM once per layer in
_SOLVER_ALT_LAYERS_KM,
min rms_delay wins"]
+ mode -->|"free"| freealt["_solve_best_altitude:
ONE pool call to
solve_multinode_multistart,
SOLVER_FREE_ALT_STARTS start
layers (1 by default), z solved"]
alt -->|"no, n=2"| single["_solve_best_altitude_n2:
one LM call at the
association altitude"]
classDef inert fill:#eee,stroke:#999,color:#888,stroke-dasharray: 4 3
```
-| Constant | Value | File:line |
+#### `SOLVER_ALT_MODE` — how the n>=3 solve gets its altitude
+
+`solve_multinode` pins altitude from `initial_guess.alt_km`, so the fix is only
+as good as the altitude the caller found for it. `sweep`, the default, searches
+the six fixed layers of `_SOLVER_ALT_LAYERS_KM` — 2 km apart, so the pin is
+systematically up to 1 km wrong. On noise-free replay of this fleet's geometry
+that quantisation alone left `rms_delay` at a 1.76 us median against the 3.0 us
+gate at 6.5, while a solve at the true altitude reaches 0. Most of the gate's
+budget is spent on the ladder, and the residual left over gets blamed on nodes:
+trimming (6.4) drops measurements that were never the problem.
+
+`free` instead calls `solve_multinode_multistart`, which runs the LM with
+altitude as a sixth unknown (state `[x, y, z, vx, vy, vz]`, z bounded
+0.05–20 km, the `vz` bound unchanged) from `SOLVER_FREE_ALT_STARTS` start
+layers, keeping the lowest `rms_delay`. It is also cheaper: **one** process-pool
+round trip per candidate instead of six, each of which pickles the node configs
+the input needs.
+
+`SOLVER_FREE_ALT_STARTS` defaults to **1** — the layer nearest the association
+guess, or the guess altitude itself when that came from ADS-B and was spliced
+into the ladder (the same splice the sweep does). Freeing z removes the
+ladder's quantisation but not the LM's locality, and extra starts are what
+would stop a solve settling on the wrong side of a bistatic ellipse; on this
+fleet's geometry they had almost nothing to stop. Over a 20-minute window of
+1019 free-mode solves on test, the three starts' `rms_delay` differed by more
+than 0.1 us in **13** of them, and the nearest-layer start was more than 0.5 us
+worse than the best start in **2** — ~0.2% of solves helped, at three times the
+solver CPU, while the pool is the binding constraint (~1.7 attempts/s against a
+2.0 s average latency on two workers). Set it above 1 for a geometry where that
+locality does bite; `_free_alt_starts` clamps it into `[1, len(layers)]` and
+values above 1 give the same neighbour window as before, so `3` restores the
+original behaviour exactly.
+
+At n=2 the mode is inert — four residuals cannot support six unknowns, so the
+geolocator pins altitude regardless and `_solve_best_altitude_n2` is unchanged.
+Trimming re-solves through `_solve_best_altitude`, so a trim round inherits
+whichever mode its first solve used.
+
+Both modes stamp `altitude_mode` (`"free"` / `"pinned"`) on every
+`mlat_solve_history` record, published or rejected; `free` adds `alt_starts_km`,
+`alt_start_rms_us` (each start's residual) and `z_saturated` (the altitude
+analogue of `vz_saturated` — z stopped on a bound rather than converging, so
+`alt_m` is the bound and not a fit). That is the comparison channel: deploy one
+mode per environment and read the two lanes' `rms_delay` and `gt_error_km` off
+`/api/test/mlat-history`.
+
+| Mode | Pool calls per n>=3 candidate | Altitude |
|---|---|---|
-| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py:701` |
-| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py:744` |
-| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py:160-162` |
-| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py:132` |
-| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py:173` |
-| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py:205` |
-| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py:56` |
-| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py:807` |
-| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py:154` |
-| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py:51,52` |
-| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py:57,63` |
+| `sweep` (default) | 6 (one per layer) | quantised to the nearest layer |
+| `free` | 1 (`SOLVER_FREE_ALT_STARTS` starts inside it, 1 by default) | solved, 0.05–20 km |
+
+| Constant | Value | Defined in |
+|---|---|---|
+| `_SOLVER_MAX_QUEUE_AGE_S` (6.1) | 45.0 s | `services/tasks/solver.py` |
+| `SOLVER_RESOLVE_INTERVAL_S` (6.2) | 12 s (0 disables) | `services/tasks/solver.py` (`_SOLVER_RESOLVE_INTERVAL_S`) |
+| `_TRIM_MAX_ROUNDS` / `_TRIM_RESID_FACTOR` / `_TRIM_MIN_NODES` (6.4) | 4 / 1.5 / 3 | `services/tasks/solver.py` |
+| `SOLVER_RMS_DELAY_MAX_US` (6.5) | 3.0 us | `services/tasks/solver.py` (`_SOLVER_RMS_DELAY_MAX_US`) |
+| `_SOLVER_RMS_DOPPLER_MAX_HZ` (6.6) | 200.0 Hz (hardcoded) | `services/tasks/solver.py` |
+| `_MAX_DISPLACEMENT_KM` (6.8) | 2.0 km | `services/tasks/solver.py` |
+| `N2_CONFIRM_CHI2_MAX` (6.9) | 2.0 | `config/constants.py` |
+| `_TRACK_CLAIM_TTL_S` (6.10) | 60.0 s | `services/tasks/solver.py` |
+| `_CONSENSUS_MIN_NODES` | 3 | `services/tasks/solver.py` |
+| `_SIGMA_DELAY_US` / `_SIGMA_DOPPLER_HZ` | 0.1 / 2.0 | `multinode_solver.py` |
+| `_V_BOUND_MS` / `_VZ_BOUND_MS` | 300.0 / 20.0 m/s | `multinode_solver.py` |
---
@@ -525,26 +582,61 @@ flowchart TD
| Value | Set at | Meaning |
|---|---|---|
-| `multinode_solve` | `aircraft_feed.py:132` | published multi-node solve |
-| `solver_adsb_seed` | `track_gates.py:330` | single-node LM with fresh ADS-B fix |
-| `solver_single_node` | `track_gates.py:330` | single-node LM, no ADS-B |
-| `single_node_ellipse_arc` | `track_gates.py:378` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint |
-| `adsb_single_node` | `aircraft_feed.py:_claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` |
-| `known_lane_truth_match` / `known_lane_ghost` | `known_lane.py:260` | accuracy-sample-only, not a feed entry |
-
-| Constant | Value | File:line |
+| `multinode_solve` | `aircraft_feed.multinode_to_aircraft` | published multi-node solve |
+| `solver_adsb_seed` | `track_gates.track_entry` | single-node LM with fresh ADS-B fix |
+| `solver_single_node` | `track_gates.track_entry` | single-node LM, no ADS-B |
+| `single_node_ellipse_arc` | `track_gates.track_entry` | overwrites either when an ambiguity arc exists — displayed point is the arc midpoint |
+| `adsb_single_node` | `aircraft_feed._claimed_single_node_entries` | exactly one node claiming the hex within `CLAIMED_DISPLAY_FRESH_S`; position is the claim's ADS-B fix, the entry carries the node's full ambiguity arc. Two or more claiming nodes emit nothing here — that is the known-lane solver's `mn-adsb-` |
+| `known_lane_truth_match` / `known_lane_ghost` | `known_lane._record_accuracy` | accuracy-sample-only, not a feed entry |
+
+| Constant | Value | Defined in |
|---|---|---|
| `_MN_ASSOC_MAX_DIST_KM` / `_MN_ASSOC_MAX_AGE_S` (identity step 2/3) | 6.0 km / 60.0 s | `services/tasks/solver.py` |
| `_MN_ASSOC_DRIFT_KM_PER_S` / `_MN_ASSOC_MAX_DIST_CAP_KM` (step 3 only — the gate grows with the matched entry's age) | 0.13 km/s / 12.0 km | `services/tasks/solver.py` |
| Supersession gate (`_supersession_match`) — the same age-scaled `_mn_assoc_gate_km` and `_MN_ASSOC_MAX_AGE_S` as step 3, applied to the solve's RAW position | 6.0 + 0.13·dt km, cap 12.0 / 60.0 s | `services/tasks/solver.py` |
-| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py:77` |
-| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py:63` |
-| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py:66` |
-| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py:37-43` |
-| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py:131-139` |
-| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py:49-50` |
-| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py:167` |
-| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py:206,213` |
+| `CV_VEL_ADOPT_CHI2_MAX` | 5.0 | `config/constants.py` |
+| `MN_N2_MIN_SOLVES` | 2 | `config/constants.py` |
+| `MN_ONESHOT_TTL_S` | 15.0 s | `config/constants.py` |
+| `_DEDUP_SOURCE_RANK` order | multinode_solve 0 < adsb_single_node 1 < solver_adsb_seed 2 < solver_single_node 3 < single_node_ellipse_arc 4 | `services/feed_helpers.py` |
+| `CLAIMED_DISPLAY_FRESH_S` | 5.0 s | `config/constants.py` |
+| Dedup proximity / altitude gate | 3.0 km / 2000 ft | `services/feed_helpers.py` (`_DEDUP_PROXIMITY_KM`, `_DEDUP_ALT_GATE_FT`) |
+| `AIRCRAFT_FLUSH_INTERVAL_S` | 1.0 s | `config/constants.py` |
+| `DISPLAY_STALE_TRACK_S` / `GATE_MAX_HOLD_S` | 15 s / 10 s | `config/constants.py` |
+
+---
+
+## 7. Reading the pipeline from outside
+
+Three endpoints answer questions about the two lanes, and each has a shape
+worth knowing before it is trusted.
+
+**`/api/test/mlat-history`** dumps solve records. Both lanes write their own
+deque (`state.mlat_solve_history`, `state.mlat_solve_history_known`) and every
+reader merges them. `?lane=dark|known|adsb|all` narrows the answer;
+`?limit=` (default 1 000, max 5 000) is applied **per lane**, so a known-lane
+burst can never push dark records out of the response — the flat cap that
+preceded it left a 30 min request holding only the newest ~6 min of dark
+records, which reads exactly like a quiet dark lane. `lane_counts` is
+reported pre-cap so a truncated `records` list is legible.
+`?kind=resolve_skips` dumps a different store entirely — see below.
+
+**`/api/test/solver-stats`** is the Solver Report panel's source. Its funnel,
+error percentiles, ghosts, fragmentation, `contamination` and `resolve_skips`
+are all the DARK lane; `lane_split` gives the per-lane record counts and
+`known_lane` that lane's own numbers.
+
+| Block | Says | Watch for |
+|---|---|---|
+| `contamination` | Of the dark records that matched ground truth, how many carried a node that could not see the aircraft (`foreign_node_ids` on the record; verdict is the associator's own `_point_in_beam`, the same gate known-lane claiming uses) | `pct` is the live version of the offline ~60 % the cluster-splitting work exists to move. Records with no GT match, or no registered geometry for any contributing node, are **out of the denominator** — abstention, not innocence |
+| `resolve_skips` | Candidates the re-solve suppression refused in this window, from `state.solver_resolve_skips_recent`, with the claims that blocked each one | `attempts_ratio` is all-lane skips over DARK attempts (live baseline ~2.4). The deque holds 500 entries against ~50 skips/min, so read `window_effective_minutes` before reading `total` as a window count |
+| `counters.resolve_skips_dark` | Dark share of the since-boot skip counter | — |
+| `counters.node_frames_rate_limited` | Frames `NODE_FRAME_MIN_INTERVAL_S` refused before the tracker saw them (Gate B in §2) | Not the same event as `/api/admin/metrics`' `frames_dropped`, which is `frame_queue` saturation and normally reads zero |
+
+A skip is deliberately **not** a solve-history record: skips outrun dark
+records roughly two to one on the live fleet, so writing them into
+`mlat_solve_history` would evict exactly the solves an investigation needs.
+They are also not counted as attempts or rejects — a skipped candidate never
+reached a solve.
---
@@ -566,15 +658,15 @@ flowchart TD
(`fragmentation`) and `superseded_keys` / `superseded_blocked` on each
published `mlat_solve_history` record are how this is watched.
- **Node-trust residuals are measure-only.** `node_bias.py` computes them but
- nothing in the solver consumes them yet (`node_bias.py:33-40` docstring).
+ nothing in the solver consumes them yet (`node_bias.py` module docstring).
- **`docs/pipeline.md` §3 is stale.** It predates the known lane and the
process-pool inline fallback; this doc supersedes it for both topics.
- **The bottom-up doppler gate is inert.** `doppler_gate_hz` in the dark
lane's coarse pairing step is defined but the grid gate is delay-only in
- practice (`libs/retina-analytics/.../association.py:884`).
+ practice (`association.compute_overlap_zone`'s `doppler_gate_hz`).
- **Production runs with every mode flag off** except `KNOWN_LANE_MODE`, which
is `binding` everywhere by code default and is set in no environment's
`.env`. The in-repo statement of what each environment sets is
- [`architecture.md:94-110`](architecture.md#feature-gates); the actual
+ [`architecture.md`](architecture.md#feature-gates); the actual
values live in the gitignored `backend/.env` on each host, not in this
repo.
diff --git a/libs/retina-analytics b/libs/retina-analytics
index 14504176..4439c2c7 160000
--- a/libs/retina-analytics
+++ b/libs/retina-analytics
@@ -1 +1 @@
-Subproject commit 145041767422723d89d98ec003f843347ddbb880
+Subproject commit 4439c2c7d2d4fb49ca6646000f3bedc25f9701c4
diff --git a/libs/retina-geolocator b/libs/retina-geolocator
index 2da39822..6979943a 160000
--- a/libs/retina-geolocator
+++ b/libs/retina-geolocator
@@ -1 +1 @@
-Subproject commit 2da3982220374f05dd622633576108e57bd9e34f
+Subproject commit 6979943aa7ae3d05e61a9cb355490eafcc8b45b3